diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml index afd897817904947..e15729fac3443db 100644 --- a/.azure-pipelines/windows-layout-steps.yml +++ b/.azure-pipelines/windows-layout-steps.yml @@ -12,7 +12,7 @@ steps: displayName: Show layout info (${{ parameters.kind }}) - ${{ if eq(parameters.fulltest, 'true') }}: - - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" -i test_launcher + - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) displayName: ${{ parameters.kind }} Tests env: diff --git a/.editorconfig b/.editorconfig index 25bc5935258bd14..ab1f7ce8425769e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,toml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h,gram}] +[*.{py,c,cpp,h,toml,gram}] indent_size = 4 [*.rst] diff --git a/.gitattributes b/.gitattributes index b8189f12ded0f44..7ca49c3e44fe4bd 100644 --- a/.gitattributes +++ b/.gitattributes @@ -19,7 +19,7 @@ *.zip binary # Specific binary files -PC/classicAppCompat.* binary +# -- None right now -- # Text files that should not be subject to eol conversion [attr]noeol -text @@ -34,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf @@ -77,11 +80,12 @@ Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated +Include/internal/pycore_token.h generated Include/internal/pycore_uop_ids.h generated Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated Include/opcode_ids.h generated -Include/token.h generated +Include/slots_generated.h generated Lib/_opcode_metadata.py generated Lib/idlelib/help.html generated Lib/keyword.py generated @@ -96,7 +100,6 @@ Lib/token.py generated Misc/sbom.spdx.json generated Modules/_testinternalcapi/test_cases.c.h generated Modules/_testinternalcapi/test_targets.h generated -Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated Parser/token.c generated @@ -107,6 +110,7 @@ Python/generated_cases.c.h generated Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/record_functions.c.h generated +Python/slots_generated.c generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index af904a567cfb7e2..4d9e73db48ae57c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -63,17 +63,17 @@ .azure-pipelines/ @AA-Turner # GitHub & related scripts -.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz +.github/ @ezio-melotti @hugovk @AA-Turner @webknjaz @itamaro @JacobCoffee Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz Tools/build/verify_ensurepip_wheels.py @AA-Turner @pfmoore @pradyunsg # Pre-commit -.pre-commit-config.yaml @hugovk -.ruff.toml @hugovk @AlexWaygood @AA-Turner +.pre-commit-config.yaml @hugovk @JacobCoffee +.ruff.toml @hugovk @AlexWaygood @AA-Turner @JacobCoffee # Patchcheck -Tools/patchcheck/ @AA-Turner +Tools/patchcheck/ @AA-Turner @itamaro # ---------------------------------------------------------------------------- @@ -81,11 +81,11 @@ Tools/patchcheck/ @AA-Turner # ---------------------------------------------------------------------------- # Autotools -configure* @erlend-aasland @corona10 @AA-Turner @emmatyping -Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping -Modules/makesetup @erlend-aasland @AA-Turner @emmatyping -Modules/Setup* @erlend-aasland @AA-Turner @emmatyping -Tools/build/regen-configure.sh @AA-Turner +configure* @erlend-aasland @corona10 @AA-Turner @emmatyping @itamaro +Makefile.pre.in @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/makesetup @erlend-aasland @AA-Turner @emmatyping @itamaro +Modules/Setup* @erlend-aasland @AA-Turner @emmatyping @itamaro +Tools/build/regen-configure.sh @AA-Turner @itamaro # generate-build-details Tools/build/generate-build-details.py @FFY00 @@ -156,7 +156,7 @@ Misc/libabigail.abignore @encukou # ---------------------------------------------------------------------------- # Android -Android/ @mhsmith @freakboy3742 +Platforms/Android/ @mhsmith @freakboy3742 Doc/using/android.rst @mhsmith @freakboy3742 Lib/_android_support.py @mhsmith @freakboy3742 Lib/test/test_android.py @mhsmith @freakboy3742 @@ -164,20 +164,16 @@ Lib/test/test_android.py @mhsmith @freakboy3742 # iOS Doc/using/ios.rst @freakboy3742 Lib/_ios_support.py @freakboy3742 -Apple/ @freakboy3742 -iOS/ @freakboy3742 +Platforms/Apple/ @freakboy3742 # macOS Mac/ @python/macos-team Lib/_osx_support.py @python/macos-team Lib/test/test__osx_support.py @python/macos-team -# WebAssembly -Tools/wasm/README.md @brettcannon @freakboy3742 @emmatyping - # WebAssembly (Emscripten) -Tools/wasm/config.site-wasm32-emscripten @freakboy3742 @emmatyping -Tools/wasm/emscripten @freakboy3742 @emmatyping +Platforms/emscripten @freakboy3742 @emmatyping +Tools/wasm/emscripten @freakboy3742 @emmatyping # WebAssembly (WASI) Platforms/WASI @brettcannon @emmatyping @savannahostrowski @@ -189,12 +185,8 @@ Tools/wasm/wasi @brettcannon @emmatyping @savannahostrowski PC/ @python/windows-team PCbuild/ @python/windows-team -# Windows installer packages -Tools/msi/ @python/windows-team -Tools/nuget/ @python/windows-team - -# Windows Launcher -PC/launcher.c @python/windows-team @vsajip +# Windows Venv launcher/redirector +PC/venvlauncher.c @python/windows-team @vsajip # ---------------------------------------------------------------------------- @@ -231,7 +223,7 @@ Tools/cases_generator/ @markshannon Python/assemble.c @markshannon @iritkatriel Python/codegen.c @markshannon @iritkatriel Python/compile.c @markshannon @iritkatriel -Python/flowgraph.c @markshannon @iritkatriel +Python/flowgraph.c @markshannon @iritkatriel @eclips4 Python/instruction_sequence.c @iritkatriel Python/symtable.c @JelleZijlstra @carljm @@ -340,7 +332,6 @@ Modules/_remote_debugging/ @pablogsal # Sub-Interpreters **/*crossinterp* @ericsnowcurrently -**/*interpreteridobject.* @ericsnowcurrently Doc/library/concurrent.interpreters.rst @ericsnowcurrently Lib/concurrent/futures/interpreter.py @ericsnowcurrently Lib/concurrent/interpreters/ @ericsnowcurrently @@ -574,9 +565,9 @@ Lib/shutil.py @giampaolo Lib/test/test_shutil.py @giampaolo # Site -Lib/site.py @FFY00 -Lib/test/test_site.py @FFY00 -Doc/library/site.rst @FFY00 +Lib/site.py @FFY00 @warsaw +Lib/test/test_site.py @FFY00 @warsaw +Doc/library/site.rst @FFY00 @warsaw # string.templatelib Doc/library/string.templatelib.rst @lysnikolaou @AA-Turner @@ -587,10 +578,10 @@ Lib/test/test_string/test_templatelib.py @lysnikolaou @AA-Turner **/*sysconfig* @FFY00 # SQLite 3 -Doc/library/sqlite3.rst @berkerpeksag @erlend-aasland -Lib/sqlite3/ @berkerpeksag @erlend-aasland -Lib/test/test_sqlite3/ @berkerpeksag @erlend-aasland -Modules/_sqlite/ @berkerpeksag @erlend-aasland +Doc/library/sqlite3.rst @erlend-aasland +Lib/sqlite3/ @erlend-aasland +Lib/test/test_sqlite3/ @erlend-aasland +Modules/_sqlite/ @erlend-aasland # Subprocess Lib/subprocess.py @gpshead @@ -607,6 +598,12 @@ Doc/library/tomllib.rst @encukou @hauntsaninja Lib/test/test_tomllib/ @encukou @hauntsaninja Lib/tomllib/ @encukou @hauntsaninja +# Turtle +Doc/library/turtle.rst @StanFromIreland +Doc/library/turtle-star.png @StanFromIreland +Lib/test/test_turtle.py @StanFromIreland +Lib/turtle.py @StanFromIreland + # Typing Doc/library/typing.rst @JelleZijlstra @AlexWaygood Lib/test/test_typing.py @JelleZijlstra @AlexWaygood @@ -623,15 +620,18 @@ Modules/_typesmodule.c @AA-Turner Lib/unittest/mock.py @cjw296 Lib/test/test_unittest/testmock/ @cjw296 -# Urllib -**/*robotparser* @berkerpeksag - # Venv **/*venv* @vsajip @FFY00 # Weakref **/*weakref* @kumaraditya303 +# Zlib +Doc/library/zlib.rst @StanFromIreland +Lib/compression/zlib.py @StanFromIreland +Lib/test/test_zlib.py @StanFromIreland +Modules/_zlibmodule.c @StanFromIreland + # Zipfile.Path Lib/test/test_zipfile/_path/ @jaraco Lib/zipfile/_path/ @jaraco @@ -654,3 +654,6 @@ Modules/**/clinic/ Objects/**/clinic/ PC/**/clinic/ Python/**/clinic/ + +# Exclude HTML IDs list +Doc/tools/removed-ids.txt diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index da70710b7ecfa39..177615621f6b8c7 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -34,13 +34,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml index 470ad581367b103..81ae91e5b0af973 100644 --- a/.github/ISSUE_TEMPLATE/crash.yml +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -27,13 +27,13 @@ body: label: "CPython versions tested on:" multiple: true options: - - "3.9" - "3.10" - "3.11" - "3.12" - "3.13" - "3.14" - "3.15" + - "3.16" - "CPython main branch" validations: required: true diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 923720bce0bc3bf..5bf4b40947bea70 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -1,17 +1,24 @@ # Security Policy -## Supported Versions +Python [provides a security policy and threat model](https://devguide.python.org/security/policy/) +in the Python Developer's Guide documenting what bugs are vulnerabilities, +how to structure reports, and what versions of Python accept reports. -The Python team applies security fixes according to the table -in [the devguide]( -https://devguide.python.org/versions/#supported-versions -). +Python Security Response Team (PSRT) members +balance security work against many other responsibilities. Please be thoughtful +about the time and attention your report requires. Repeated failure to respect +the security policy will result in future reports being rejected, or the +reporter being banned from the ``python`` GitHub organization, regardless of +technical merit. ## Reporting a Vulnerability -Please read the guidelines on reporting security issues [on the -official website](https://www.python.org/dev/security/) for -instructions on how to report a security-related problem to -the Python team responsibly. +The [Python security policy](https://devguide.python.org/security/policy/) +documents [how to submit a vulnerability report](https://devguide.python.org/security/policy/#how-to-submit-a-vulnerability-report) +using GitHub Security Advisories. Please read the security policy +prior to filing a vulnerability report, especially the section on [what information to +include and exclude](https://devguide.python.org/security/policy/#what-to-include-and-how-to-structure-a-vulnerability-report) +in vulnerability reports. Following the security policy means the PSRT can +quickly and efficiently triage your report, not following the security policy +will only delay triaging your report. -To reach the response team, email `security at python dot org`. diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index eacfff24889021b..61ee08db9919bb9 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,5 +1,12 @@ config-variables: null +# Pending release of actionlint > 1.7.12 for ubuntu-26.04* support: +# https://github.com/rhysd/actionlint/pull/683 +self-hosted-runner: + labels: + - ubuntu-26.04 + - ubuntu-26.04-arm + paths: .github/workflows/**/*.yml: ignore: diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 00b7ae50cb99356..55095f17bb4cce9 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,16 +12,17 @@ on: # Only ever run once - opened -permissions: {} +permissions: + contents: read jobs: add-header: - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c34f8f699d8edb6..7fdc81ae0ade1d2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,8 @@ on: - 'main' - '3.*' -permissions: {} +permissions: + contents: read concurrency: # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency @@ -52,7 +53,7 @@ jobs: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 container: image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 @@ -63,7 +64,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 1 persist-credentials: false @@ -95,12 +96,12 @@ jobs: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -180,22 +181,6 @@ jobs: free-threading: ${{ matrix.free-threading }} interpreter: ${{ matrix.interpreter }} - build-windows-msi: - # ${{ '' } is a hack to nest jobs under the same sidebar category. - name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] - needs: build-context - if: fromJSON(needs.build-context.outputs.run-windows-msi) - strategy: - fail-fast: false - matrix: - arch: - - x86 - - x64 - - arm64 - uses: ./.github/workflows/reusable-windows-msi.yml - with: - arch: ${{ matrix.arch }} - build-macos: name: >- macOS @@ -205,16 +190,16 @@ jobs: strategy: fail-fast: false matrix: - # macos-26 is Apple Silicon, macos-15-intel is Intel. - # macos-15-intel only runs tests against the GIL-enabled CPython. + # macos-26 is Apple Silicon, macos-26-intel is Intel. + # macos-26-intel only runs tests against the GIL-enabled CPython. os: - macos-26 - - macos-15-intel + - macos-26-intel free-threading: - false - true exclude: - - os: macos-15-intel + - os: macos-26-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: @@ -238,18 +223,18 @@ jobs: - false - true os: - - ubuntu-24.04 - - ubuntu-24.04-arm + - ubuntu-26.04 + - ubuntu-26.04-arm exclude: # Do not test BOLT with free-threading, to conserve resources - bolt: true free-threading: true # BOLT currently crashes during instrumentation on aarch64 - - os: ubuntu-24.04-arm + - os: ubuntu-26.04-arm bolt: true include: # Enable CPU-intensive tests on ARM (default build only) - - os: ubuntu-24.04-arm + - os: ubuntu-26.04-arm bolt: false free-threading: false test-opts: '-u cpu' @@ -260,6 +245,12 @@ jobs: os: ${{ matrix.os }} test-opts: ${{ matrix.test-opts || '' }} + build-ubuntu-installed: + name: 'Ubuntu (installed)' + needs: build-context + if: needs.build-context.outputs.run-ubuntu == 'true' + uses: ./.github/workflows/reusable-install.yml + build-ubuntu-ssltests: name: 'Ubuntu SSL tests' runs-on: ${{ matrix.os }} @@ -269,7 +260,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04] + os: [ubuntu-26.04] ssllib: # See Tools/ssl/make_ssl_data.py for notes on adding a new version ## OpenSSL @@ -277,20 +268,20 @@ jobs: # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs # supported by important vendors such as AWS-LC. - { name: openssl, version: 1.1.1w } - - { name: openssl, version: 3.0.19 } - - { name: openssl, version: 3.3.6 } - - { name: openssl, version: 3.4.4 } - - { name: openssl, version: 3.5.5 } - - { name: openssl, version: 3.6.1 } + - { name: openssl, version: 3.0.22 } + - { name: openssl, version: 3.4.7 } + - { name: openssl, version: 3.5.8 } + - { name: openssl, version: 3.6.4 } + - { name: openssl, version: 4.0.2 } ## AWS-LC - - { name: aws-lc, version: 1.68.0 } + - { name: aws-lc, version: 5.5.0 } env: SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -345,38 +336,29 @@ jobs: - arch: aarch64 runs-on: macos-26 - arch: x86_64 - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Build and test - run: ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android + run: python3 Platforms/Android ci --fast-ci ${{ matrix.arch }}-linux-android build-ios: name: iOS needs: build-context if: needs.build-context.outputs.run-ios == 'true' timeout-minutes: 60 - runs-on: macos-14 + runs-on: macos-26 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - # GitHub recommends explicitly selecting the desired Xcode version: - # https://github.com/actions/runner-images/issues/12541#issuecomment-3083850140 - # This became a necessity as a result of - # https://github.com/actions/runner-images/issues/12541 and - # https://github.com/actions/runner-images/issues/12751. - - name: Select Xcode version - run: | - sudo xcode-select --switch /Applications/Xcode_15.4.app - - name: Build and test - run: python3 Platforms/Apple ci iOS --fast-ci --simulator 'iPhone SE (3rd generation),OS=17.5' + run: python3 Platforms/Apple ci iOS --fast-ci build-emscripten: name: 'Emscripten' @@ -392,15 +374,15 @@ jobs: test-hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.5.5 + OPENSSL_VER: 3.5.8 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Register gcc problem matcher @@ -503,13 +485,13 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-24.04] + os: [ubuntu-26.04] env: - OPENSSL_VER: 3.5.5 + OPENSSL_VER: 3.5.8 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -518,10 +500,6 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@a2861a8b8538f49cf2850980acccf6b05a1b2ae4 # v2.0 - with: - version: 10 - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" @@ -571,12 +549,12 @@ jobs: cross-build-linux: name: Cross build Linux - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -600,6 +578,9 @@ jobs: run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - name: Install cross Python run: make -j8 install + - name: Display build info + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test.pythoninfo - name: Run test subset with host build run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed @@ -612,6 +593,7 @@ jobs: needs.build-context.outputs.run-ci-fuzz == 'true' || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: + contents: read security-events: write strategy: fail-fast: false @@ -658,9 +640,9 @@ jobs: - check-generated-files - check-c-api-docs - build-windows - - build-windows-msi - build-macos - build-ubuntu + - build-ubuntu-installed - build-ubuntu-ssltests - build-ios - build-emscripten @@ -679,7 +661,6 @@ jobs: allowed-failures: >- build-android, build-emscripten, - build-windows-msi, build-ubuntu-ssltests, test-hypothesis, cifuzz, @@ -711,6 +692,7 @@ jobs: !fromJSON(needs.build-context.outputs.run-ubuntu) && ' build-ubuntu, + build-ubuntu-installed, build-ubuntu-ssltests, test-hypothesis, build-asan, diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index 19314dd0c889b0a..000000000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 81d75ef18209039..813589c2bc14c8b 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -15,7 +15,8 @@ on: paths: *paths workflow_dispatch: -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -28,12 +29,15 @@ env: jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -60,15 +64,15 @@ jobs: include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-2025-vs2026 + runner: windows-2025 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-2025-vs2026 + runner: windows-2025 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -98,11 +102,11 @@ jobs: - false include: - target: x86_64-apple-darwin/clang - runner: macos-15-intel + runner: macos-26-intel - target: aarch64-apple-darwin/clang - runner: macos-15 + runner: macos-26 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -141,20 +145,22 @@ jobs: - false include: - target: x86_64-unknown-linux-gnu/gcc - runner: ubuntu-24.04 + runner: ubuntu-26.04 - target: aarch64-unknown-linux-gnu/gcc - runner: ubuntu-24.04-arm + runner: ubuntu-26.04-arm steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} - export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + # On ubuntu-26.04 image, clang is clang-21 by default ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 - name: Test @@ -164,7 +170,7 @@ jobs: linux-extras: name: ${{ matrix.name }} - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 strategy: fail-fast: false @@ -176,21 +182,26 @@ jobs: - name: JIT without optimizations (Debug) configure_flags: --enable-experimental-jit --with-pydebug test_env: "PYTHON_UOPS_OPTIMIZE=0" + - name: JIT with stress testing (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_JIT_STRESS=1" - name: JIT with tail calling interpreter configure_flags: --enable-experimental-jit --with-tail-call-interp --with-pydebug use_clang: true run_tests: false steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} - export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + # On ubuntu-26.04 image, clang is clang-21 by default if [ "${{ matrix.use_clang }}" = "true" ]; then export CC=clang-${{ env.LLVM_VERSION }} fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index fb2b94b7362308e..8a79ea20d5f50bc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -2,7 +2,8 @@ name: Lint on: [push, pull_request, workflow_dispatch] -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -18,7 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: j178/prek-action@0bb87d7f00b0c99306c8bcb8b8beba1eb581c037 # v1.1.1 + - uses: j178/prek-action@bdca6f102f98e2b4c7029491a53dfd366469e33d # v2.0.4 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 583dc1808dfc35d..3cdce4f5952e3d0 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -19,6 +19,7 @@ on: - "Tools/build/consts_getter.py" - "Tools/build/deepfreeze.py" - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate_stdlib_module_names.py" - "Tools/build/mypy.ini" @@ -33,7 +34,8 @@ on: - "Tools/requirements-dev.txt" workflow_dispatch: -permissions: {} +permissions: + contents: read env: PIP_DISABLE_PIP_VERSION_CHECK: 1 @@ -64,14 +66,14 @@ jobs: "Tools/peg_generator", ] steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: - python-version: "3.13" - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + python-version: "3.15" + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index be375a970a475cd..864d0f48904bcc3 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -5,11 +5,12 @@ on: types: - opened -permissions: {} +permissions: + contents: read jobs: notify-new-bugs-announce: - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: issues: read timeout-minutes: 10 @@ -19,7 +20,7 @@ jobs: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 7994a01ee4624ed..803204635abfe11 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -5,30 +5,23 @@ apt-get -yq --no-install-recommends install \ build-essential \ pkg-config \ cmake \ + curl \ gdb \ lcov \ libb2-dev \ libbz2-dev \ libffi-dev \ - libgdbm-dev \ libgdbm-compat-dev \ + libgdbm-dev \ liblzma-dev \ + libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ libzstd-dev \ - lzma \ - lzma-dev \ strace \ tk-dev \ uuid-dev \ xvfb \ zlib1g-dev - -# Workaround missing libmpdec-dev on ubuntu 24.04: -# https://launchpad.net/~ondrej/+archive/ubuntu/php -# https://deb.sury.org/ -sudo add-apt-repository ppa:ondrej/php -apt-get update -apt-get -yq --no-install-recommends install libmpdec-dev diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 262299fc30f9899..8af254c10786e1b 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,13 +4,14 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: {} +permissions: + contents: read jobs: label-dnm: name: DO-NOT-MERGE if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read timeout-minutes: 10 @@ -27,7 +28,7 @@ jobs: label-reviews: name: Unresolved review if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read timeout-minutes: 10 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml index 5fae57a1dbda362..db030c80008b1ed 100644 --- a/.github/workflows/reusable-check-c-api-docs.yml +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -3,7 +3,8 @@ name: Reusable C API Docs Check on: workflow_call: -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -14,7 +15,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/reusable-check-html-ids.yml b/.github/workflows/reusable-check-html-ids.yml index 03ed714ca585fe9..41ba1288be1ecf4 100644 --- a/.github/workflows/reusable-check-html-ids.yml +++ b/.github/workflows/reusable-check-html-ids.yml @@ -3,7 +3,8 @@ name: Reusable check HTML IDs on: workflow_call: -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -15,7 +16,7 @@ jobs: timeout-minutes: 30 steps: - name: 'Check out PR head' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false ref: ${{ github.event.pull_request.head.sha }} diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml index 093b2c859eff7b4..0d02232686339bf 100644 --- a/.github/workflows/reusable-cifuzz.yml +++ b/.github/workflows/reusable-cifuzz.yml @@ -13,7 +13,8 @@ on: required: true type: string -permissions: {} +permissions: + contents: read jobs: cifuzz: diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index cc9841ebf32f27d..c998cbff181dd12 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -47,14 +47,12 @@ on: # yamllint disable-line rule:truthy run-wasi: description: Whether to run the WASI tests value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool - run-windows-msi: - description: Whether to run the MSI installer smoke tests - value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool run-windows-tests: description: Whether to run the Windows tests value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool -permissions: {} +permissions: + contents: read jobs: compute-changes: @@ -72,7 +70,6 @@ jobs: run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} run-emscripten: ${{ steps.changes.outputs.run-emscripten }} run-wasi: ${{ steps.changes.outputs.run-wasi }} - run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python @@ -83,7 +80,7 @@ jobs: - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false ref: >- diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 3d534feb2ed3ea3..199e0fd8d181f0b 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -3,8 +3,20 @@ name: Reusable Docs on: workflow_call: workflow_dispatch: + # Pushes to CPython branches seed the pip caches under the exact keys every + # docs PR restores from. Without a branch-scoped copy, each PR saves a + # duplicate into its own refs/pull/N/merge scope that nothing else can read. + push: + branches: + - main + - '3.*' + paths: + - 'Doc/pylock.toml' + - 'Doc/requirements.txt' + - '.github/workflows/reusable-docs.yml' -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -26,7 +38,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false ref: >- @@ -55,7 +67,7 @@ jobs: with: python-version: '3' cache: 'pip' - cache-dependency-path: 'Doc/requirements.txt' + cache-dependency-path: 'Doc/pylock.toml' - name: 'Install build dependencies' run: make -C Doc/ venv @@ -94,10 +106,10 @@ jobs: # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 @@ -123,7 +135,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: 'Set up Python' diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml index 300731deb78959e..c8832342d9892da 100644 --- a/.github/workflows/reusable-emscripten.yml +++ b/.github/workflows/reusable-emscripten.yml @@ -3,7 +3,8 @@ name: Reusable Emscripten on: workflow_call: -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -11,10 +12,10 @@ env: jobs: build-emscripten-reusable: name: 'build and test' - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 40 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: "Read Emscripten config" @@ -62,15 +63,19 @@ jobs: run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" run: python3 Platforms/emscripten make-build-python + - name: "Display build info of the build Python" + run: python3 Platforms/emscripten pythoninfo-build - name: "Make dependencies" run: >- python3 Platforms/emscripten make-dependencies ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} - - name: "Configure host Python" + - name: "Configure host/Emscripten Python" run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache - - name: "Make host Python" + - name: "Make host/Emscripten Python" run: python3 Platforms/emscripten make-host - - name: "Display build info" - run: python3 Platforms/emscripten run --pythoninfo + - name: "Display build info of the host/Emscripten Python" + run: python3 Platforms/emscripten pythoninfo-host - name: "Test" run: python3 Platforms/emscripten run --test + - name: "Test Repl" + run: Platforms/emscripten/browser_test/run_test.sh diff --git a/.github/workflows/reusable-install.yml b/.github/workflows/reusable-install.yml new file mode 100644 index 000000000000000..337da684882d29f --- /dev/null +++ b/.github/workflows/reusable-install.yml @@ -0,0 +1,44 @@ +name: Reusable Ubuntu (installed) + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build-install-test: + name: build, install and test + runs-on: ubuntu-26.04-arm + timeout-minutes: 60 + env: + PYTHONSTRICTEXTENSIONBUILD: 1 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Set install dir + run: + echo "INSTALL_DIR=$(realpath "${GITHUB_WORKSPACE}/../installed-python")" >> "$GITHUB_ENV" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure CPython + run: ./configure --config-cache --prefix="$INSTALL_DIR" + - name: Build CPython + run: make -j + - name: Install CPython + run: make install + - name: Set installed interpreter + run: | + ldversion=$(./python -c 'import sysconfig; print(sysconfig.get_config_var("LDVERSION"))') + echo "INSTALLED_PYTHON=${INSTALL_DIR}/bin/python${ldversion}" >> "$GITHUB_ENV" + - name: Display build info + run: | + "$INSTALLED_PYTHON" -m test.pythoninfo + - name: Test the installed Python + run: xvfb-run "$INSTALLED_PYTHON" -m test --fast-ci --timeout=900 diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index a372d5715290db0..5393e9e34040b95 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -12,7 +12,8 @@ on: required: true type: string -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -30,16 +31,15 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.5 xz gdbm tcl-tk@9 make - # Because alternate versions are not symlinked into place by default: - brew link --overwrite tcl-tk@9 + brew bundle --file=Misc/Brewfile + brew install make - name: Configure CPython run: | MACOSX_DEPLOYMENT_TARGET=10.15 \ @@ -54,15 +54,15 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.5)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-15-intel' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-26-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-15-intel' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml index 33cfd578d6819a0..449bddadf8b2c7c 100644 --- a/.github/workflows/reusable-san.yml +++ b/.github/workflows/reusable-san.yml @@ -12,10 +12,12 @@ on: type: boolean default: false -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 + OPENSSL_VER: 3.5.8 jobs: build-san-reusable: @@ -25,10 +27,10 @@ jobs: && ' (free-threading)' || '' }} - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Runner image version @@ -36,38 +38,58 @@ jobs: - name: Install dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh - # Install clang - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - - if [ "${SANITIZER}" = "TSan" ]; then - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 - # Reduce ASLR to avoid TSan crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - else - sudo ./llvm.sh 20 - fi - - - name: Sanitizer option setup - run: | - if [ "${SANITIZER}" = "TSan" ]; then - echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ - fromJSON(inputs.free-threading) - && '_free_threading' - || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - else - echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" - fi + # On ubuntu-26.04 image, clang is clang-21 by default + # NOTE: when bumping to a new version of clang, + # please update the versions in `Tools/pixi-packages/variants.yaml` too. echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" + - name: TSan option setup + if: inputs.sanitizer == 'TSan' + run: | + sudo sysctl -w vm.mmap_rnd_bits=28 # Reduce ASLR to avoid TSan crashing + + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${SUPPRESSIONS_SUFFIX}.txt handle_segv=0" >> "$GITHUB_ENV" + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + SUPPRESSIONS_SUFFIX: >- + ${{ + inputs.free-threading + && '_free_threading' + || '' + }} + - name: UBSan option setup + if: inputs.sanitizer != 'TSan' + run: >- + echo + "UBSAN_OPTIONS=${SAN_LOG_OPTION} + halt_on_error=1 + suppressions=${GITHUB_WORKSPACE}/Tools/ubsan/suppressions.txt" + >> "$GITHUB_ENV" env: - SANITIZER: ${{ inputs.sanitizer }} SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build (TSan)' + id: cache-openssl + if: inputs.sanitizer == 'TSan' + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ env.IMAGE_OS_VERSION }}-multissl-openssl-tsan-${{ env.OPENSSL_VER }} + - name: Install OpenSSL (TSan) + if: >- + inputs.sanitizer == 'TSan' + && steps.cache-openssl.outputs.cache-hit != 'true' + run: >- + python3 Tools/ssl/multissltests.py + --steps=library + --base-directory="${MULTISSL_DIR}" + --openssl="${OPENSSL_VER}" + --system=Linux + --tsan - name: Configure CPython run: >- ./configure @@ -75,10 +97,11 @@ jobs: ${{ inputs.sanitizer == 'TSan' && '--with-thread-sanitizer' - || '--with-undefined-behavior-sanitizer' + || '--with-undefined-behavior-sanitizer --with-strict-overflow' }} --with-pydebug - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }} + ${{ inputs.free-threading && '--disable-gil' || '' }} - name: Build CPython run: make -j4 - name: Display build info @@ -87,12 +110,12 @@ jobs: run: >- ./python -m test ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} - -j4 + -j4 -W --timeout=900 --slowest - name: Parallel tests if: >- inputs.sanitizer == 'TSan' - && fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + && inputs.free-threading + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 -W --timeout=600 --slowest - name: Display logs if: always() run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 @@ -102,7 +125,7 @@ jobs: with: name: >- ${{ inputs.sanitizer }}-logs-${{ - fromJSON(inputs.free-threading) + inputs.free-threading && 'free-threading' || 'default' }} diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index b2ab525c976330c..97a0c22517e32ce 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -14,16 +14,17 @@ on: type: boolean default: false os: - description: OS to run the job - required: true - type: string + description: OS to run the job + required: true + type: string test-opts: - description: Extra options to pass to the test runner via TESTOPTS - required: false - type: string - default: '' + description: Extra options to pass to the test runner via TESTOPTS + required: false + type: string + default: '' -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -34,11 +35,11 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.5.5 + OPENSSL_VER: 3.5.8 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Register gcc problem matcher @@ -46,11 +47,11 @@ jobs: - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Install Clang and BOLT - if: ${{ fromJSON(inputs.bolt-optimizations) }} + if: inputs.bolt-optimizations run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 - sudo apt-get install --no-install-recommends bolt-19 - echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV + # On ubuntu-26.04 image, LLVM is LLVM-21 by default + sudo apt-get install --no-install-recommends bolt-21 + echo PATH="$(llvm-config-21 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" @@ -87,10 +88,10 @@ jobs: --enable-slower-safety --enable-safety --with-openssl="$OPENSSL_DIR" - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }} + ${{ inputs.free-threading && '--disable-gil' || '' }} + ${{ inputs.bolt-optimizations && '--enable-bolt' || '' }} - name: Build CPython out-of-tree - if: ${{ inputs.free-threading }} + if: inputs.free-threading working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j - name: Build CPython out-of-tree (for compiler warning check) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 83f9d2399ce1006..b0dda62d0b291c2 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -3,7 +3,8 @@ name: Reusable WASI on: workflow_call: -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 @@ -11,14 +12,13 @@ env: jobs: build-wasi-reusable: name: 'build and test' - runs-on: ubuntu-24.04-arm + runs-on: ubuntu-26.04-arm timeout-minutes: 60 env: WASMTIME_VERSION: 38.0.3 - CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. @@ -53,14 +53,16 @@ jobs: run: python3 Platforms/WASI configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" run: python3 Platforms/WASI make-build-python - - name: "Configure host" + - name: "Display build info of the build Python" + run: python3 Platforms/WASI pythoninfo-build + - name: "Configure host/WASI Python" # `--with-pydebug` inferred from configure-build-python run: python3 Platforms/WASI configure-host -- --config-cache env: WASI_SDK_PATH: ${{ steps.install-wasi-sdk.outputs.wasi-sdk-path }} - - name: "Make host" + - name: "Make host/WASI Python" run: python3 Platforms/WASI make-host - - name: "Display build info" - run: make --directory "${CROSS_BUILD_WASI}" pythoninfo + - name: "Display build info of the host/WASI Python" + run: python3 Platforms/WASI pythoninfo-host - name: "Test" run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml deleted file mode 100644 index 7c724f184f3ef66..000000000000000 --- a/.github/workflows/reusable-windows-msi.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Reusable Windows MSI - -on: - workflow_call: - inputs: - arch: - description: CPU architecture - required: true - type: string - -permissions: {} - -env: - FORCE_COLOR: 1 - -jobs: - build: - name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} - timeout-minutes: 60 - env: - ARCH: ${{ inputs.arch }} - IncludeFreethreaded: true - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: Build CPython installer - run: ./Tools/msi/build.bat --doc -"${ARCH}" - shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 2cfe338a6525e60..b1b711be4538fda 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -17,22 +17,21 @@ on: required: true type: string -permissions: {} +permissions: + contents: read env: FORCE_COLOR: 1 - IncludeUwp: >- - true jobs: build: name: Build and test (${{ inputs.arch }}, ${{ inputs.interpreter }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025-vs2026' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2025' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Register MSVC problem matcher @@ -46,7 +45,7 @@ jobs: -e -v ${{ inputs.interpreter == 'switch-case' && '-d' || '--tail-call-interp -c Release' }} -p "${ARCH}" - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + ${{ inputs.free-threading && '--disable-gil' || '' }} shell: bash - name: Display build info run: .\\python.bat -m test.pythoninfo @@ -56,5 +55,5 @@ jobs: -p "${ARCH}" -q --fast-ci ${{ inputs.interpreter == 'switch-case' && '-d' || '' }} - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + ${{ inputs.free-threading && '--disable-gil' || '' }} shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index a862fde5e14eb49..0283e9f621dcf05 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,12 +4,13 @@ on: schedule: - cron: "0 */6 * * *" -permissions: {} +permissions: + contents: read jobs: stale: if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: actions: write pull-requests: write @@ -20,10 +21,10 @@ jobs: uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' + stale-pr-message: 'This PR is stale because it has been open for 90 days with no activity.' stale-pr-label: 'stale' days-before-issue-stale: -1 - days-before-pr-stale: 30 + days-before-pr-stale: 90 days-before-close: -1 ascending: true operations-per-run: 120 diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 35c62acb28b761f..81e8235f236ea70 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -11,7 +11,8 @@ on: paths: *paths workflow_dispatch: -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -31,11 +32,11 @@ jobs: matrix: include: - target: x86_64-apple-darwin/clang - runner: macos-15-intel + runner: macos-26-intel - target: aarch64-apple-darwin/clang - runner: macos-15 + runner: macos-26 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -65,16 +66,16 @@ jobs: matrix: include: - target: x86_64-unknown-linux-gnu/gcc - runner: ubuntu-24.04 + runner: ubuntu-26.04 configure_flags: --with-pydebug - target: x86_64-unknown-linux-gnu/gcc-free-threading - runner: ubuntu-24.04 + runner: ubuntu-26.04 configure_flags: --disable-gil - target: aarch64-unknown-linux-gnu/gcc - runner: ubuntu-24.04-arm + runner: ubuntu-26.04-arm configure_flags: --with-pydebug steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -82,8 +83,7 @@ jobs: python-version: '3.11' - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} - export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + # On ubuntu-26.04 image, clang is clang-21 by default CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp ${{ matrix.configure_flags }} make all --jobs 4 - name: Test diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 4ac25bc909b13f9..7def8c9e78abb8d 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -13,7 +13,8 @@ on: - '.github/workflows/verify-ensurepip-wheels.yml' - 'Tools/build/verify_ensurepip_wheels.py' -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -24,7 +25,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml index e193dfa4603e8ac..b96e71b487dbe57 100644 --- a/.github/workflows/verify-expat.yml +++ b/.github/workflows/verify-expat.yml @@ -11,7 +11,8 @@ on: - 'Modules/expat/**' - '.github/workflows/verify-expat.yml' -permissions: {} +permissions: + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -19,10 +20,10 @@ concurrency: jobs: verify: - runs-on: ubuntu-latest + runs-on: ubuntu-slim timeout-minutes: 5 steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Download and verify bundled libexpat files diff --git a/.gitignore b/.gitignore index f9d2cdfc32b3835..e53548f1fd45a03 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.cover *.iml *.o +*.o.tmp *.lto *.a *.so @@ -49,7 +50,7 @@ gmon.out *.exe -# Ignore core dumps... but not Tools/msi/core/ or the like. +# Ignore core dumps... but not .../core/ subdirectories core !core/ @@ -123,10 +124,14 @@ PCbuild/*-pgo PCbuild/*.VC.db PCbuild/*.VC.opendb PCbuild/amd64/ +PCbuild/amd64t/ PCbuild/arm32/ +PCbuild/arm32t/ PCbuild/arm64/ +PCbuild/arm64t/ PCbuild/obj/ PCbuild/win32/ +PCbuild/win32t/ Tools/unicode/data/ /autom4te.cache /build/ @@ -138,7 +143,10 @@ Tools/unicode/data/ /config.status.lineno /.ccache /cross-build*/ +/dist/ /jit_stencils*.h +/jit_unwind_info*.h +.jit-stamp /platform /profile-clean-stamp /profile-run-stamp @@ -155,7 +163,6 @@ Tools/unicode/data/ /coverage/ /externals/ /htmlcov/ -Tools/msi/obj Tools/ssl/amd64 Tools/ssl/win32 Tools/freeze/test/outdir diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c77610e209ebbd8..615ba587c6a4005 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: e05c5c0818279e5ac248ac9e954431ba58865e61 # frozen: v0.15.7 + rev: 3b3f7c3f57fe9925356faf5fe6230835138be230 # frozen: v0.15.17 hooks: - id: ruff-check name: Run Ruff (lint) on Platforms/Apple/ @@ -10,6 +10,11 @@ repos: name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ + - id: ruff-check + name: Run Ruff (lint) on Lib/ + args: [--exit-non-zero-on-fix] + files: ^Lib/ + exclude: ^Lib/test/ - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] @@ -18,6 +23,11 @@ repos: name: Run Ruff (lint) on Platforms/WASI/ args: [--exit-non-zero-on-fix, --config=Platforms/WASI/.ruff.toml] files: ^Platforms/WASI/ + - id: ruff-check + name: Run Ruff (lint) on Tools/ + args: [--exit-non-zero-on-fix] + files: ^Tools/ + exclude: ^Tools/(build|clinic|i18n|peg_generator|wasm)/ - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] @@ -85,6 +95,9 @@ repos: exclude: Lib/test/tokenizedata/coding20731.py - id: end-of-file-fixer files: '^\.github/CODEOWNERS$' + - id: mixed-line-ending + args: [--fix=auto] + exclude: '^Lib/test/.*data/' - id: trailing-whitespace types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace @@ -98,7 +111,7 @@ repos: - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: 393031adb9afb225ee52ae2ccd7a5af5525e03e8 # frozen: v1.7.11 + rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 hooks: - id: actionlint diff --git a/.readthedocs.yml b/.readthedocs.yml index 0a2c3f8345367f4..038417e4bb34385 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,24 +11,50 @@ build: os: ubuntu-24.04 tools: python: "3" + apt_packages: + - jq - commands: - # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - # - # Cancel building pull requests when there aren't changes in the Doc directory. - # - # If there are no changes (git diff exits with 0) we force the command to return with 183. - # This is a special exit code on Read the Docs that will cancel the build immediately. - - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; - then - echo "No changes to Doc/ - exiting the build."; - exit 183; - fi - - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make -C Doc venv html - - mkdir _readthedocs - - mv Doc/build/html _readthedocs/html + jobs: + post_system_dependencies: + # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions + # + # Cancel building pull requests when there are no changes in the Doc + # directory or RTD configuration, or if we can't cleanly merge the base + # branch. + - | + set -eEux; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; + then + base_branch=$(wget -qO- "https://api.github.com/repos/python/cpython/pulls/$READTHEDOCS_VERSION" | jq -er ".base.ref"); + git fetch --depth=50 origin $base_branch:origin-$base_branch; + for attempt in $(seq 10); + do + if ! git merge-base HEAD origin-$base_branch; + then + git fetch --deepen=50 origin $base_branch; + else + break; + fi; + done; + if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch; + then + echo "Unsuccessful merge with '$base_branch' branch, skipping the build"; + exit 183; + fi; + if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml; + then + echo "No changes to Doc/ - skipping the build."; + exit 183; + fi; + fi; + create_environment: + - echo "Skipping default environment creation" + install: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - make -C Doc venv html + - mkdir -p "$READTHEDOCS_OUTPUT" + - mv Doc/build/html "$READTHEDOCS_OUTPUT/" diff --git a/Android/README.md b/Android/README.md deleted file mode 100644 index 0004f26e72b21c4..000000000000000 --- a/Android/README.md +++ /dev/null @@ -1,160 +0,0 @@ -# Python for Android - -If you obtained this README as part of a release package, then the only -applicable sections are "Prerequisites", "Testing", and "Using in your own app". - -If you obtained this README as part of the CPython source tree, then you can -also follow the other sections to compile Python for Android yourself. - -However, most app developers should not need to do any of these things manually. -Instead, use one of the tools listed -[here](https://docs.python.org/3/using/android.html), which will provide a much -easier experience. - - -## Prerequisites - -If you already have an Android SDK installed, export the `ANDROID_HOME` -environment variable to point at its location. Otherwise, here's how to install -it: - -* Download the "Command line tools" from . -* Create a directory `android-sdk/cmdline-tools`, and unzip the command line - tools package into it. -* Rename `android-sdk/cmdline-tools/cmdline-tools` to - `android-sdk/cmdline-tools/latest`. -* `export ANDROID_HOME=/path/to/android-sdk` - -The `android.py` script will automatically use the SDK's `sdkmanager` to install -any packages it needs. - -The script also requires the following commands to be on the `PATH`: - -* `curl` -* `java` (or set the `JAVA_HOME` environment variable) - - -## Building - -Python can be built for Android on any POSIX platform supported by the Android -development tools, which currently means Linux or macOS. - -First we'll make a "build" Python (for your development machine), then use it to -help produce a "host" Python for Android. So make sure you have all the usual -tools and libraries needed to build Python for your development machine. - -The easiest way to do a build is to use the `android.py` script. You can either -have it perform the entire build process from start to finish in one step, or -you can do it in discrete steps that mirror running `configure` and `make` for -each of the two builds of Python you end up producing. - -The discrete steps for building via `android.py` are: - -```sh -./android.py configure-build -./android.py make-build -./android.py configure-host HOST -./android.py make-host HOST -``` - -`HOST` identifies which architecture to build. To see the possible values, run -`./android.py configure-host --help`. - -To do all steps in a single command, run: - -```sh -./android.py build HOST -``` - -In the end you should have a build Python in `cross-build/build`, and a host -Python in `cross-build/HOST`. - -You can use `--` as a separator for any of the `configure`-related commands – -including `build` itself – to pass arguments to the underlying `configure` -call. For example, if you want a pydebug build that also caches the results from -`configure`, you can do: - -```sh -./android.py build HOST -- -C --with-pydebug -``` - - -## Packaging - -After building an architecture as described in the section above, you can -package it for release with this command: - -```sh -./android.py package HOST -``` - -`HOST` is defined in the section above. - -This will generate a tarball in `cross-build/HOST/dist`, whose structure is -similar to the `Android` directory of the CPython source tree. - - -## Testing - -The Python test suite can be run on Linux, macOS, or Windows. - -On Linux, the emulator needs access to the KVM virtualization interface. This may -require adding your user to a group, or changing your udev rules. On GitHub -Actions, the test script will do this automatically using the commands shown -[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). - -You can run the test suite either: - -* Within the CPython repository, after doing a build as described above. On - Windows, you won't be able to do the build on the same machine, so you'll have - to copy the `cross-build/HOST/prefix` directory from somewhere else. - -* Or by taking a release package built using the `package` command, extracting - it wherever you want, and using its own copy of `android.py`. - -The test script supports the following modes: - -* In `--connected` mode, it runs on a device or emulator you have already - connected to the build machine. List the available devices with - `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the - script like this: - - ```sh - ./android.py test --connected emulator-5554 - ``` - -* In `--managed` mode, it uses a temporary headless emulator defined in the - `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, - but more reproducible. - - We currently define two devices: `minVersion` and `maxVersion`, corresponding - to our minimum and maximum supported Android versions. For example: - - ```sh - ./android.py test --managed maxVersion - ``` - -By default, the only messages the script will show are Python's own stdout and -stderr. Add the `-v` option to also show Gradle output, and non-Python logcat -messages. - -Any other arguments on the `android.py test` command line will be passed through -to `python -m test` – use `--` to separate them from android.py's own options. -See the [Python Developer's -Guide](https://devguide.python.org/testing/run-write-tests/) for common options -– most of them will work on Android, except for those that involve subprocesses, -such as `-j`. - -Every time you run `android.py test`, changes in pure-Python files in the -repository's `Lib` directory will be picked up immediately. Changes in C files, -and architecture-specific files such as sysconfigdata, will not take effect -until you re-run `android.py make-host` or `build`. - -The testbed app can also be used to test third-party packages. For more details, -run `android.py test --help`, paying attention to the options `--site-packages`, -`--cwd`, `-c` and `-m`. - - -## Using in your own app - -See https://docs.python.org/3/using/android.html. diff --git a/Android/android.py b/Android/android.py deleted file mode 100755 index adcc7c708d95f71..000000000000000 --- a/Android/android.py +++ /dev/null @@ -1,1069 +0,0 @@ -#!/usr/bin/env python3 - -import asyncio -import argparse -import json -import os -import platform -import re -import shlex -import shutil -import signal -import subprocess -import sys -import sysconfig -from asyncio import wait_for -from contextlib import asynccontextmanager -from datetime import datetime, timezone -from enum import IntEnum, auto -from glob import glob -from os.path import abspath, basename, relpath -from pathlib import Path -from subprocess import CalledProcessError -from tempfile import TemporaryDirectory - - -SCRIPT_NAME = Path(__file__).name -ANDROID_DIR = Path(__file__).resolve().parent -PYTHON_DIR = ANDROID_DIR.parent -in_source_tree = ( - ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() -) - -ENV_SCRIPT = ANDROID_DIR / "android-env.sh" -TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" - -HOSTS = [ - "aarch64-linux-android", - "arm-linux-androideabi", - "i686-linux-android", - "x86_64-linux-android", -] -APP_ID = "org.python.testbed" -DECODE_ARGS = ("UTF-8", "backslashreplace") - - -try: - android_home = Path(os.environ['ANDROID_HOME']) -except KeyError: - sys.exit("The ANDROID_HOME environment variable is required.") - -adb = Path( - f"{android_home}/platform-tools/adb" - + (".exe" if os.name == "nt" else "") -) - -gradlew = Path( - f"{TESTBED_DIR}/gradlew" - + (".bat" if os.name == "nt" else "") -) - -# Whether we've seen any output from Python yet. -python_started = False - -# Buffer for verbose output which will be displayed only if a test fails and -# there has been no output from Python. -hidden_output = [] - - -# Based on android/log.h in the NDK. -class LogPriority(IntEnum): - UNKNOWN = 0 - DEFAULT = auto() - VERBOSE = auto() - DEBUG = auto() - INFO = auto() - WARN = auto() - ERROR = auto() - FATAL = auto() - SILENT = auto() - - -def log_verbose(context, line, stream=sys.stdout): - if context.verbose: - stream.write(line) - else: - hidden_output.append((stream, line)) - - -def delete_glob(pattern): - # Path.glob doesn't accept non-relative patterns. - for path in glob(str(pattern)): - path = Path(path) - print(f"Deleting {path} ...") - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink() - - -def subdir(*parts, create=False): - path = CROSS_BUILD_DIR.joinpath(*parts) - if not path.exists(): - if not create: - sys.exit( - f"{path} does not exist. Create it by running the appropriate " - f"`configure` subcommand of {SCRIPT_NAME}.") - else: - path.mkdir(parents=True) - return path - - -def run(command, *, host=None, env=None, log=True, **kwargs): - kwargs.setdefault("check", True) - if env is None: - env = os.environ.copy() - - if host: - host_env = android_env(host) - print_env(host_env) - env.update(host_env) - - if log: - print(">", join_command(command)) - return subprocess.run(command, env=env, **kwargs) - - -# Format a command so it can be copied into a shell. Like shlex.join, but also -# accepts arguments which are Paths, or a single string/Path outside of a list. -def join_command(args): - if isinstance(args, (str, Path)): - return str(args) - else: - return shlex.join(map(str, args)) - - -# Format the environment so it can be pasted into a shell. -def print_env(env): - for key, value in sorted(env.items()): - print(f"export {key}={shlex.quote(value)}") - - -def android_env(host): - if host: - prefix = subdir(host) / "prefix" - else: - prefix = ANDROID_DIR / "prefix" - sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") - sysconfig_filename = next(sysconfig_files).name - host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] - - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={prefix}; " - f". {ENV_SCRIPT}; " - f"export", - check=True, shell=True, capture_output=True, encoding='utf-8', - ).stdout - - env = {} - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if os.environ.get(key) != value: - env[key] = value - - if not env: - raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n" - + env_output) - return env - - -def build_python_path(): - """The path to the build Python binary.""" - build_dir = subdir("build") - binary = build_dir / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{build_dir}") - - return binary - - -def configure_build_python(context): - if context.clean: - clean("build") - os.chdir(subdir("build", create=True)) - - command = [relpath(PYTHON_DIR / "configure")] - if context.args: - command.extend(context.args) - run(command) - - -def make_build_python(context): - os.chdir(subdir("build")) - run(["make", "-j", str(os.cpu_count())]) - - -# To create new builds of these dependencies, usually all that's necessary is to -# push a tag to the cpython-android-source-deps repository, and GitHub Actions -# will do the rest. -# -# If you're a member of the Python core team, and you'd like to be able to push -# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. -def unpack_deps(host, prefix_dir, cache_dir): - os.chdir(prefix_dir) - deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.5.5-0", - "sqlite-3.50.4-0", "xz-5.4.6-1", "zstd-1.5.7-2"]: - filename = f"{name_ver}-{host}.tar.gz" - out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir) - shutil.unpack_archive(out_path) - - -def download(url, cache_dir): - out_path = cache_dir / basename(url) - cache_dir.mkdir(parents=True, exist_ok=True) - if not out_path.is_file(): - run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) - else: - print(f"Using cached version of {basename(url)}") - return out_path - - -def configure_host_python(context, host=None): - if host is None: - host = context.host - if context.clean: - clean(host) - - host_dir = subdir(host, create=True) - prefix_dir = host_dir / "prefix" - if not prefix_dir.exists(): - prefix_dir.mkdir() - cache_dir = ( - Path(context.cache_dir).resolve() - if context.cache_dir - else CROSS_BUILD_DIR / "downloads" - ) - unpack_deps(host, prefix_dir, cache_dir) - - os.chdir(host_dir) - command = [ - # Basic cross-compiling configuration - relpath(PYTHON_DIR / "configure"), - f"--host={host}", - f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", - f"--with-build-python={build_python_path()}", - "--without-ensurepip", - - # Android always uses a shared libpython. - "--enable-shared", - "--without-static-libpython", - - # Dependent libraries. The others are found using pkg-config: see - # android-env.sh. - f"--with-openssl={prefix_dir}", - ] - - if context.args: - command.extend(context.args) - run(command, host=host) - - -def make_host_python(context, host=None): - if host is None: - host = context.host - # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so - # delete any previous Python installation to prevent it being used during - # the build. - host_dir = subdir(host) - prefix_dir = host_dir / "prefix" - for pattern in ("include/python*", "lib/libpython*", "lib/python*"): - delete_glob(f"{prefix_dir}/{pattern}") - - # The Android environment variables were already captured in the Makefile by - # `configure`, and passing them again when running `make` may cause some - # flags to be duplicated. So we don't use the `host` argument here. - os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())]) - - # The `make install` output is very verbose and rarely useful, so - # suppress it by default. - run( - ["make", "install", f"prefix={prefix_dir}"], - capture_output=not context.verbose, - ) - - -def build_targets(context): - if context.target in {"all", "build"}: - configure_build_python(context) - make_build_python(context) - - for host in HOSTS: - if context.target in {"all", "hosts", host}: - configure_host_python(context, host) - make_host_python(context, host) - - -def clean(host): - delete_glob(CROSS_BUILD_DIR / host) - - -def clean_targets(context): - if context.target in {"all", "build"}: - clean("build") - - for host in HOSTS: - if context.target in {"all", "hosts", host}: - clean(host) - - -def setup_ci(): - if "GITHUB_ACTIONS" in os.environ: - # Enable emulator hardware acceleration - # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). - if platform.system() == "Linux": - run( - ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], - input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', - text=True, - ) - run(["sudo", "udevadm", "control", "--reload-rules"]) - run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) - - # Free up disk space by deleting unused versions of the NDK - # (https://github.com/freakboy3742/pyspamsum/pull/108). - for line in ENV_SCRIPT.read_text().splitlines(): - if match := re.fullmatch(r"ndk_version=(.+)", line): - ndk_version = match[1] - break - else: - raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") - - for item in (android_home / "ndk").iterdir(): - if item.name[0].isdigit() and item.name != ndk_version: - delete_glob(item) - - -def setup_sdk(): - sdkmanager = android_home / ( - "cmdline-tools/latest/bin/sdkmanager" - + (".bat" if os.name == "nt" else "") - ) - - # Gradle will fail if it needs to install an SDK package whose license - # hasn't been accepted, so pre-accept all licenses. - if not all((android_home / "licenses" / path).exists() for path in [ - "android-sdk-arm-dbt-license", "android-sdk-license" - ]): - run( - [sdkmanager, "--licenses"], - text=True, - capture_output=True, - input="y\n" * 100, - ) - - # Gradle may install this automatically, but we can't rely on that because - # we need to run adb within the logcat task. - if not adb.exists(): - run([sdkmanager, "platform-tools"]) - - -# To avoid distributing compiled artifacts without corresponding source code, -# the Gradle wrapper is not included in the CPython repository. Instead, we -# extract it from the Gradle GitHub repository. -def setup_testbed(): - paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] - if all((TESTBED_DIR / path).exists() for path in paths): - return - - # The wrapper version isn't important, as any version of the wrapper can - # download any version of Gradle. The Gradle version actually used for the - # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. - version = "8.9.0" - - for path in paths: - out_path = TESTBED_DIR / path - out_path.parent.mkdir(exist_ok=True) - download( - f"https://raw.githubusercontent.com/gradle/gradle/v{version}/{path}", - out_path.parent, - ) - os.chmod(out_path, 0o755) - - -# run_testbed will build the app automatically, but it's useful to have this as -# a separate command to allow running the app outside of this script. -def build_testbed(context): - setup_sdk() - setup_testbed() - run( - [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], - cwd=TESTBED_DIR, - ) - - -# Work around a bug involving sys.exit and TaskGroups -# (https://github.com/python/cpython/issues/101515). -def exit(*args): - raise MySystemExit(*args) - - -class MySystemExit(Exception): - pass - - -# The `test` subcommand runs all subprocesses through this context manager so -# that no matter what happens, they can always be cancelled from another task, -# and they will always be cleaned up on exit. -@asynccontextmanager -async def async_process(*args, **kwargs): - process = await asyncio.create_subprocess_exec(*args, **kwargs) - try: - yield process - finally: - if process.returncode is None: - # Allow a reasonably long time for Gradle to clean itself up, - # because we don't want stale emulators left behind. - timeout = 10 - process.terminate() - try: - await wait_for(process.wait(), timeout) - except TimeoutError: - print( - f"Command {args} did not terminate after {timeout} seconds " - f" - sending SIGKILL" - ) - process.kill() - - # Even after killing the process we must still wait for it, - # otherwise we'll get the warning "Exception ignored in __del__". - await wait_for(process.wait(), timeout=1) - - -async def async_check_output(*args, **kwargs): - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) as process: - stdout, stderr = await process.communicate() - if process.returncode == 0: - return stdout.decode(*DECODE_ARGS) - else: - raise CalledProcessError( - process.returncode, args, - stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) - ) - - -# Return a list of the serial numbers of connected devices. Emulators will have -# serials of the form "emulator-5678". -async def list_devices(): - serials = [] - header_found = False - - lines = (await async_check_output(adb, "devices")).splitlines() - for line in lines: - # Ignore blank lines, and all lines before the header. - line = line.strip() - if line == "List of devices attached": - header_found = True - elif header_found and line: - try: - serial, status = line.split() - except ValueError: - raise ValueError(f"failed to parse {line!r}") - if status == "device": - serials.append(serial) - - if not header_found: - raise ValueError(f"failed to parse {lines}") - return serials - - -async def find_device(context, initial_devices): - if context.managed: - print("Waiting for managed device - this may take several minutes") - while True: - new_devices = set(await list_devices()).difference(initial_devices) - if len(new_devices) == 0: - await asyncio.sleep(1) - elif len(new_devices) == 1: - serial = new_devices.pop() - print(f"Serial: {serial}") - return serial - else: - exit(f"Found more than one new device: {new_devices}") - else: - return context.connected - - -# An older version of this script in #121595 filtered the logs by UID instead. -# But logcat can't filter by UID until API level 31. If we ever switch back to -# filtering by UID, we'll also have to filter by time so we only show messages -# produced after the initial call to `stop_app`. -# -# We're more likely to miss the PID because it's shorter-lived, so there's a -# workaround in PythonSuite.kt to stop it being *too* short-lived. -async def find_pid(serial): - print("Waiting for app to start - this may take several minutes") - shown_error = False - while True: - try: - # `pidof` requires API level 24 or higher. The level 23 emulator - # includes it, but it doesn't work (it returns all processes). - pid = (await async_check_output( - adb, "-s", serial, "shell", "pidof", "-s", APP_ID - )).strip() - except CalledProcessError as e: - # If the app isn't running yet, pidof gives no output. So if there - # is output, there must have been some other error. However, this - # sometimes happens transiently, especially when running a managed - # emulator for the first time, so don't make it fatal. - if (e.stdout or e.stderr) and not shown_error: - print_called_process_error(e) - print("This may be transient, so continuing to wait") - shown_error = True - else: - # Some older devices (e.g. Nexus 4) return zero even when no process - # was found, so check whether we actually got any output. - if pid: - print(f"PID: {pid}") - return pid - - # Loop fairly rapidly to avoid missing a short-lived process. - await asyncio.sleep(0.2) - - -async def logcat_task(context, initial_devices): - # Gradle may need to do some large downloads of libraries and emulator - # images. This will happen during find_device in --managed mode, or find_pid - # in --connected mode. - startup_timeout = 600 - serial = await wait_for(find_device(context, initial_devices), startup_timeout) - pid = await wait_for(find_pid(serial), startup_timeout) - - # `--pid` requires API level 24 or higher. - # - # `--binary` mode is used in order to detect which messages end with a - # newline, which most of the other modes don't indicate (except `--format - # long`). For example, every time pytest runs a test, it prints a "." and - # flushes the stream. Each "." becomes a separate log message, but we should - # show them all on the same line. - args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"] - logcat_started = False - async with async_process( - *args, stdout=subprocess.PIPE, stderr=None - ) as process: - while True: - try: - priority, tag, message = await read_logcat(process.stdout) - logcat_started = True - except asyncio.IncompleteReadError: - break - - # Exclude high-volume messages which are rarely useful. - if context.verbose < 2 and "from python test_syslog" in message: - continue - - # Put high-level messages on stderr so they're highlighted in the - # buildbot logs. This will include Python's own stderr. - stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout - - # The app's stdout and stderr should be passed through transparently - # to our own corresponding streams. - if tag in ["python.stdout", "python.stderr"]: - global python_started - python_started = True - stream.write(message) - stream.flush() - else: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. Format them in the same way - # as `logcat --format tag`. - formatted = f"{priority.name[0]}/{tag}: {message}" - if not formatted.endswith("\n"): - formatted += "\n" - log_verbose(context, formatted, stream) - - # If the device disconnects while logcat is running, which always - # happens in --managed mode, some versions of adb return non-zero. - # Distinguish this from a logcat startup error by checking whether we've - # received any logcat messages yet. - status = await wait_for(process.wait(), timeout=1) - if status != 0 and not logcat_started: - raise CalledProcessError(status, args) - - -# Read one binary log message from the given StreamReader. The message format is -# described at https://android.stackexchange.com/a/74660. All supported versions -# of Android use format version 2 or later. -async def read_logcat(stream): - async def read_bytes(size): - return await stream.readexactly(size) - - async def read_int(size): - return int.from_bytes(await read_bytes(size), "little") - - payload_len = await read_int(2) - if payload_len < 2: - # 1 byte for priority, 1 byte for null terminator of tag. - raise ValueError(f"payload length {payload_len} is too short") - - header_len = await read_int(2) - if header_len < 4: - raise ValueError(f"header length {header_len} is too short") - await read_bytes(header_len - 4) # Ignore other header fields. - - priority_int = await read_int(1) - try: - priority = LogPriority(priority_int) - except ValueError: - priority = LogPriority.UNKNOWN - - payload_fields = (await read_bytes(payload_len - 1)).split(b"\0") - if len(payload_fields) < 2: - raise ValueError( - f"payload {payload!r} does not contain at least 2 " - f"null-separated fields" - ) - tag, message, *_ = [ - field.decode(*DECODE_ARGS) for field in payload_fields - ] - return priority, tag, message - - -def stop_app(serial): - run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) - - -async def gradle_task(context): - env = os.environ.copy() - if context.managed: - task_prefix = context.managed - else: - task_prefix = "connected" - env["ANDROID_SERIAL"] = context.connected - - if context.ci_mode: - context.args[0:0] = [ - # See _add_ci_python_opts in libregrtest/main.py. - "-W", "error", "-bb", "-E", - - # Randomization is disabled because order-dependent failures are - # much less likely to pass on a rerun in single-process mode. - "-m", "test", - f"--{context.ci_mode}-ci", "--single-process", "--no-randomize", - "--pythoninfo", - ] - - if not any(arg in context.args for arg in ["-c", "-m"]): - context.args[0:0] = ["-m", "test"] - - args = [ - gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - ] + [ - f"-P{name}={value}" - for name, value in [ - ("python.sitePackages", context.site_packages), - ("python.cwd", context.cwd), - ( - "android.testInstrumentationRunnerArguments.pythonArgs", - json.dumps(context.args), - ), - ] - if value - ] - if context.verbose >= 2: - args.append("--info") - log_verbose(context, f"> {join_command(args)}\n") - - try: - async with async_process( - *args, cwd=TESTBED_DIR, env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Gradle may take several minutes to install SDK packages, so - # it's worth showing those messages even in non-verbose mode. - if line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - log_verbose(context, line) - - status = await wait_for(process.wait(), timeout=1) - if status == 0: - exit(0) - else: - raise CalledProcessError(status, args) - finally: - # Gradle does not stop the tests when interrupted. - if context.connected: - stop_app(context.connected) - - -async def run_testbed(context): - setup_ci() - setup_sdk() - setup_testbed() - - if context.managed: - # In this mode, Gradle will create a device with an unpredictable name. - # So we save a list of the running devices before starting Gradle, and - # find_device then waits for a new device to appear. - initial_devices = await list_devices() - else: - # In case the previous shutdown was unclean, make sure the app isn't - # running, otherwise we might show logs from a previous run. This is - # unnecessary in --managed mode, because Gradle creates a new emulator - # every time. - stop_app(context.connected) - initial_devices = None - - try: - async with asyncio.TaskGroup() as tg: - tg.create_task(logcat_task(context, initial_devices)) - tg.create_task(gradle_task(context)) - except* MySystemExit as e: - raise SystemExit(*e.exceptions[0].args) from None - except* CalledProcessError as e: - # If Python produced no output, then the user probably wants to see the - # verbose output to explain why the test failed. - if not python_started: - for stream, line in hidden_output: - stream.write(line) - - # Extract it from the ExceptionGroup so it can be handled by `main`. - raise e.exceptions[0] - - -def package_version(prefix_dir): - patchlevel_glob = f"{prefix_dir}/include/python*/patchlevel.h" - patchlevel_paths = glob(patchlevel_glob) - if len(patchlevel_paths) != 1: - sys.exit(f"{patchlevel_glob} matched {len(patchlevel_paths)} paths.") - - for line in open(patchlevel_paths[0]): - if match := re.fullmatch(r'\s*#define\s+PY_VERSION\s+"(.+)"\s*', line): - version = match[1] - break - else: - sys.exit(f"Failed to find Python version in {patchlevel_paths[0]}.") - - # If not building against a tagged commit, add a timestamp to the version. - # Follow the PyPA version number rules, as this will make it easier to - # process with other tools. - if version.endswith("+"): - version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") - - return version - - -def package(context): - prefix_dir = subdir(context.host, "prefix") - version = package_version(prefix_dir) - - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - temp_dir = Path(temp_dir) - - # Include all tracked files from the Android directory. - for line in run( - ["git", "ls-files"], - cwd=ANDROID_DIR, capture_output=True, text=True, log=False, - ).stdout.splitlines(): - src = ANDROID_DIR / line - dst = temp_dir / line - dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(src, dst, follow_symlinks=False) - - # Include anything from the prefix directory which could be useful - # either for embedding Python in an app, or building third-party - # packages against it. - for rel_dir, patterns in [ - ("include", ["openssl*", "python*", "sqlite*"]), - ("lib", ["engines-3", "libcrypto*.so", "libpython*", "libsqlite*", - "libssl*.so", "ossl-modules", "python*"]), - ("lib/pkgconfig", ["*crypto*", "*ssl*", "*python*", "*sqlite*"]), - ]: - for pattern in patterns: - for src in glob(f"{prefix_dir}/{rel_dir}/{pattern}"): - dst = temp_dir / relpath(src, prefix_dir.parent) - dst.parent.mkdir(parents=True, exist_ok=True) - if Path(src).is_dir(): - shutil.copytree( - src, dst, symlinks=True, - ignore=lambda *args: ["__pycache__"] - ) - else: - shutil.copy2(src, dst, follow_symlinks=False) - - # Strip debug information. - if not context.debug: - so_files = glob(f"{temp_dir}/**/*.so", recursive=True) - run([android_env(context.host)["STRIP"], *so_files], log=False) - - dist_dir = subdir(context.host, "dist", create=True) - package_path = shutil.make_archive( - f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir - ) - print(f"Wrote {package_path}") - return package_path - - -def ci(context): - for step in [ - configure_build_python, - make_build_python, - configure_host_python, - make_host_python, - package, - ]: - caption = ( - step.__name__.replace("_", " ") - .capitalize() - .replace("python", "Python") - ) - print(f"::group::{caption}") - result = step(context) - if step is package: - package_path = result - print("::endgroup::") - - if ( - "GITHUB_ACTIONS" in os.environ - and (platform.system(), platform.machine()) != ("Linux", "x86_64") - ): - print( - "Skipping tests: GitHub Actions does not support the Android " - "emulator on this platform." - ) - else: - with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: - print("::group::Tests") - - # Prove the package is self-contained by using it to run the tests. - shutil.unpack_archive(package_path, temp_dir) - launcher_args = [ - "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci" - ] - run( - ["./android.py", "test", *launcher_args], - cwd=temp_dir - ) - print("::endgroup::") - - -def env(context): - print_env(android_env(getattr(context, "host", None))) - - -# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated -# by the buildbot worker, we'll make an attempt to clean up our subprocesses. -def install_signal_handler(): - def signal_handler(*args): - os.kill(os.getpid(), signal.SIGINT) - - signal.signal(signal.SIGTERM, signal_handler) - - -def parse_args(): - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand", required=True) - - def add_parser(*args, **kwargs): - parser = subcommands.add_parser(*args, **kwargs) - parser.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show verbose output. Use twice to be even more verbose.") - return parser - - # Subcommands - build = add_parser( - "build", - help="Run configure and make for the selected target" - ) - configure_build = add_parser( - "configure-build", help="Run `configure` for the build Python") - make_build = add_parser( - "make-build", help="Run `make` for the build Python") - configure_host = add_parser( - "configure-host", help="Run `configure` for Android") - make_host = add_parser( - "make-host", help="Run `make` for Android") - - clean = add_parser( - "clean", - help="Delete build directories for the selected target" - ) - - add_parser("build-testbed", help="Build the testbed app") - test = add_parser("test", help="Run the testbed app") - package = add_parser("package", help="Make a release package") - ci = add_parser("ci", help="Run build, package and test") - env = add_parser("env", help="Print environment variables") - - # Common arguments - # --cross-build-dir argument - for cmd in [ - clean, - configure_build, - make_build, - configure_host, - make_host, - build, - package, - test, - ci, - ]: - cmd.add_argument( - "--cross-build-dir", - action="store", - default=os.environ.get("CROSS_BUILD_DIR"), - dest="cross_build_dir", - type=Path, - help=( - "Path to the cross-build directory " - f"(default: {CROSS_BUILD_DIR}). Can also be set " - "with the CROSS_BUILD_DIR environment variable." - ), - ) - - # --cache-dir option - for cmd in [configure_host, build, ci]: - cmd.add_argument( - "--cache-dir", - default=os.environ.get("CACHE_DIR"), - help="The directory to store cached downloads.", - ) - - # --clean option - for subcommand in [build, configure_build, configure_host, ci]: - subcommand.add_argument( - "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build directories first") - - # Allow "all", "build" and "hosts" targets for some commands - for subcommand in [clean, build]: - subcommand.add_argument( - "target", - nargs="?", - default="all", - choices=["all", "build", "hosts"] + HOSTS, - help=( - "The host triplet (e.g., aarch64-linux-android), " - "or 'build' for just the build platform, or 'hosts' for all " - "host platforms, or 'all' for the build platform and all " - "hosts. Defaults to 'all'" - ), - ) - - host_commands = [configure_host, make_host, package, ci] - if in_source_tree: - host_commands.append(env) - for subcommand in host_commands: - subcommand.add_argument( - "host", metavar="HOST", choices=HOSTS, - help="Host triplet: choices=[%(choices)s]") - - for subcommand in [build, configure_build, configure_host, ci]: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") - - # Test arguments - device_group = test.add_mutually_exclusive_group(required=True) - device_group.add_argument( - "--connected", metavar="SERIAL", help="Run on a connected device. " - "Connect it yourself, then get its serial from `adb devices`.") - device_group.add_argument( - "--managed", metavar="NAME", help="Run on a Gradle-managed device. " - "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") - - test.add_argument( - "--site-packages", metavar="DIR", type=abspath, - help="Directory to copy as the app's site-packages.") - test.add_argument( - "--cwd", metavar="DIR", type=abspath, - help="Directory to copy as the app's working directory.") - test.add_argument( - "args", nargs="*", help=f"Python command-line arguments. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. " - f"If neither -c nor -m are included, `-m test` will be prepended, " - f"which will run Python's own test suite.") - - # Package arguments. - for subcommand in [package, ci]: - subcommand.add_argument( - "-g", action="store_true", default=False, dest="debug", - help="Include debug information in package") - - # CI arguments - for subcommand in [test, ci]: - group = subcommand.add_mutually_exclusive_group(required=subcommand is ci) - group.add_argument( - "--fast-ci", action="store_const", dest="ci_mode", const="fast", - help="Add test arguments for GitHub Actions") - group.add_argument( - "--slow-ci", action="store_const", dest="ci_mode", const="slow", - help="Add test arguments for buildbots") - - return parser.parse_args() - - -def main(): - install_signal_handler() - - # Under the buildbot, stdout is not a TTY, but we must still flush after - # every line to make sure our output appears in the correct order relative - # to the output of our subprocesses. - for stream in [sys.stdout, sys.stderr]: - stream.reconfigure(line_buffering=True) - - context = parse_args() - - # Set the CROSS_BUILD_DIR if an argument was provided - if context.cross_build_dir: - global CROSS_BUILD_DIR - CROSS_BUILD_DIR = context.cross_build_dir.resolve() - - dispatch = { - "configure-build": configure_build_python, - "make-build": make_build_python, - "configure-host": configure_host_python, - "make-host": make_host_python, - "build": build_targets, - "clean": clean_targets, - "build-testbed": build_testbed, - "test": run_testbed, - "package": package, - "ci": ci, - "env": env, - } - - try: - result = dispatch[context.subcommand](context) - if asyncio.iscoroutine(result): - asyncio.run(result) - except CalledProcessError as e: - print_called_process_error(e) - sys.exit(1) - - -def print_called_process_error(e): - for stream_name in ["stdout", "stderr"]: - content = getattr(e, stream_name) - if isinstance(content, bytes): - content = content.decode(*DECODE_ARGS) - stream = getattr(sys, stream_name) - if content: - stream.write(content) - if not content.endswith("\n"): - stream.write("\n") - - # shlex uses single quotes, so we surround the command with double quotes. - print( - f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' - ) - - -if __name__ == "__main__": - main() diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts deleted file mode 100644 index b58edc04a929d9f..000000000000000 --- a/Android/testbed/app/build.gradle.kts +++ /dev/null @@ -1,429 +0,0 @@ -import com.android.build.api.variant.* -import kotlin.math.max - -plugins { - id("com.android.application") - id("org.jetbrains.kotlin.android") -} - -val ANDROID_DIR = file("../..") -val PYTHON_DIR = ANDROID_DIR.parentFile!! -val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") -val inSourceTree = ( - ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() -) - -val KNOWN_ABIS = mapOf( - "aarch64-linux-android" to "arm64-v8a", - "arm-linux-androideabi" to "armeabi-v7a", - "i686-linux-android" to "x86", - "x86_64-linux-android" to "x86_64", -) - -val osArch = System.getProperty("os.arch") -val NATIVE_ABI = mapOf( - "aarch64" to "arm64-v8a", - "amd64" to "x86_64", - "arm64" to "arm64-v8a", - "x86_64" to "x86_64", -)[osArch] ?: throw GradleException("Unknown os.arch '$osArch'") - -// Discover prefixes. -val prefixes = ArrayList() -if (inSourceTree) { - for ((triplet, _) in KNOWN_ABIS.entries) { - val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } - } -} else { - // Testbed is inside a release package. - val prefix = file("$ANDROID_DIR/prefix") - if (prefix.exists()) { - prefixes.add(prefix) - } -} -if (prefixes.isEmpty()) { - throw GradleException( - "No Android prefixes found: see README.md for testing instructions" - ) -} - -// Detect Python versions and ABIs. -lateinit var pythonVersion: String -var abis = HashMap() -for ((i, prefix) in prefixes.withIndex()) { - val libDir = file("$prefix/lib") - val version = run { - for (filename in libDir.list()!!) { - """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] - } - } - throw GradleException("Failed to find Python version in $libDir") - } - if (i == 0) { - pythonVersion = version - } else if (pythonVersion != version) { - throw GradleException( - "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" - ) - } - - val libPythonDir = file("$libDir/python$pythonVersion") - val triplet = run { - for (filename in libPythonDir.list()!!) { - """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() - .matchEntire(filename)?.let { - return@run it.groupValues[1] - } - } - throw GradleException("Failed to find Python triplet in $libPythonDir") - } - abis[prefix] = KNOWN_ABIS[triplet]!! -} - - -android { - val androidEnvFile = file("../../android-env.sh").absoluteFile - - namespace = "org.python.testbed" - compileSdk = 35 - - defaultConfig { - applicationId = "org.python.testbed" - - minSdk = androidEnvFile.useLines { - for (line in it) { - """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { - return@useLines it.groupValues[1].toInt() - } - } - throw GradleException("Failed to find API level in $androidEnvFile") - } - - // This controls the API level of the maxVersion managed emulator, which is used - // by CI and cibuildwheel. - // * 33 has excessive buffering in the logcat client - // (https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826). - // * 34 consumes too much disk space on GitHub Actions (#142289). - // * 35 has issues connecting to the internet (#142387). - // * 36 and later are not available as aosp_atd images yet. - targetSdk = 32 - - versionCode = 1 - versionName = "1.0" - - ndk.abiFilters.addAll(abis.values) - externalNativeBuild.cmake.arguments( - "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { - // AGP uses the ${} syntax for its own purposes, so use a Jinja style - // placeholder. - "$PYTHON_CROSS_DIR/{{triplet}}/prefix" - } else { - prefixes[0] - }, - "-DPYTHON_VERSION=$pythonVersion", - "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", - ) - - testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - } - - ndkVersion = androidEnvFile.useLines { - for (line in it) { - """ndk_version=(\S+)""".toRegex().find(line)?.let { - return@useLines it.groupValues[1] - } - } - throw GradleException("Failed to find NDK version in $androidEnvFile") - } - externalNativeBuild.cmake { - path("src/main/c/CMakeLists.txt") - } - - // Set this property to something nonexistent but non-empty. Otherwise it'll use the - // default list, which ignores asset directories beginning with an underscore, and - // maybe also other files required by tests. - aaptOptions.ignoreAssetsPattern = "android-testbed-dont-ignore-anything" - - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - kotlinOptions { - jvmTarget = "1.8" - } - - testOptions { - managedDevices { - localDevices { - // systemImageSource should use what its documentation calls an - // "explicit source", i.e. the sdkmanager package name format, because - // that will be required in CreateEmulatorTask below. - create("minVersion") { - device = "Small Phone" - - // Managed devices have a minimum API level of 27. - apiLevel = max(27, defaultConfig.minSdk!!) - - // ATD devices are smaller and faster, but have a minimum - // API level of 30. - systemImageSource = if (apiLevel >= 30) "aosp_atd" else "default" - } - - create("maxVersion") { - device = "Small Phone" - apiLevel = defaultConfig.targetSdk!! - systemImageSource = "aosp_atd" - } - } - - // If the previous test run succeeded and nothing has changed, - // Gradle thinks there's no need to run it again. Override that. - afterEvaluate { - (localDevices.names + listOf("connected")).forEach { - tasks.named("${it}DebugAndroidTest") { - outputs.upToDateWhen { false } - } - } - } - } - } -} - -dependencies { - implementation("androidx.appcompat:appcompat:1.6.1") - implementation("com.google.android.material:material:1.11.0") - implementation("androidx.constraintlayout:constraintlayout:2.1.4") - androidTestImplementation("androidx.test.ext:junit:1.1.5") - androidTestImplementation("androidx.test:rules:1.5.0") -} - - -afterEvaluate { - // Every new emulator has a maximum of 2 GB RAM, regardless of its hardware profile - // (https://cs.android.com/android-studio/platform/tools/base/+/refs/tags/studio-2025.3.2:sdklib/src/main/java/com/android/sdklib/internal/avd/EmulatedProperties.java;l=68). - // This is barely enough to test Python, and not enough to test Pandas - // (https://github.com/python/cpython/pull/137186#issuecomment-3136301023, - // https://github.com/pandas-dev/pandas/pull/63405#issuecomment-3667846159). - // So we'll increase it by editing the emulator configuration files. - // - // If the emulator doesn't exist yet, we want to edit it after it's created, but - // before it starts for the first time. Otherwise it'll need to be cold-booted - // again, which would slow down the first run, which is likely the only run in CI - // environments. But the Setup task both creates and starts the emulator if it - // doesn't already exist. So we create it ourselves before the Setup task runs. - for (device in android.testOptions.managedDevices.localDevices) { - val createTask = tasks.register("${device.name}Create") { - this.device = device.device - apiLevel = device.apiLevel - systemImageSource = device.systemImageSource - abi = NATIVE_ABI - } - tasks.named("${device.name}Setup") { - dependsOn(createTask) - } - } -} - -abstract class CreateEmulatorTask : DefaultTask() { - @get:Input abstract val device: Property - @get:Input abstract val apiLevel: Property - @get:Input abstract val systemImageSource: Property - @get:Input abstract val abi: Property - @get:Inject abstract val execOps: ExecOperations - - private val avdName by lazy { - listOf( - "dev${apiLevel.get()}", - systemImageSource.get(), - abi.get(), - device.get().replace(' ', '_'), - ).joinToString("_") - } - - private val avdDir by lazy { - // XDG_CONFIG_HOME is respected by both avdmanager and Gradle. - val userHome = System.getenv("ANDROID_USER_HOME") ?: ( - (System.getenv("XDG_CONFIG_HOME") ?: System.getProperty("user.home")!!) - + "/.android" - ) - File("$userHome/avd/gradle-managed", "$avdName.avd") - } - - @TaskAction - fun run() { - if (!avdDir.exists()) { - createAvd() - } - updateAvd() - } - - fun createAvd() { - val systemImage = listOf( - "system-images", - "android-${apiLevel.get()}", - systemImageSource.get(), - abi.get(), - ).joinToString(";") - - runCmdlineTool("sdkmanager", systemImage) - runCmdlineTool( - "avdmanager", "create", "avd", - "--name", avdName, - "--path", avdDir, - "--device", device.get().lowercase().replace(" ", "_"), - "--package", systemImage, - ) - - val iniName = "$avdName.ini" - if (!File(avdDir.parentFile.parentFile, iniName).renameTo( - File(avdDir.parentFile, iniName) - )) { - throw GradleException("Failed to rename $iniName") - } - } - - fun updateAvd() { - for (filename in listOf( - "config.ini", // Created by avdmanager; always exists - "hardware-qemu.ini", // Created on first run; might not exist - )) { - val iniFile = File(avdDir, filename) - if (!iniFile.exists()) { - if (filename == "config.ini") { - throw GradleException("$iniFile does not exist") - } - continue - } - - val iniText = iniFile.readText() - val pattern = Regex( - """^\s*hw.ramSize\s*=\s*(.+?)\s*$""", RegexOption.MULTILINE - ) - val matches = pattern.findAll(iniText).toList() - if (matches.size != 1) { - throw GradleException( - "Found ${matches.size} instances of $pattern in $iniFile; expected 1" - ) - } - - val expectedRam = "4096" - if (matches[0].groupValues[1] != expectedRam) { - iniFile.writeText( - iniText.replace(pattern, "hw.ramSize = $expectedRam") - ) - } - } - } - - fun runCmdlineTool(tool: String, vararg args: Any) { - val androidHome = System.getenv("ANDROID_HOME")!! - val exeSuffix = - if (System.getProperty("os.name").lowercase().startsWith("win")) ".exe" - else "" - val command = - listOf("$androidHome/cmdline-tools/latest/bin/$tool$exeSuffix", *args) - println(command.joinToString(" ")) - execOps.exec { - commandLine(command) - } - } -} - - -// Create some custom tasks to copy Python and its standard library from -// elsewhere in the repository. -androidComponents.onVariants { variant -> - val pyPlusVer = "python$pythonVersion" - generateTask(variant, variant.sources.assets!!) { - into("python") { - // Include files such as pyconfig.h are used by some of the tests. - into("include/$pyPlusVer") { - for (prefix in prefixes) { - from("$prefix/include/$pyPlusVer") - } - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - } - - into("lib/$pyPlusVer") { - // To aid debugging, the source directory takes priority when - // running inside a CPython source tree. - if (inSourceTree) { - from("$PYTHON_DIR/Lib") - } - for (prefix in prefixes) { - from("$prefix/lib/$pyPlusVer") - } - - into("site-packages") { - from("$projectDir/src/main/python") - - val sitePackages = findProperty("python.sitePackages") as String? - if (!sitePackages.isNullOrEmpty()) { - if (!file(sitePackages).exists()) { - throw GradleException("$sitePackages does not exist") - } - from(sitePackages) - } - } - - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - exclude("**/__pycache__") - } - - into("cwd") { - val cwd = findProperty("python.cwd") as String? - if (!cwd.isNullOrEmpty()) { - if (!file(cwd).exists()) { - throw GradleException("$cwd does not exist") - } - from(cwd) - } - } - - // A filename ending with .gz will be automatically decompressed - // while building the APK. Avoid this by adding a dash to the end, - // and add an extra dash to any filenames that already end with one. - // This will be undone in MainActivity.kt. - rename(""".*(\.gz|-)""", "$0-") - } - } - - generateTask(variant, variant.sources.jniLibs!!) { - for ((prefix, abi) in abis.entries) { - into(abi) { - from("$prefix/lib") - include("libpython*.*.so") - include("lib*_python.so") - } - } - } -} - - -fun generateTask( - variant: ApplicationVariant, directories: SourceDirectories, - configure: GenerateTask.() -> Unit -) { - val taskName = "generate" + - listOf(variant.name, "Python", directories.name) - .map { it.replaceFirstChar(Char::uppercase) } - .joinToString("") - - directories.addGeneratedSourceDirectory( - tasks.register(taskName) { - into(outputDir) - configure() - }, - GenerateTask::outputDir) -} - - -// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. -abstract class GenerateTask: Sync() { - @get:OutputDirectory - abstract val outputDir: DirectoryProperty -} diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts deleted file mode 100644 index 451517b3f1aeab6..000000000000000 --- a/Android/testbed/build.gradle.kts +++ /dev/null @@ -1,5 +0,0 @@ -// Top-level build file where you can add configuration options common to all sub-projects/modules. -plugins { - id("com.android.application") version "8.10.0" apply false - id("org.jetbrains.kotlin.android") version "1.9.22" apply false -} diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties deleted file mode 100644 index e9f345c8c262501..000000000000000 --- a/Android/testbed/gradle.properties +++ /dev/null @@ -1,28 +0,0 @@ -# Project-wide Gradle settings. -# IDE (e.g. Android Studio) users: -# Gradle settings configured through the IDE *will override* -# any settings specified in this file. -# For more details on how to configure your build environment visit -# http://www.gradle.org/docs/current/userguide/build_environment.html -# Specifies the JVM arguments used for the daemon process. -# The setting is particularly useful for tweaking memory settings. -org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 -# When configured, Gradle will run in incubating parallel mode. -# This option should only be used with decoupled projects. More details, visit -# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects -# org.gradle.parallel=true -# AndroidX package structure to make it clearer which packages are bundled with the -# Android operating system, and which are packaged with your app's APK -# https://developer.android.com/topic/libraries/support-library/androidx-rn -android.useAndroidX=true -# Kotlin code style for this project: "official" or "obsolete": -kotlin.code.style=official -# Enables namespacing of each library's R class so that its R class includes only the -# resources declared in the library itself and none from the library's dependencies, -# thereby reducing the size of the R class for that library -android.nonTransitiveRClass=true - -# By default, the app will be uninstalled after the tests finish (apparently -# after 10 seconds in case of an unclean shutdown). We disable this, because -# when using android.py it can conflict with the installation of the next run. -android.injected.androidTest.leaveApksInstalledAfterRun=true diff --git a/Doc/Makefile b/Doc/Makefile index 7bdabd8bf168fe7..ef6ef8c42e3636b 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -13,7 +13,7 @@ JOBS = auto PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -REQUIREMENTS = requirements.txt +REQUIREMENTS = pylock.toml SPHINXERRORHANDLING = --fail-on-warning # Internal variables. @@ -32,6 +32,7 @@ help: @echo "Please use \`make ' where is one of" @echo " clean to remove build files" @echo " venv to create a venv with necessary tools" + @echo " lock to regenerate the pinned dependencies in $(REQUIREMENTS)" @echo " html to make standalone HTML files" @echo " gettext to generate POT files" @echo " htmlview to open the index page built by the html target in your browser" @@ -88,9 +89,9 @@ htmlhelp: build "build/htmlhelp/pydoc.hhp project file." .PHONY: latex -latex: _ensure-sphinxcontrib-svg2pdfconverter latex: BUILDER = latex -latex: build +latex: _ensure-sphinxcontrib-svg2pdfconverter + $(MAKE) build BUILDER=$(BUILDER) @echo "Build finished; the LaTeX files are in build/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." @@ -185,6 +186,19 @@ venv: echo "The venv has been created in the $(VENVDIR) directory"; \ fi +.PHONY: lock +lock: +# Dependencies have a 14 day cooldown period to mitigate supply chain attacks, +# except for sphinx_linklint and python-docs-theme, which are maintained by +# core team members. + uv pip compile requirements.txt \ + --exclude-newer P14D \ + --exclude-newer-package sphinx_linklint=PT0S \ + --exclude-newer-package python-docs-theme=PT0S \ + --no-cache --output-file $(REQUIREMENTS) \ + --python-version 3.12 --universal \ + --custom-compile-command="make lock" + .PHONY: dist-no-html dist-no-html: dist-text dist-epub dist-texinfo diff --git a/Doc/about.rst b/Doc/about.rst index 5c1b497ca6bcea8..84ae3492804c328 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -10,7 +10,7 @@ and now maintained as an independent project. .. _reStructuredText: https://docutils.sourceforge.io/rst.html .. _Sphinx: https://www.sphinx-doc.org/ -.. In the online version of these documents, you can submit comments and suggest +.. In the online version of this documentation, you can submit comments and suggest changes directly on the documentation pages. Development of the documentation and its toolchain is an entirely volunteer diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 254a22f2622bd8e..a6ea0a72e76f9db 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -12,6 +12,9 @@ It can be sometimes faster to fix bugs yourself and contribute patches to Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. + +.. _reporting-documentation-bugs: + Documentation bugs ================== diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 59044d2d88cc168..09c9ed3ca54cf69 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -153,10 +153,12 @@ Allocating Objects on the Heap To allocate and create extension modules. -Deprecated aliases -^^^^^^^^^^^^^^^^^^ +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ -These are :term:`soft deprecated` aliases to existing functions and macros. +.. soft-deprecated:: 3.15 + +These are aliases to existing functions and macros. They exist solely for backwards compatibility. @@ -164,7 +166,7 @@ They exist solely for backwards compatibility. :widths: auto :header-rows: 1 - * * Deprecated alias + * * Soft-deprecated alias * Function * * .. c:macro:: PyObject_NEW(type, typeobj) * :c:macro:`PyObject_New` diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 58456a36b96c151..a878b20c928ea3d 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -399,13 +399,18 @@ inside nested parentheses. They are: their default value --- when an optional argument is not specified, :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding C variable(s). + For example, the format string ``"OO|OO"`` corresponds to the Python + signature ``f(a, b, c=None, d=None)``. ``$`` :c:func:`PyArg_ParseTupleAndKeywords` only: Indicates that the remaining arguments in the Python argument list are - keyword-only. Currently, all keyword-only arguments must also be optional - arguments, so ``|`` must always be specified before ``$`` in the format - string. + keyword-only. + They are optional if ``|`` was specified before ``$``, and required otherwise. + ``|`` cannot be specified after ``$``. + For example, the format string ``"O|O$O"`` corresponds to the Python + signature ``f(a, b=None, *, c=None)``, + and the format string ``"OO$OO"`` corresponds to ``f(a, b, *, c, d)``. .. versionadded:: 3.3 diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index fe950196297a24b..dc3e0f37c36c5b2 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -258,7 +258,9 @@ readonly, format .. c:macro:: PyBUF_WRITEABLE - This is a :term:`soft deprecated` alias to :c:macro:`PyBUF_WRITABLE`. + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 .. c:macro:: PyBUF_FORMAT diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d1fde1baf71a457..ff68ecafcda4d08 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -47,9 +47,9 @@ called with a non-bytes parameter. *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of the bytes object are uninitialized. - .. deprecated:: 3.15 - ``PyBytes_FromStringAndSize(NULL, len)`` is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead of + ``PyBytes_FromStringAndSize(NULL, len)``. .. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) @@ -184,10 +184,11 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_Concat(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*; the caller will own the new reference. The reference to - the old value of *bytes* will be stolen. If the new object cannot be - created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. + appended to *bytes*; the caller will own the new reference. + The reference to the old value of *bytes* will be ":term:`stolen `". + If the new object cannot be created, the old reference to *bytes* will still + be "stolen", the value of *\*bytes* will be set to ``NULL``, and + the appropriate exception will be set. .. note:: If *newpart* implements the buffer protocol, then the buffer @@ -238,9 +239,8 @@ called with a non-bytes parameter. *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. - .. deprecated:: 3.15 - The function is :term:`soft deprecated`, - use the :c:type:`PyBytesWriter` API instead. + .. soft-deprecated:: 3.15 + Use the :c:type:`PyBytesWriter` API instead. .. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) @@ -385,14 +385,18 @@ Getters Get the writer size. + The function does not invalidate pointers returned by + :c:func:`PyBytesWriter_GetData`. + The function cannot fail. .. c:function:: void* PyBytesWriter_GetData(PyBytesWriter *writer) Get the writer data: start of the internal buffer. - The pointer is valid until :c:func:`PyBytesWriter_Finish` or - :c:func:`PyBytesWriter_Discard` is called on *writer*. + The pointer remains valid until a :c:type:`PyBytesWriter` function other + than :c:func:`PyBytesWriter_GetData` or :c:func:`PyBytesWriter_GetSize` is + called on *writer*. The function cannot fail. diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 03a848d68ed7aba..e3ffe494c0b6ef9 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -108,25 +108,20 @@ Refer to :ref:`using-capsules` for more information on using these objects. Import a pointer to a C object from a capsule attribute in a module. The *name* parameter should specify the full name to the attribute, as in - ``module.attribute``. The *name* stored in the capsule must match this - string exactly. - - This function splits *name* on the ``.`` character, and imports the first - element. It then processes further elements using attribute lookups. + ``package.module.attribute``. + Modules are imported if needed, + other components are looked up as attributes. + The *name* stored in the capsule must match this string exactly. Return the capsule's internal *pointer* on success. On failure, set an exception and return ``NULL``. - .. note:: - - If *name* points to an attribute of some submodule or subpackage, this - submodule or subpackage must be previously imported using other means - (for example, by using :c:func:`PyImport_ImportModule`) for the - attribute lookups to succeed. - .. versionchanged:: 3.3 *no_block* has no effect anymore. + .. versionchanged:: next + Submodules are now imported if needed. + .. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index be2c85ec97489e1..57b77f92a7d2e6a 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -212,7 +212,7 @@ bound into a function. .. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) - This is a :term:`soft deprecated` function that does nothing. + This is a function that does nothing. Prior to Python 3.10, this function would perform basic optimizations to a code object. @@ -220,6 +220,8 @@ bound into a function. .. versionchanged:: 3.10 This function now does nothing. + .. soft-deprecated:: 3.13 + .. _c_codeobject_flags: diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 629312bd771beb2..10f96c7cb75e882 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -130,7 +130,7 @@ rather than dereferencing them through pointers. Please note, that these functions are :term:`soft deprecated` since Python 3.15. Avoid using this API in a new code to do complex arithmetic: either use -the `Number Protocol `_ API or use native complex types, like +the :ref:`Number Protocol ` API or use native complex types, like :c:expr:`double complex`. diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 1746fe95eaaca93..3f38411a52de6b0 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -112,6 +112,7 @@ Other Objects picklebuffer.rst weakref.rst capsule.rst + sentinel.rst frame.rst gen.rst coro.rst diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index caa855a10d20ca9..06422fb63f34f1a 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -33,3 +33,9 @@ return. with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. A reference to *frame* is stolen by this function. The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b913e24b3c7cc81..539c4610ce4f5bc 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -140,7 +140,7 @@ found in the dictionary of type objects. .. c:macro:: PyDescr_COMMON - This is a :term:`soft deprecated` macro including the common fields for a + This is a macro including the common fields for a descriptor object. This was included in Python's C API by mistake; do not use it in extensions. @@ -148,6 +148,8 @@ found in the dictionary of type objects. descriptor protocol (:c:member:`~PyTypeObject.tp_descr_get` and :c:member:`~PyTypeObject.tp_descr_set`). + .. soft-deprecated:: 3.15 + Built-in descriptors ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 7a925870a62a578..87d09ad2412e405 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -76,6 +76,11 @@ Dictionary objects The first argument can be a :class:`dict` or a :class:`frozendict`. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionchanged:: 3.15 Also accept :class:`frozendict`. @@ -102,8 +107,13 @@ Dictionary objects Insert *val* into the dictionary *p* with a key of *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. This function *does not* steal a - reference to *val*. + ``0`` on success or ``-1`` on failure. + This function *does not* ":term:`steal`" a reference to *val*. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) @@ -120,6 +130,11 @@ Dictionary objects If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) @@ -136,10 +151,15 @@ Dictionary objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. The first argument can be a :class:`dict` or a :class:`frozendict`. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 .. versionchanged:: 3.15 @@ -162,6 +182,13 @@ Dictionary objects :meth:`~object.__eq__` methods are silently ignored. Prefer the :c:func:`PyDict_GetItemWithError` function instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. versionchanged:: 3.10 Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. @@ -177,6 +204,13 @@ Dictionary objects occurred. Return ``NULL`` **without** an exception set if the key wasn't present. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. versionchanged:: 3.15 Also accept :class:`frozendict`. @@ -195,6 +229,13 @@ Dictionary objects Prefer using the :c:func:`PyDict_GetItemWithError` function with your own :c:func:`PyUnicode_FromString` *key* instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemStringRef`, + which returns a :term:`strong reference`. + .. versionchanged:: 3.15 Also accept :class:`frozendict`. @@ -221,6 +262,14 @@ Dictionary objects .. versionadded:: 3.4 + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_SetDefaultRef`, + which returns a :term:`strong reference`. + + .. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) @@ -240,6 +289,11 @@ Dictionary objects These may refer to the same object: in that case you hold two separate references to it. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 @@ -257,6 +311,11 @@ Dictionary objects Similar to :meth:`dict.pop`, but without the default value and not raising :exc:`KeyError` if the key is missing. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 @@ -403,6 +462,13 @@ Dictionary objects only be added if there is not a matching key in *a*. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) @@ -412,6 +478,13 @@ Dictionary objects argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) @@ -427,6 +500,13 @@ Dictionary objects if override or key not in a: a[key] = value + .. note:: + + In the :term:`free-threaded ` build, only *a* is locked. + The iteration over *seq2* is not synchronized; *seq2* may be concurrently + modified by another thread. + + .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) Register *callback* as a dictionary watcher. Return a non-negative integer @@ -434,6 +514,13 @@ Dictionary objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_ClearWatcher(int watcher_id) @@ -442,6 +529,13 @@ Dictionary objects :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. if the given *watcher_id* was never registered.) + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 8ecd7c625171045..40522f8c7b13756 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -119,6 +119,21 @@ Printing and clearing .. versionadded:: 3.12 +.. c:function:: void PyErr_Display(PyObject *unused, PyObject *value, PyObject *tb) + + Legacy variant of :c:func:`PyErr_DisplayException`. + + Print the exception *value* with its traceback to :data:`sys.stderr`. + If *value* has no traceback set, *tb* is used as its traceback. + The first argument is ignored. + + If :data:`sys.stderr` is ``None``, nothing is printed. + If :data:`sys.stderr` is not set, the exception is dumped to the + C ``stderr`` stream instead. + + .. deprecated:: 3.12 + Use :c:func:`PyErr_DisplayException` instead. + Raising exceptions ================== @@ -412,7 +427,7 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. @@ -499,11 +514,12 @@ Querying the error indicator .. c:function:: void PyErr_SetRaisedException(PyObject *exc) Set *exc* as the exception currently being raised, - clearing the existing exception if one is set. + clearing the existing exception if one is set. If *exc* is ``NULL``, + just clear the existing exception. - .. warning:: + *exc* must be a valid exception or ``NULL``. - This call steals a reference to *exc*, which must be a valid exception. + This call ":term:`steals `" a reference to *exc*. .. versionadded:: 3.12 @@ -641,7 +657,8 @@ Querying the error indicator Set the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. + freshly raised. This function ":term:`steals `" the references + of the arguments. To clear the exception state, pass ``NULL`` for all three arguments. This function is kept for backwards compatibility. Prefer using :c:func:`PyErr_SetHandledException`. @@ -658,8 +675,8 @@ Querying the error indicator .. versionchanged:: 3.11 The ``type`` and ``traceback`` arguments are no longer used and can be NULL. The interpreter now derives them from the exception - instance (the ``value`` argument). The function still steals - references of all three arguments. + instance (the ``value`` argument). The function still + ":term:`steals `" references of all three arguments. Signal Handling @@ -818,7 +835,7 @@ Exception Classes .. c:macro:: PyException_HEAD - This is a :term:`soft deprecated` macro including the base fields for an + This is a macro including the base fields for an exception object. This was included in Python's C API by mistake and is not designed for use @@ -826,6 +843,8 @@ Exception Classes :c:func:`PyErr_NewException` or otherwise create a class inheriting from :c:data:`PyExc_BaseException`. + .. soft-deprecated:: 3.15 + Exception Objects ================= @@ -867,7 +886,7 @@ Exception Objects Set the context associated with the exception to *ctx*. Use ``NULL`` to clear it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. + This ":term:`steals `" a reference to *ctx*. .. c:function:: PyObject* PyException_GetCause(PyObject *ex) @@ -882,7 +901,8 @@ Exception Objects Set the cause associated with the exception to *cause*. Use ``NULL`` to clear it. There is no type check to make sure that *cause* is either an exception - instance or ``None``. This steals a reference to *cause*. + instance or ``None``. + This ":term:`steals `" a reference to *cause*. The :attr:`~BaseException.__suppress_context__` attribute is implicitly set to ``True`` by this function. @@ -1036,7 +1056,7 @@ Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requ special recursion handling. In addition to protecting the stack, :c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. +these are the C equivalent to :deco:`reprlib.recursive_repr`. .. c:function:: int Py_ReprEnter(PyObject *object) @@ -1346,3 +1366,67 @@ Tracebacks This function returns ``0`` on success, and returns ``-1`` with an exception set on failure. + +.. c:function:: const char* PyUnstable_DumpTraceback(int fd, PyThreadState *tstate) + + Write a trace of the Python stack in *tstate* into the file *fd*. The format + looks like:: + + Traceback (most recent call first): + File "xxx", line xxx in + File "xxx", line xxx in + ... + File "xxx", line xxx in + + This function is meant to debug situations such as segfaults, fatal errors, + and similar. The file and function names it outputs are encoded to ASCII with + backslashreplace and truncated to 500 characters. It writes only the first + 100 frames; further frames are truncated with the line ``...``. + + This function will return ``NULL`` on success, or an error message on error. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *tstate* need to be attached. + + .. versionadded:: 3.15 + +.. c:function:: const char* PyUnstable_DumpTracebackThreads(int fd, PyInterpreterState *interp, PyThreadState *current_tstate, Py_ssize_t max_threads) + + Write the traces of all Python threads in *interp* into the file *fd*. + + If *interp* is ``NULL`` then this function will try to identify the current + interpreter using thread-specific storage. If it cannot, it will return an + error. + + If *current_tstate* is not ``NULL`` then it will be used to identify what the + current thread is in the written output. If it is ``NULL`` then this function + will identify the current thread using thread-specific storage. It is not an + error if the function is unable to get the current Python thread state. + + This function will return ``NULL`` on success, or an error message on error. + + This function is meant to debug situations such as segfaults, fatal + errors, and similar. It calls :c:func:`PyUnstable_DumpTraceback` for each + thread. It only writes the tracebacks of the first *max_threads* threads, + further output is truncated with the line ``...``. If *max_threads* is 0, the + function will use a default value of 100 for the argument. + + This function is intended for use in crash scenarios such as signal handlers + for SIGSEGV, where the interpreter may be in an inconsistent state. Given + that it reads interpreter data structures that may be partially modified, the + function might produce incomplete output or it may even crash itself. + + The caller does not need to hold an :term:`attached thread state`, nor does + *current_tstate* need to be attached. + + .. warning:: + On the :term:`free-threaded build`, this function is not thread-safe. If + another thread deletes its :term:`thread state` while this function is being + called, the process will likely crash. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst index 92b531665e135dd..879b1ec50369ff4 100644 --- a/Doc/c-api/extension-modules.rst +++ b/Doc/c-api/extension-modules.rst @@ -38,7 +38,7 @@ Extension export hook The export hook must be an exported function with the following signature: -.. c:function:: PyModuleDef_Slot *PyModExport_modulename(void) +.. c:function:: PySlot *PyModExport_modulename(void) For modules with ASCII-only names, the :ref:`export hook ` must be named :samp:`PyModExport_{}`, @@ -57,7 +57,7 @@ Python's *punycode* encoding with hyphens replaced by underscores. In Python: suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') return b'PyModExport' + suffix -The export hook returns an array of :c:type:`PyModuleDef_Slot` entries, +The export hook returns an array of :c:type:`PySlot` entries, terminated by an entry with a slot ID of ``0``. These slots describe how the module should be created and initialized. @@ -75,7 +75,7 @@ It is recommended to define the export hook function using a helper macro: Declare an extension module export hook. This macro: - * specifies the :c:expr:`PyModuleDef_Slot*` return type, + * specifies the :c:expr:`PySlot*` return type, * adds any special linkage declarations required by the platform, and * for C++, declares the function as ``extern "C"``. @@ -83,12 +83,12 @@ For example, a module called ``spam`` would be defined like this:: PyABIInfo_VAR(abi_info); - static PyModuleDef_Slot spam_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "spam"}, - {Py_mod_init, spam_init_function}, + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_FUNC(Py_mod_init, spam_init_function), ... - {0, NULL}, + PySlot_END }; PyMODEXPORT_FUNC @@ -100,11 +100,35 @@ For example, a module called ``spam`` would be defined like this:: The export hook is typically the only non-\ ``static`` item defined in the module's C source. -The hook should be kept short -- ideally, one line as above. -If you do need to use Python C API in this function, it is recommended to call -``PyABIInfo_Check(&abi_info, "modulename")`` first to raise an exception, -rather than crash, in common cases of ABI mismatch. +.. _pymodexport-api-caveats: +The hook should be kept short. +If it does more than ``return`` a static array, several caveats apply: + +- If you need to use any Python C API, it is recommended to call + :c:func:`PyABIInfo_Check` first to raise an exception, + rather than crash, in common cases of ABI mismatch. +- Code in the export hook must never rely on the :term:`GIL`: + :term:`free-threaded builds ` of Python can only check + the :c:macro:`Py_mod_gil` slot (or the lack of it) after the hook returns, +- Similarly, the hook may be called in any subinterpreter, since the + :c:macro:`Py_mod_multiple_interpreters` slot (or lack of it) + is only checked after the hook returns. + +For example:: + + PyMODEXPORT_FUNC + PyModExport_modulename(void) + { + if (PyABIInfo_Check(&abi_info, "modulename") < 0) { + /* ABI mismatch. It's not safe to examine the raised exception. */ + return NULL; + } + + /* use Python API (as little as possible); don't rely on GIL */ + + return modulename_slots; + } .. note:: @@ -191,10 +215,10 @@ the :c:data:`Py_mod_multiple_interpreters` slot. ``PyInit`` function ................... -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - This functionality is :term:`soft deprecated`. - It will not get new features, but there are no plans to remove it. + This functionality will not get new features, + but there are no plans to remove it. Instead of :c:func:`PyModExport_modulename`, an extension module can define an older-style :dfn:`initialization function` with the signature: @@ -272,10 +296,9 @@ For example, a module called ``spam`` would be defined like this:: Legacy single-phase initialization ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. deprecated:: 3.15 +.. soft-deprecated:: 3.15 - Single-phase initialization is :term:`soft deprecated`. - It is a legacy mechanism to initialize extension + Single-phase initialization is a legacy mechanism to initialize extension modules, with known drawbacks and design flaws. Extension module authors are encouraged to use multi-phase initialization instead. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index d89072ab24e241d..dcafefdc0458722 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -136,11 +136,12 @@ the :mod:`io` APIs instead. failure; the appropriate exception will be set. -Deprecated API -^^^^^^^^^^^^^^ +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ +.. soft-deprecated:: 3.15 -These are :term:`soft deprecated` APIs that were included in Python's C API +These are APIs that were included in Python's C API by mistake. They are documented solely for completeness; use other ``PyFile*`` APIs instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 929b56bd8e82033..c484b3e5bdb5132 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -86,8 +86,7 @@ Floating-Point Objects It is equivalent to the :c:macro:`!INFINITY` macro from the C11 standard ```` header. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_NAN @@ -103,8 +102,7 @@ Floating-Point Objects Equivalent to :c:macro:`!INFINITY`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 .. c:macro:: Py_MATH_E @@ -161,8 +159,8 @@ Floating-Point Objects that is, it is normal, subnormal or zero, but not infinite or NaN. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isfinite` instead. .. c:macro:: Py_IS_INFINITY(X) @@ -170,8 +168,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is positive or negative infinity. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isinf` instead. .. c:macro:: Py_IS_NAN(X) @@ -179,8 +177,8 @@ Floating-Point Objects Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) value. Return ``0`` otherwise. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + .. soft-deprecated:: 3.14 + Use :c:macro:`!isnan` instead. Pack and Unpack functions @@ -235,9 +233,6 @@ most likely :exc:`OverflowError`). Pack a C double as the IEEE 754 binary64 double precision format. - .. impl-detail:: - This function always succeeds in CPython. - Unpack functions ^^^^^^^^^^^^^^^^ @@ -253,9 +248,6 @@ Return value: The unpacked double. On error, this is ``-1.0`` and :c:func:`PyErr_Occurred` is true (and an exception is set, most likely :exc:`OverflowError`). -.. impl-detail:: - These functions always succeed in CPython. - .. c:function:: double PyFloat_Unpack2(const char *p, int le) Unpack the IEEE 754 binary16 half-precision format as a C double. diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 967cfc727655ecb..4159ff6e5965fbd 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -147,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -169,7 +169,7 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Legacy Local Variable APIs +Legacy local variable APIs ^^^^^^^^^^^^^^^^^^^^^^^^^^ These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. @@ -178,40 +178,34 @@ They exist solely for backwards compatibility. .. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` attribute of *f* to the internal "fast" array of local variables, allowing changes in frame objects to be visible to the interpreter. If *clear* was true, this function would process variables that were unset in the locals dictionary. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function would copy the internal "fast" array of local variables (which is used by the interpreter) to the :attr:`~frame.f_locals` attribute of *f*, allowing changes in local variables to be visible to frame objects. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. .. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) - This function is :term:`soft deprecated` and does nothing. - Prior to Python 3.13, this function was similar to :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and ``-1`` with an exception set on failure. - .. versionchanged:: 3.13 + .. soft-deprecated:: 3.13 This function now does nothing. @@ -219,7 +213,7 @@ They exist solely for backwards compatibility. :pep:`667` -Internal Frames +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -249,5 +243,3 @@ Unless using :pep:`523`, you will not need this. Return the currently executing line number, or -1 if there is no line number. .. versionadded:: 3.12 - - diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 9c9c97f7b853d85..9c71bb3d59d5e99 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -414,7 +414,7 @@ In some cases, failure is not distinguishable from a successful ``NULL`` result. See :ref:`duringgc-functions` for common information. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: @@ -433,7 +433,7 @@ In some cases, failure is not distinguishable from a successful ``NULL`` result. The reference is valid for the duration of the :c:member:`!tp_traverse` handler call. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: :c:func:`PyType_GetBaseByToken` @@ -445,7 +445,7 @@ In some cases, failure is not distinguishable from a successful ``NULL`` result. These functions return a :term:`borrowed reference`, which is valid for the duration of the :c:member:`!tp_traverse` handler call. - .. versionadded:: next + .. versionadded:: 3.15 .. seealso:: diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 74db49a6814800b..88c3cc31186826a 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -35,15 +35,27 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) Create and return a new generator object based on the *frame* object. - A reference to *frame* is stolen by this function. The argument must not be - ``NULL``. + A reference to *frame* is ":term:`stolen `" by this function (even + on error). The argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be ``NULL``. + A reference to *frame* is ":term:`stolen `" by this function (even + on error). The *frame* argument must not be ``NULL``. + + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. .. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) @@ -68,8 +80,9 @@ Asynchronous Generator Objects .. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create a new asynchronous generator wrapping *frame*, with ``__name__`` and - ``__qualname__`` set to *name* and *qualname*. *frame* is stolen by this - function and must not be ``NULL``. + ``__qualname__`` set to *name* and *qualname*. + *frame* is ":term:`stolen `" by this function (even on error) and + must not be ``NULL``. On success, this function returns a :term:`strong reference` to the new asynchronous generator. On failure, this function returns ``NULL`` @@ -77,6 +90,12 @@ Asynchronous Generator Objects .. versionadded:: 3.6 + .. deprecated-removed:: 3.16 3.18 + + This function has not been used since 3.10. + It is also impossible to construct a proper *frame* + object to call this function. + .. c:function:: int PyAsyncGen_CheckExact(PyObject *op) Return true if *op* is an asynchronous generator object, false otherwise. @@ -90,7 +109,9 @@ Deprecated API .. c:macro:: PyAsyncGenASend_CheckExact(op) - This is a :term:`soft deprecated` API that was included in Python's C API + This is an API that was included in Python's C API by mistake. It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index e2d363b911a87c6..7a3e5357e0c2cac 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -146,7 +146,7 @@ Importing Modules alternatives. .. versionchanged:: 3.15 - ``__cached__`` is no longer set. + The ``__cached__`` attribute is no longer set. .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) @@ -170,7 +170,7 @@ Importing Modules :class:`~importlib.machinery.ModuleSpec` for alternatives. .. versionchanged:: 3.15 - ``__cached__`` no longer set. + The ``__cached__`` attribute no longer set. .. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) @@ -304,6 +304,11 @@ Importing Modules Initialization function for a module built into the interpreter. + Note that the inittab uses "``PyInit``" + :ref:`initialization functions `; + there is currently no way to include "``PyModExport_``" + :ref:`export hooks `. + .. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) @@ -393,11 +398,6 @@ Importing Modules Make all imports lazy by default. - .. c:enumerator:: PyImport_LAZY_NONE - - Disable lazy imports entirely. Even explicit ``lazy`` statements become - eager imports. - .. versionadded:: 3.15 .. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index eabe00f4004001f..051f6fd765e8506 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -18,6 +18,7 @@ document the API functions in detail. refcounting.rst exceptions.rst extension-modules.rst + slots.rst utilities.rst abstract.rst concrete.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 209e48767ccfd6c..ef09639189a6c27 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -623,6 +623,10 @@ Some options are read from the :mod:`sys` attributes. For example, the option .. versionadded:: 3.14 + .. versionchanged:: next + The function now replaces :data:`sys.flags` (create a new object), + instead of modifying :data:`sys.flags` in-place. + .. _pyconfig_api: @@ -1231,9 +1235,9 @@ PyConfig .. c:member:: wchar_t* base_executable - Python base executable: :data:`sys._base_executable`. + Python base executable: ``sys._base_executable``. - Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. + Set by the ``__PYVENV_LAUNCHER__`` environment variable. Set from :c:member:`PyConfig.executable` if ``NULL``. @@ -1744,7 +1748,7 @@ PyConfig * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use - :envvar:`__PYVENV_LAUNCHER__` environment variable if set. + ``__PYVENV_LAUNCHER__`` environment variable if set. * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and non-empty. * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other @@ -1980,8 +1984,7 @@ PyConfig The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse order: the last :c:member:`PyConfig.warnoptions` item becomes the first - item of :data:`warnings.filters` which is checked first (highest - priority). + item of ``warnings.filters`` which is checked first (highest priority). The :option:`-W` command line options adds its value to :c:member:`~PyConfig.warnoptions`, it can be used multiple times. diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst index 189d8e424f68147..46f5b1dd33963c0 100644 --- a/Doc/c-api/interp-lifecycle.rst +++ b/Doc/c-api/interp-lifecycle.rst @@ -104,7 +104,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_DebugFlag @@ -119,7 +119,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_DontWriteBytecodeFlag @@ -134,7 +134,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_FrozenFlag @@ -145,7 +145,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_HashRandomizationFlag @@ -161,7 +161,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_IgnoreEnvironmentFlag @@ -175,7 +175,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-E` and :option:`-I` options. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_InspectFlag @@ -191,7 +191,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_InteractiveFlag @@ -202,7 +202,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_IsolatedFlag @@ -218,7 +218,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_LegacyWindowsFSEncodingFlag @@ -238,7 +238,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_LegacyWindowsStdioFlag @@ -257,7 +257,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_NoSiteFlag @@ -273,7 +273,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_NoUserSiteDirectory @@ -288,7 +288,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_OptimizeFlag @@ -300,7 +300,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_QuietFlag @@ -315,7 +315,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_UnbufferedStdioFlag @@ -329,7 +329,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 .. c:var:: int Py_VerboseFlag @@ -346,7 +346,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment variable. - .. deprecated-removed:: 3.12 3.15 + .. deprecated-removed:: 3.12 3.16 Initializing and finalizing the interpreter @@ -410,6 +410,11 @@ Initializing and finalizing the interpreter (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until :c:func:`Py_Initialize` is called again. + .. versionchanged:: 3.15 + This function no longer returns true until initialization has fully + completed, including import of the :mod:`site` module. Previously it + could return true while :c:func:`Py_Initialize` was still running. + .. c:function:: int Py_IsFinalizing() @@ -573,31 +578,203 @@ Initializing and finalizing the interpreter .. _cautions-regarding-runtime-finalization: -Cautions regarding runtime finalization ---------------------------------------- +Cautions regarding interpreter finalization +------------------------------------------- In the late stage of :term:`interpreter shutdown`, after attempting to wait for non-daemon threads to exit (though this can be interrupted by :class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. +is marked as finalizing, meaning that :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the finalization +thread (the thread that initiated finalization; this is typically the main thread) +is allowed to :term:`attach ` a thread state. + +Other threads that attempt to attach during finalization, either explicitly +(such as via :c:func:`PyThreadState_Ensure` or :c:macro:`Py_END_ALLOW_THREADS`) +or implicitly (such as in-between bytecode instructions), will enter a +**permanently blocked state**. Generally, this is harmless, but this can +result in deadlocks. For example, a thread may be permanently blocked while +holding a lock, meaning that the finalization thread can never acquire that +lock. + +Prior to CPython 3.13, the thread would exit instead of hanging, +which led to other issues (see the warning note at +:c:func:`PyThread_exit_thread`). + +Gross? Yes. Starting in Python 3.15, there are a number of C APIs that make +it possible to avoid these issues by temporarily preventing finalization: + +.. _interpreter-guards: + +.. seealso:: + + :pep:`788` explains the design, motivation and rationale + for these APIs. + +.. c:type:: PyInterpreterGuard + + An opaque interpreter guard structure. + + By holding an interpreter guard, the caller can ensure that the interpreter + will not finalize until the guard is closed (through + :c:func:`PyInterpreterGuard_Close`). + + When a guard is held, a thread attempting to finalize the interpreter will + block until the guard is closed before starting finalization. + After finalization has started, threads are forever unable to acquire + guards for that interpreter. This means that if you forget to close an + interpreter guard, the process will **permanently hang** during + finalization! + + Holding a guard for an interpreter is similar to holding a + :term:`strong reference` to a Python object, except finalization does not happen + automatically after all guards are released: it requires an explicit + :c:func:`Py_EndInterpreter` call. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromCurrent(void) + + Create a finalization guard for the current interpreter. This will prevent + finalization until the guard is closed. + + For example: + + .. code-block:: c + + // Temporarily prevent finalization. + PyInterpreterGuard *guard = PyInterpreterGuard_FromCurrent(); + if (guard == NULL) { + // Finalization has already started or we're out of memory. + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + // Do some critical processing here. For example, we can safely acquire + // locks that might be acquired by the finalization thread. + Py_END_ALLOW_THREADS; + + // Now that we're done with our critical processing, the interpreter is + // allowed to finalize again. + PyInterpreterGuard_Close(guard); + + On success, this function returns a guard for the current interpreter; + on failure, it returns ``NULL`` with an exception set. + + This function will fail only if the current interpreter has already started + finalizing, or if the process is out of memory. + + The guard pointer returned by this function must be eventually closed + with :c:func:`PyInterpreterGuard_Close`; failing to do so will result in + the Python process infinitely hanging. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterGuard *PyInterpreterGuard_FromView(PyInterpreterView *view) + + Create a finalization guard for an interpreter through a view. + + On success, this function returns a guard to the interpreter + represented by *view*. The view is still valid after calling this + function. The guard must eventually be closed with + :c:func:`PyInterpreterGuard_Close`. + + If the interpreter no longer exists, is already finalizing, or out of memory, + then this function returns ``NULL`` without setting an exception. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterGuard_Close(PyInterpreterGuard *guard) + + Close an interpreter guard, allowing the interpreter to start + finalization if no other guards remain. If an interpreter guard + is never closed, the interpreter will infinitely wait when trying + to enter finalization! + + After an interpreter guard is closed, it may not be used in + :c:func:`PyThreadState_Ensure`. Doing so will result in undefined + behavior. + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. _interpreter-views: + +Interpreter views +----------------- + +In some cases, it may be necessary to access an interpreter that may have been +deleted. This can be done using interpreter views. + +.. c:type:: PyInterpreterView + + An opaque view of an interpreter. + + This is a thread-safe way to access an interpreter that may have be + finalizing or already destroyed. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromCurrent(void) + + Create a view to the current interpreter. + + This function is generally meant to be used alongside + :c:func:`PyInterpreterGuard_FromView` or :c:func:`PyThreadState_EnsureFromView`. + + On success, this function returns a view to the current interpreter; on + failure, it returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: void PyInterpreterView_Close(PyInterpreterView *view) + + Close an interpreter view. + + If an interpreter view is never closed, the view's memory will never be + freed, but there are no other consequences. (In contrast, forgetting to + close a guard will infinitely hang the main thread during finalization.) + + This function cannot fail, and the caller doesn't need to hold an + :term:`attached thread state`. + + .. versionadded:: 3.15 + + +.. c:function:: PyInterpreterView *PyInterpreterView_FromMain(void) + + Create a view for the main interpreter (the first and default + interpreter in a Python process; see + :c:func:`PyInterpreterState_Main`). + + On success, this function returns a view to the main + interpreter; on failure, it returns ``NULL`` without an exception set. + Failure indicates that the process is out of memory. + + Use this function when an interpreter pointer or view cannot be supplied + by the caller, such as when a native threading library does not provide a + ``void *arg`` parameter that could carry a :c:type:`PyInterpreterGuard` or + :c:type:`PyInterpreterView`. In code that supports subinterpreters, prefer + :c:func:`PyInterpreterView_FromCurrent` so the guard tracks the calling + interpreter rather than the main one. + + The caller does not need to hold an :term:`attached thread state`. + + .. versionadded:: 3.15 Process-wide parameters @@ -627,7 +804,7 @@ Process-wide parameters Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a :c:expr:`wchar_t*` string. - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 .. c:function:: const char* Py_GetVersion() @@ -752,7 +929,7 @@ Process-wide parameters .. versionadded:: 3.1.3 - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 .. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) @@ -773,7 +950,7 @@ Process-wide parameters .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 .. c:function:: void Py_SetPythonHome(const wchar_t *home) @@ -794,4 +971,4 @@ Process-wide parameters Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a :c:expr:`wchar_t*` string. - .. deprecated-removed:: 3.11 3.15 + .. deprecated-removed:: 3.11 3.16 diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 2a22a023bdafb4b..701d2a31b8c9059 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -536,16 +536,14 @@ have been standardized in C11 (or previous standards). Use the standard ``alignas`` specifier rather than this macro. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_FORMAT_SIZE_T The :c:func:`printf` formatting modifier for :c:type:`size_t`. Use ``"z"`` directly instead. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_LL(number) Py_ULL(number) @@ -558,8 +556,7 @@ have been standardized in C11 (or previous standards). Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LONG_LONG PY_INT32_T @@ -572,8 +569,7 @@ have been standardized in C11 (or previous standards). respectively. Historically, these types needed compiler-specific extensions. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: PY_LLONG_MIN PY_LLONG_MAX @@ -587,16 +583,14 @@ have been standardized in C11 (or previous standards). The required header, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - These macros are :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_MEMCPY(dest, src, n) - This is a :term:`soft deprecated` alias to :c:func:`!memcpy`. - Use :c:func:`!memcpy` directly instead. + This is an alias to :c:func:`!memcpy`. - .. deprecated:: 3.14 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. .. c:macro:: Py_UNICODE_SIZE @@ -606,29 +600,25 @@ have been standardized in C11 (or previous standards). The required header for the latter, ````, :ref:`is included ` in ``Python.h``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_UNICODE_WIDE Defined if ``wchar_t`` can hold a Unicode character (UCS-4). Use ``sizeof(wchar_t) >= 4`` instead - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. c:macro:: Py_VA_COPY - This is a :term:`soft deprecated` alias to the C99-standard ``va_copy`` - function. + This is an alias to the C99-standard ``va_copy`` function. Historically, this would use a compiler-specific method to copy a ``va_list``. .. versionchanged:: 3.6 This is now an alias to ``va_copy``. - .. deprecated:: 3.15 - The macro is :term:`soft deprecated`. + .. soft-deprecated:: 3.15 .. _api-objects: @@ -747,9 +737,12 @@ the caller is said to *borrow* the reference. Nothing needs to be done for a Conversely, when a calling function passes in a reference to an object, there are two possibilities: the function *steals* a reference to the object, or it -does not. *Stealing a reference* means that when you pass a reference to a -function, that function assumes that it now owns that reference, and you are not -responsible for it any longer. +does not. + +*Stealing a reference* means that when you pass a reference to a +function, that function assumes that it now owns that reference. +Since the new owner can use :c:func:`!Py_DECREF` at its discretion, +you (the caller) must not use that reference after the call. .. index:: single: PyList_SetItem (C function) @@ -1170,7 +1163,7 @@ when defined by the compiler, will also implicitly enable :c:macro:`!Py_DEBUG`. In addition to the reference count debugging described below, extra checks are performed. See :ref:`Python Debug Build ` for more details. -Defining :c:macro:`Py_TRACE_REFS` enables reference tracing +Defining ``Py_TRACE_REFS`` enables reference tracing (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). When defined, a circular doubly linked list of active objects is maintained by adding two extra fields to every :c:type:`PyObject`. Total allocations are tracked as well. Upon diff --git a/Doc/c-api/lifecycle.dot.pdf b/Doc/c-api/lifecycle.dot.pdf deleted file mode 100644 index ed5b5039c83e2c4..000000000000000 Binary files a/Doc/c-api/lifecycle.dot.pdf and /dev/null differ diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst index 531c4080a0131c3..5e3dca29f2ad901 100644 --- a/Doc/c-api/lifecycle.rst +++ b/Doc/c-api/lifecycle.rst @@ -19,7 +19,7 @@ object's life. An arrow from *A* to *B* indicates that event *B* can occur after event *A* has occurred, with the arrow's label indicating the condition that must be true for *B* to occur after *A*. -.. only:: html and not epub +.. only:: builder_html .. raw:: html @@ -50,20 +50,13 @@ that must be true for *B* to occur after *A*. })(); -.. only:: epub or not (html or latex) +.. only:: not builder_html .. image:: lifecycle.dot.svg :align: center :class: invert-in-dark-mode :alt: Diagram showing events in an object's life. Explained in detail below. -.. only:: latex - - .. image:: lifecycle.dot.pdf - :align: center - :class: invert-in-dark-mode - :alt: Diagram showing events in an object's life. Explained in detail below. - .. container:: :name: life-events-graph-description diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 8f560699d355e49..21769c4c5c869c8 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -102,8 +102,10 @@ List Objects .. note:: - This function "steals" a reference to *item* and discards a reference to - an item already in the list at the affected position. + This function ":term:`steals `" a reference to *item*, + even on error. + On success, it discards a reference to an item already in the list + at the affected position (unless it was ``NULL``). .. c:function:: void PyList_SET_ITEM(PyObject *list, Py_ssize_t i, PyObject *o) @@ -117,7 +119,7 @@ List Objects .. note:: - This macro "steals" a reference to *item*, and, unlike + This macro ":term:`steals `" a reference to *item*, and, unlike :c:func:`PyList_SetItem`, does *not* discard a reference to any item that is being replaced; any reference in *list* at position *i* will be leaked. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 790ec8da109ba8e..874e422d4701dd8 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -71,6 +71,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) + + Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, + or ``NULL`` on failure. + + .. c:function:: PyObject* PyLong_FromInt32(int32_t value) PyObject* PyLong_FromInt64(int64_t value) @@ -81,12 +87,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - .. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) PyObject* PyLong_FromUInt64(uint64_t value) @@ -197,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 9f84e4bc6dfd91d..73310670ac371c9 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -77,7 +77,7 @@ memory footprint as a whole. Consequently, under certain circumstances, the Python memory manager may or may not trigger appropriate actions, like garbage collection, memory compaction or other preventive procedures. Note that by using the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. +for the I/O buffer completely escapes the Python memory manager. .. seealso:: @@ -157,7 +157,7 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -235,7 +235,7 @@ In the GIL-enabled build (default build) the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -368,7 +368,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 8b967c285ac865c..31df4451f15e663 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -133,14 +133,16 @@ Module Objects unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. +.. _c_module_slots: .. _pymoduledef_slot: Module definition ----------------- Modules created using the C API are typically defined using an -array of :dfn:`slots`. -The slots provide a "description" of how a module should be created. +array of :c:type:`PySlot` structs, which provides a "description" of how a +module should be created. +See :ref:`capi-slots` for more information on slots in general. .. versionchanged:: 3.15 @@ -158,30 +160,12 @@ Unless specified otherwise, the same slot ID may not be repeated in an array of slots. -.. c:type:: PyModuleDef_Slot - - .. c:member:: int slot - - A slot ID, chosen from the available ``Py_mod_*`` values explained below. - - An ID of 0 marks the end of a :c:type:`!PyModuleDef_Slot` array. - - .. c:member:: void* value - - Value of the slot, whose meaning depends on the slot ID. - - The value may not be NULL. - To leave a slot out, omit the :c:type:`PyModuleDef_Slot` entry entirely. - - .. versionadded:: 3.5 - - Metadata slots .............. .. c:macro:: Py_mod_name - :c:type:`Slot ID ` for the name of the new module, + :c:member:`Slot ID ` for the name of the new module, as a NUL-terminated UTF8-encoded ``const char *``. Note that modules are typically created using a @@ -196,7 +180,7 @@ Metadata slots .. c:macro:: Py_mod_doc - :c:type:`Slot ID ` for the docstring of the new + :c:type:`Slot ID ` for the docstring of the new module, as a NUL-terminated UTF8-encoded ``const char *``. Usually it is set to a variable created with :c:macro:`PyDoc_STRVAR`. @@ -211,7 +195,7 @@ Feature slots .. c:macro:: Py_mod_abi - :c:type:`Slot ID ` whose value points to + :c:member:`Slot ID ` whose value points to a :c:struct:`PyABIInfo` structure describing the ABI that the extension is using. @@ -222,8 +206,8 @@ Feature slots PyABIInfo_VAR(abi_info); - static PyModuleDef_Slot mymodule_slots[] = { - {Py_mod_abi, &abi_info}, + static PySlot mymodule_slots[] = { + PySlot_DATA(Py_mod_abi, &abi_info), ... }; @@ -237,7 +221,7 @@ Feature slots .. c:macro:: Py_mod_multiple_interpreters - :c:type:`Slot ID ` whose value is one of: + :c:member:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -263,11 +247,20 @@ Feature slots If ``Py_mod_multiple_interpreters`` is not specified, the import machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_multiple_interpreters`: + + .. code-block:: c + + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) + .. versionadded:: 3.12 .. c:macro:: Py_mod_gil - :c:type:`Slot ID ` whose value is one of: + :c:member:`Slot ID ` whose value is one of: .. c:namespace:: NULL @@ -288,6 +281,14 @@ Feature slots If ``Py_mod_gil`` is not specified, the import machinery defaults to ``Py_MOD_GIL_USED``. + For historical reasons, the values are declared as pointers (``void *``). + When using :c:type:`PySlot` arrays, use :c:macro:`PySlot_DATA` for + :c:macro:`!Py_mod_gil`: + + .. code-block:: c + + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED) + .. versionadded:: 3.13 @@ -296,7 +297,7 @@ Creation and initialization slots .. c:macro:: Py_mod_create - :c:type:`Slot ID ` for a function that creates + :c:member:`Slot ID ` for a function that creates the module object itself. The function must have the signature: @@ -346,7 +347,7 @@ Creation and initialization slots .. c:macro:: Py_mod_exec - :c:type:`Slot ID ` for a function that will + :c:member:`Slot ID ` for a function that will :dfn:`execute`, or initialize, the module. This function does the equivalent to executing the code of a Python module: typically, it adds classes and constants to the module. @@ -375,7 +376,7 @@ Creation and initialization slots .. c:macro:: Py_mod_methods - :c:type:`Slot ID ` for a table of module-level + :c:member:`Slot ID ` for a table of module-level functions, as an array of :c:type:`PyMethodDef` values suitable as the *functions* argument to :c:func:`PyModule_AddFunctions`. @@ -446,12 +447,12 @@ To retrieve the state from a given module, use the following functions: Slots for defining module state ............................... -The following :c:member:`PyModuleDef_Slot.slot` IDs are available for +The following :c:member:`slot IDs ` are available for defining the module state. .. c:macro:: Py_mod_state_size - :c:type:`Slot ID ` for the size of the module state, + :c:member:`Slot ID ` for the size of the module state, in bytes. Setting the value to a non-negative value means that the module can be @@ -468,7 +469,7 @@ defining the module state. .. c:macro:: Py_mod_state_traverse - :c:type:`Slot ID ` for a traversal function to call + :c:member:`Slot ID ` for a traversal function to call during GC traversal of the module object. The signature of the function, and meanings of the arguments, @@ -487,11 +488,11 @@ defining the module state. .. versionadded:: 3.15 - Use :c:member:`PyModuleDef.m_size` instead to support previous versions. + Use :c:member:`PyModuleDef.m_traverse` instead to support previous versions. .. c:macro:: Py_mod_state_clear - :c:type:`Slot ID ` for a clear function to call + :c:member:`Slot ID ` for a clear function to call during GC clearing of the module object. The signature of the function is: @@ -519,7 +520,7 @@ defining the module state. .. c:macro:: Py_mod_state_free - :c:type:`Slot ID ` for a function to call during + :c:member:`Slot ID ` for a function to call during deallocation of the module object. The signature of the function is: @@ -578,7 +579,7 @@ A module's token -- and the *your_token* value to use in the above code -- is: .. c:macro:: Py_mod_token - :c:type:`Slot ID ` for the module token. + :c:member:`Slot ID ` for the module token. If you use this slot to set the module token (rather than rely on the default), you must ensure that: @@ -617,14 +618,14 @@ Creating extension modules dynamically The following functions may be used to create an extension module dynamically, rather than from an extension's :ref:`export hook `. -.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, PyObject *spec) +.. c:function:: PyObject *PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec) Create a new module object, given an array of :ref:`slots ` and the :py:class:`~importlib.machinery.ModuleSpec` *spec*. - The *slots* argument must point to an array of :c:type:`PyModuleDef_Slot` + The *slots* argument must point to an array of :c:type:`PySlot` structures, terminated by an entry with slot ID of 0 - (typically written as ``{0}`` or ``{0, NULL}`` in C). + (typically written as :c:macro:`PySlot_END`). The array must include a :c:data:`Py_mod_abi` entry. The *spec* argument may be any ``ModuleSpec``-like object, as described @@ -640,10 +641,6 @@ rather than from an extension's :ref:`export hook `. must be called to fully initialize a module. (See also :ref:`multi-phase-initialization`.) - The *slots* array only needs to be valid for the duration of the - :c:func:`!PyModule_FromSlotsAndSpec` call. - In particular, it may be heap-allocated. - .. versionadded:: 3.15 .. c:function:: int PyModule_Exec(PyObject *module) @@ -685,6 +682,12 @@ remove it. Usually, there is only one variable of this type for each extension module defined this way. + The struct, including all members, is part of the + :ref:`Stable ABI ` for non-free-threaded builds (``abi3``). + In the Stable ABI for free-threaded builds (``abi3t``), + this struct is opaque, and unusable in practice; see :ref:`pymoduledef_slot` + for a replacement. + .. c:member:: PyModuleDef_Base m_base Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`: @@ -695,6 +698,11 @@ remove it. The type of :c:member:`!PyModuleDef.m_base`. + The struct is part of the :ref:`Stable ABI ` for + non-free-threaded builds (``abi3``). + In the Stable ABI for Free-Threaded Builds + (``abi3t``), this struct is opaque, and unusable in practice. + .. c:macro:: PyModuleDef_HEAD_INIT The required initial value for :c:member:`!PyModuleDef.m_base`. @@ -727,6 +735,8 @@ remove it. .. c:member:: PyModuleDef_Slot* m_slots An array of additional slots, terminated by a ``{0, NULL}`` entry. + Note that the entries use the older :c:type:`PyModuleDef_Slot` structure, + rather than :c:type:`PySlot`. If the array contains slots corresponding to :c:type:`PyModuleDef` members, the values must match. @@ -741,6 +751,39 @@ remove it. .. c:member:: inquiry m_reload + .. c:namespace:: NULL + + .. c:type:: PyModuleDef_Slot + + Older structure defining additional slots of a module. + + Note that a :c:type:`!PyModuleDef_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_mod_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. + + Each :c:type:`!PyModuleDef_Slot` structure ``modslot`` is interpreted + as the following :c:type:`PySlot` structure:: + + (PySlot){ + .sl_id=modslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=modslot.value + } + + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_mod_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_mod_slots` slot (if any). + + .. c:member:: int slot + + Corresponds to :c:member:`PySlot.sl_id`. + + .. c:member:: void* value + + Corresponds to :c:member:`PySlot.sl_ptr`. + + .. versionadded:: 3.5 + .. c:member:: traverseproc m_traverse inquiry m_clear freefunc m_free @@ -763,6 +806,14 @@ remove it. The type of ``PyModuleDef`` objects. +.. c:macro:: Py_mod_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyModuleDef_Slot` structures. + + .. versionadded:: 3.15 + .. _moduledef-dynamic: The following API can be used to create modules from a :c:type:`!PyModuleDef` @@ -802,6 +853,10 @@ struct: .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) Create a new module object, given the definition in *def* and the @@ -822,12 +877,22 @@ struct: .. versionadded:: 3.5 + .. soft-deprecated:: next + + Prefer :c:func:`PyModule_FromSlotsAndSpec` in new code. + .. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. .. versionadded:: 3.5 + .. soft-deprecated:: next + + To run a module's own execution slots, prefer :c:func:`PyModule_Exec`, + which works on modules that were not created from a + :c:type:`PyModuleDef` structure. + .. c:macro:: PYTHON_API_VERSION PYTHON_API_STRING @@ -909,8 +974,8 @@ or code that creates modules dynamically. .. c:function:: int PyModule_Add(PyObject *module, const char *name, PyObject *value) - Similar to :c:func:`PyModule_AddObjectRef`, but "steals" a reference - to *value*. + Similar to :c:func:`PyModule_AddObjectRef`, but ":term:`steals `" + a reference to *value* (even on error). It can be called with a result of function that returns a new reference without bothering to check its result or even saving it to a variable. @@ -925,8 +990,8 @@ or code that creates modules dynamically. .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to - *value* on success (if it returns ``0``). + Similar to :c:func:`PyModule_AddObjectRef`, but :term:`steals ` + a reference to *value* on success (if it returns ``0``). The new :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions are recommended, since it is @@ -954,9 +1019,7 @@ or code that creates modules dynamically. // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index b0227c2f4faf15c..4bfcb86abf58ed1 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -205,6 +205,4 @@ would typically correspond to a Python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 76a1e9f528dc70c..a962c4ee09ad77d 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -31,7 +31,7 @@ Note that holding an :term:`attached thread state` is not required for these API or ``-2`` on failure to create a lock. Check ``errno`` for more information about the cause of a failure. -.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name) +.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, size_t code_size, const char *entry_name) Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is thread safe. Here is what an example entry looks like:: @@ -49,3 +49,43 @@ Note that holding an :term:`attached thread state` is not required for these API This is called by the runtime itself during interpreter shut-down. In general, there shouldn't be a reason to explicitly call this, except to handle specific scenarios such as forking. + +.. c:function:: int PyUnstable_CopyPerfMapFile(const char *parent_filename) + + Open the ``/tmp/perf-$pid.map`` file and append the content of *parent_filename* + to it. + + This function is available on all platforms but only generates output on platforms + that support perf maps (currently only Linux). On other platforms, it does nothing. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *code) + + Compile the given code object using the current perf trampoline. + + The "current" trampoline is the one set by the runtime or the most recent + :c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` call. + + If no trampoline is set, falls back to normal compilation (no perf map entry). + + :param code: The code object to compile. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable) + + Set whether the perf trampoline should persist after a fork. + + * If ``enable`` is true (non-zero): perf map file remains open/valid post-fork. + Child process inherits all existing perf map entries. + * If ``enable`` is false (zero): perf map closes post-fork. + Child process gets empty perf map. + + Default: false (clears on fork). + + :param enable: 1 to enable, 0 to disable. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/sentinel.rst b/Doc/c-api/sentinel.rst new file mode 100644 index 000000000000000..b1b7329a5d42c59 --- /dev/null +++ b/Doc/c-api/sentinel.rst @@ -0,0 +1,47 @@ +.. highlight:: c + +.. _sentinelobjects: + +Sentinel objects +---------------- + +.. c:var:: PyTypeObject PySentinel_Type + + This instance of :c:type:`PyTypeObject` represents the Python + :class:`sentinel` type. This is the same object as :class:`sentinel`. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_Check(PyObject *o) + + Return true if *o* is a :class:`sentinel` object or a subtype. + The :class:`sentinel` type does not currently allow subclasses, + so this check is exact. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: int PySentinel_CheckExact(PyObject *o) + + Return true if *o* is a :class:`sentinel` object, but not a subtype. + The :class:`sentinel` type does not currently allow subclasses. + Future Python versions may choose to allow subtyping. + This function always succeeds. + + .. versionadded:: 3.15 + +.. c:function:: PyObject* PySentinel_New(const char *name, const char *module_name, const char *repr) + + Return a new :class:`sentinel` object with :attr:`~sentinel.__name__` set to + *name* and :attr:`~sentinel.__module__` set to *module_name*. + *name* must not be ``NULL``. If *module_name* is ``NULL``, :attr:`~sentinel.__module__` + is set to ``None``. If *repr* is ``NULL``, ``repr()`` returns :attr:`~sentinel.__name__`. + Return ``NULL`` with an exception set on failure. + + For pickling to work, *module_name* must be the name of an importable + module, and the sentinel must be accessible from that module under a + path matching *name*. Pickle treats *name* as a global variable name + in *module_name* (see :meth:`object.__reduce__`). + + .. versionadded:: 3.15 diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a93a00..90490cf6749b59c 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -67,7 +67,7 @@ Sequence Protocol Assign object *v* to the *i*\ th element of *o*. Raise an exception and return ``-1`` on failure; return ``0`` on success. This is the equivalent of the Python statement ``o[i] = v``. This function *does - not* steal a reference to *v*. + not* ":term:`steal`" a reference to *v*. If *v* is ``NULL``, the element is deleted, but this feature is deprecated in favour of using :c:func:`PySequence_DelItem`. @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 6974f74fbd597ac..db537aff2e6ce5a 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -89,6 +89,11 @@ the constructor functions work with any iterable Python object. actually iterable. The constructor is also useful for copying a set (``c=set(s)``). + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) @@ -97,6 +102,11 @@ the constructor functions work with any iterable Python object. set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset`, :class:`dict` or :class:`frozendict`. + The following functions and macros are available for instances of :class:`set` or :class:`frozenset` or instances of their subtypes. @@ -124,6 +134,10 @@ or :class:`frozenset` or instances of their subtypes. the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PySet_Add(PyObject *set, PyObject *key) @@ -135,6 +149,12 @@ or :class:`frozenset` or instances of their subtypes. :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + The following functions are available for instances of :class:`set` or its subtypes but not for instances of :class:`frozenset` or its subtypes. @@ -149,6 +169,11 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: PyObject* PySet_Pop(PyObject *set) @@ -164,13 +189,19 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + In the :term:`free-threaded build`, the set is emptied before its entries + are cleared, so other threads will observe an empty set rather than + intermediate states. + Deprecated API ^^^^^^^^^^^^^^ .. c:macro:: PySet_MINSIZE - A :term:`soft deprecated` constant representing the size of an internal + A constant representing the size of an internal preallocated table inside :c:type:`PySetObject` instances. This is documented solely for completeness, as there are no guarantees @@ -180,3 +211,5 @@ Deprecated API :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/slots.rst b/Doc/c-api/slots.rst new file mode 100644 index 000000000000000..84a125cb60bae7e --- /dev/null +++ b/Doc/c-api/slots.rst @@ -0,0 +1,240 @@ +.. highlight:: c + +.. _capi-slots: + +Definition slots +================ + +To define :ref:`module objects ` and +:ref:`classes ` using the C API, you may use +an array of *slots* -- essentially, key-value pairs that describe features +of the object to create. +This decouples the data from the structures used at runtime, allowing CPython +-- and other Python C API implementations -- to update the structures without +breaking backwards compatibility. + +This section documents slots in general. +For object-specific behavior and slot values, see documentation for functions +that apply slots: + +- :c:func:`PyType_FromSlots` for types; +- :c:func:`PyModule_FromSlotsAndSpec` and :ref:`extension-export-hook` + for modules. + +When slots are passed to a function that applies them, the function will +not modify the slot array, nor any data it points to (recursively). +After the function is done, the caller is allowed to modify or deallocate +the array and any data it points to (recursively), except data +explicitly marked with :c:macro:`PySlot_STATIC`. + +Except when documented otherwise, multiple slots with the same ID +(:c:member:`~PySlot.sl_id`) may not occur in a single slots array. + +.. versionadded:: 3.15 + + Slot arrays generalize an earlier way of defining objects: + using :c:type:`PyType_Spec` with :c:type:`PyType_Slot` for types, and + :c:type:`PyModuleDef` with :c:type:`PyModuleDef_Slot` for modules. + The earlier API is :term:`soft deprecated`; there are no plans to remove it. + +Entries of the slots array use the following structure: + +.. c:type:: PySlot + + An entry in a slots array. Defined as: + + .. code-block:: c + + typedef struct { + uint16_t sl_id; + uint16_t sl_flags; + uint32_t _reserved; // must be 0 + union { + void *sl_ptr; + void (*sl_func)(void); + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; + } PySlot; + + .. c:member:: uint16_t sl_id + + A slot ID, chosen from: + + - ``Py_slot_*`` values documented in :ref:`pyslot-common-ids` below; + - ``Py_mod_*`` values for modules, as documented in :ref:`c_module_slots`; + - Values for types, as documented in :ref:`pyslot_type_slot_ids`. + + A :c:member:`!sl_id` of zero (:c:macro:`Py_slot_end`) marks the end of a + slots array. + + .. c:member:: void *sl_ptr + void (*sl_func)(void) + Py_ssize_t sl_size + int64_t sl_int64 + uint64_t sl_uint64 + + The data for the slot. + These members are part of an anonymous union; + the member to use depends on which data type is required by the slot ID: + data pointer, function pointer, size, signed or unsigned + integer, respectively. + + Except when documented otherwise for a specific slot ID, pointers + (that is :c:member:`!sl_ptr` and :c:member:`!sl_func`) may not be NULL. + + .. c:member:: uint16_t sl_flags + + Zero or more of the following flags, OR-ed together: + + .. c:namespace:: NULL + + .. c:macro:: PySlot_STATIC + + All data the slot points to is statically allocated and constant. + Thus, the interpreter does not need to copy the information. + + This flag is implied for function pointers. + + The flag applies even to data the slot points to “indirectly”, + except for slots nested via :c:macro:`Py_slot_subslots` which may + have their own :c:macro:`!PySlot_STATIC` flags. + For example, if applied to a :c:macro:`Py_tp_members` slot that + points to an array of :c:type:`PyMemberDef` structures, + then the entire array, as well as the name and doc strings + in its elements, must be static and constant. + + .. c:macro:: PySlot_INTPTR + + The data is stored in ``sl_ptr``; CPython will cast it to + the appropriate type. + + This flag can simplify porting from the older :c:type:`PyType_Slot` + and :c:type:`PyModuleDef_Slot` structures. + + .. c:macro:: PySlot_OPTIONAL + + If the slot ID is unknown, the interpreter should ignore the + slot, rather than fail. + + For example, if Python 3.16 adds a new feature with a new slot ID,attr + the corresponding slot may be marked :c:macro:`!PySlot_OPTIONAL` + so that Python 3.15 ignores it. + + Note that the "optionality" only applies to unknown slot IDs. + This flag does not make Python skip invalid values of known slots. + + .. versionadded:: 3.15 + + +Convenience macros +------------------ + +.. c:macro:: PySlot_DATA(name, value) + PySlot_FUNC(name, value) + PySlot_SIZE(name, value) + PySlot_INT64(name, value) + PySlot_UINT64(name, value) + PySlot_STATIC_DATA(name, value) + + Convenience macros to define :c:type:`!PySlot` structures with + :c:member:`~PySlot.sl_id` and a particular union member set. + + :c:macro:`!PySlot_STATIC_DATA` sets the :c:macro:`PySlot_STATIC` flag; + others set no flags. + + Note that these macros use *designated initializers*, a C language feature + that C++ added in the 2020 version of the standard. + If your code needs to be compatible with C++11 or older, + use :c:macro:`PySlot_PTR` instead. + + Defined as:: + + #define PySlot_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_ptr=(void*)(VALUE)} + + #define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=NAME, .sl_func=(VALUE)} + + #define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=NAME, .sl_size=(VALUE)} + + #define PySlot_INT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_int64=(VALUE)} + + #define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=NAME, .sl_uint64=(VALUE)} + + #define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=NAME, .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_END + + Convenience macro to mark the end of a :c:type:`!PySlot` array. + + Defined as:: + + #define PySlot_END {0} + + .. versionadded:: 3.15 + +.. c:macro:: PySlot_PTR(name, value) + PySlot_PTR_STATIC(name, value) + + Convenience macros for use in C++11-compatible code. + This version of C++ does not allow setting arbitrary union members in + literals; instead, these macros set the :c:macro:`PySlot_INTPTR` flag and cast + the value to ``(void*)``. + + Defined as:: + + #define PySlot_PTR(NAME, VALUE) \ + {NAME, PySlot_INTPTR, {0}, {(void*)(VALUE)}} + + #define PySlot_PTR_STATIC(NAME, VALUE) \ + {NAME, PySlot_INTPTR|Py_SLOT_STATIC, {0}, {(void*)(VALUE)}} + + .. versionadded:: 3.15 + +.. _pyslot-common-ids: + +Common slot IDs +--------------- + +The following slot IDs may be used in both type and module definitions. + +.. c:macro:: Py_slot_end + + Marks the end of a slots array. + Defined as zero. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_subslots + + Nested slots array. + + The value (:c:member:`~PySlot.sl_ptr`) should point to an array of + :c:type:`PySlot` structures. + The slots in the array (up to but not including the zero-ID + terminator) will be treated as if they were inserted if the current + slot array, at the point :c:macro:`!Py_slot_subslots` appears. + + Slot nesting depth is limited to 5 levels. + This restriction may be lifted in the future. + + .. versionadded:: 3.15 + +.. c:macro:: Py_slot_invalid + + Reserved; will always be treated as an unknown slot ID. + Defined as ``UINT16_MAX`` (``0xFFFF``). + + When used with the :c:macro:`PySlot_OPTIONAL` flag, defines a slot with + no effect. + Without the flag, processing a slot with this ID will fail. + + .. versionadded:: 3.15 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index fe2cb89f999a848..13e5d5c96135c0e 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -51,145 +51,212 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-abi: .. _stable-application-binary-interface: -Stable Application Binary Interface -=================================== +Stable Application Binary Interfaces +==================================== -For simplicity, this document talks about *extensions*, but the Limited API -and Stable ABI work the same way for all uses of the API – for example, -embedding Python. +Python's :dfn:`Stable ABI` allows extensions to be compatible with multiple +versions of Python, without recompilation. -.. _limited-c-api: +.. note:: -Limited C API -------------- + For simplicity, this document talks about *extensions*, but Stable ABI + works the same way for all uses of the API – for example, embedding Python. -Python 3.2 introduced the *Limited API*, a subset of Python's C API. -Extensions that only use the Limited API can be -compiled once and be loaded on multiple versions of Python. -Contents of the Limited API are :ref:`listed below `. +There are two Stable ABIs: -.. c:macro:: Py_LIMITED_API +- ``abi3``, introduced in Python 3.2, is compatible with + **non**-:term:`free-threaded ` builds of CPython. - Define this macro before including ``Python.h`` to opt in to only use - the Limited API, and to select the Limited API version. +- ``abi3t``, introduced in Python 3.15, is compatible with + :term:`free-threaded ` builds of CPython. + It has stricter API limitations than ``abi3``. - Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` - corresponding to the lowest Python version your extension supports. - The extension will be ABI-compatible with all Python 3 releases - from the specified one onward, and can use Limited API introduced up to that - version. + .. versionadded:: 3.15 - Rather than using the ``PY_VERSION_HEX`` macro directly, hardcode a minimum - minor version (e.g. ``0x030A0000`` for Python 3.10) for stability when - compiling with future Python versions. + ``abi3t`` was added in :pep:`803` - You can also define ``Py_LIMITED_API`` to ``3``. This works the same as - ``0x03020000`` (Python 3.2, the version that introduced Limited API). +It is possible for an extension to be compiled for *both* ``abi3`` and +``abi3t`` at the same time; the result will be compatible with +both free-threaded and non-free-threaded builds of Python. +Currently, this has no downsides compared to compiling for ``abi3t`` only. -.. c:macro:: Py_TARGET_ABI3T +Each Stable ABI is versioned using the first two numbers of the Python version. +For example, Stable ABI 3.14 corresponds to Python 3.14. +An extension compiled for Stable ABI 3.x is ABI-compatible with Python 3.x +and above. - Define this macro before including ``Python.h`` to opt in to only use - the Limited API for :term:`free-threaded builds `, - and to select the Limited API version. +Extensions that target a stable ABI must only use a limited subset of +the C API. This subset is known as the :dfn:`Limited API`; its contents +are :ref:`listed below `. - .. seealso:: :pep:`803` +On Windows, extensions that use a Stable ABI should be linked against +``python3.dll`` rather than a version-specific library such as +``python39.dll``. +This library only exposes the relevant symbols. - .. versionadded:: 3.15 +On some platforms, Python will look for and load shared library files named +with the ``abi3`` or ``abi3t`` tag (for example, ``mymodule.abi3.so``). +:term:`Free-threaded ` interpreters only recognize the +``abi3t`` tag, while non-free-threaded ones will prefer ``abi3`` but fall back +to ``abi3t``. +Thus, extensions compatible with both ABIs should use the ``abi3t`` tag. + +Python does not necessarily check that extensions it loads +have compatible ABI. +Extension authors are encouraged to add a check using the :c:macro:`Py_mod_abi` +slot or the :c:func:`PyABIInfo_Check` function, but the user +(or their packaging tool) is ultimately responsible for ensuring that, +for example, extensions built for Stable ABI 3.10 are not installed for lower +versions of Python. +All functions in Stable ABI are present as functions in Python's shared +library, not solely as macros. +This makes them usable in languages that don't use the C +preprocessor, including Python's :py:mod:`ctypes`. -.. _stable-abi: -Stable ABI ----------- +.. _abi3-compiling: -To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain ABI-compatible across Python 3.x versions. +Compiling for Stable ABI +------------------------ .. note:: - The Stable ABI prevents ABI issues, like linker errors due to missing - symbols or data corruption due to changes in structure layouts or function - signatures. - However, other changes in Python can change the *behavior* of extensions. - See Python's Backwards Compatibility Policy (:pep:`387`) for details. + Build tools (such as, for example, meson-python, scikit-build-core, + or Setuptools) often have a mechanism for setting macros and synchronizing + them with extension filenames and other metadata. + Prefer using such a mechanism, if it exists, over defining the + macros manually. -The Stable ABI contains symbols exposed in the :ref:`Limited API -`, but also other ones – for example, functions necessary to -support older versions of the Limited API. + The rest of this section is mainly relevant for tool authors, and for + people who compile extensions manually. -On Windows, extensions that use the Stable ABI should be linked against -``python3.dll`` rather than a version-specific library such as -``python39.dll``. + .. seealso:: `list of recommended tools`_ in the Python Packaging User Guide -On some platforms, Python will look for and load shared library files named -with the ``abi3`` tag (e.g. ``mymodule.abi3.so``). -It does not check if such extensions conform to a Stable ABI. -The user (or their packaging tools) need to ensure that, for example, -extensions built with the 3.10+ Limited API are not installed for lower -versions of Python. + .. _list of recommended tools: https://packaging.python.org/en/latest/guides/tool-recommendations/#build-backends-for-extension-modules -All functions in the Stable ABI are present as functions in Python's shared -library, not solely as macros. This makes them usable from languages that don't -use the C preprocessor. +To compile for a Stable ABI, define one or both of the following macros +to the lowest Python version your extension should support, in +:c:macro:`Py_PACK_VERSION` format. +Typically, you should choose a specific value rather than the version of +the Python headers you are compiling against. +The macros must be defined before including ``Python.h``. +Since :c:macro:`Py_PACK_VERSION` is not available at this point, you +will need to use the numeric value directly. +For reference, the values for a few recent Python versions are: -Limited API Scope and Performance ---------------------------------- +.. version-hex-cheatsheet:: -The goal for the Limited API is to allow everything that is possible with the +When one of the macros is defined, ``Python.h`` will only expose API that is +compatible with the given Stable ABI -- that is, the +:ref:`Limited API ` plus some definitions that need to be +visible to the compiler but should not be used directly. +When both are defined, ``Python.h`` will only expose API compatible with +both Stable ABIs. + +.. c:macro:: Py_LIMITED_API + + Target ``abi3``, that is, + non-:term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + +.. c:macro:: Py_TARGET_ABI3T + + Target ``abi3t``, that is, + :term:`free-threaded builds ` of CPython. + See :ref:`above ` for common information. + + .. versionadded:: 3.15 + +Both macros specify a target ABI; the different naming style is due to +backwards compatibility. + +.. admonition:: Historical note + + You can also define ``Py_LIMITED_API`` as ``3``. This works the same as + ``0x03020000`` (Python 3.2, the version that introduced Stable ABI). + +When both are defined, ``Python.h`` may, or may not, redefine +:c:macro:`!Py_LIMITED_API` to match :c:macro:`!Py_TARGET_ABI3T`. + +On a :term:`free-threaded build` -- that is, when +:c:macro:`Py_GIL_DISABLED` is defined -- :c:macro:`!Py_TARGET_ABI3T` +defaults to the value of :c:macro:`!Py_LIMITED_API`. +This means that there are two ways to build for both ``abi3`` and ``abi3t``: + +- define both :c:macro:`!Py_LIMITED_API` and :c:macro:`!Py_TARGET_ABI3T`, or +- define only :c:macro:`!Py_LIMITED_API` and: + + - on Windows, define :c:macro:`!Py_GIL_DISABLED`; + - on other systems, use the headers of free-threaded build of Python. + + +.. _limited-api-scope-and-performance: + +Stable ABI Scope and Performance +-------------------------------- + +The goal for Stable ABI is to allow everything that is possible with the full C API, but possibly with a performance penalty. +Generally, compatibility with Stable ABI will require some changes to an +extension's source code. -For example, while :c:func:`PyList_GetItem` is available, its “unsafe” macro +For example, while :c:func:`PyList_GetItem` is available, its "unsafe" macro variant :c:func:`PyList_GET_ITEM` is not. The macro can be faster because it can rely on version-specific implementation details of the list object. -Without ``Py_LIMITED_API`` defined, some C API functions are inlined or -replaced by macros. -Defining ``Py_LIMITED_API`` disables this inlining, allowing stability as +For another example, when *not* compiling for Stable ABI, some C API +functions are inlined or replaced by macros. +Compiling for Stable ABI disables this inlining, allowing stability as Python's data structures are improved, but possibly reducing performance. -By leaving out the ``Py_LIMITED_API`` definition, it is possible to compile -a Limited API extension with a version-specific ABI. This can improve -performance for that Python version, but will limit compatibility. -Compiling with ``Py_LIMITED_API`` will then yield an extension that can be -distributed where a version-specific one is not available – for example, -for prereleases of an upcoming Python version. +By leaving out the :c:macro:`!Py_LIMITED_API` or :c:macro:`!Py_TARGET_ABI3T` +definition, it is possible to compile Stable-ABI-compatible source +for a version-specific ABI. +A potentially faster version-specific extension can then be distributed +alongside a version compiled for Stable ABI -- a slower but more compatible +fallback. + +.. _limited-api-caveats: -Limited API Caveats -------------------- +Stable ABI Caveats +------------------ -Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that -code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI -`. ``Py_LIMITED_API`` only covers definitions, but an API also -includes other issues, such as expected semantics. +Note that compiling for Stable ABI is *not* a complete guarantee that code will +be compatible with the expected Python versions. +Stable ABI prevents *ABI* issues, like linker errors due to missing +symbols or data corruption due to changes in structure layouts or function +signatures. +However, other changes in Python can change the *behavior* of extensions. -One issue that ``Py_LIMITED_API`` does not guard against is calling a function -with arguments that are invalid in a lower Python version. +One issue that the :c:macro:`Py_TARGET_ABI3T` and :c:macro:`Py_LIMITED_API` +macros do not guard against is calling a function with arguments that are +invalid in a lower Python version. For example, consider a function that starts accepting ``NULL`` for an argument. In Python 3.9, ``NULL`` now selects a default behavior, but in Python 3.8, the argument will be used directly, causing a ``NULL`` dereference and crash. A similar argument works for fields of structs. -Another issue is that some struct fields are currently not hidden when -``Py_LIMITED_API`` is defined, even though they're part of the Limited API. - For these reasons, we recommend testing an extension with *all* minor Python -versions it supports, and preferably to build with the *lowest* such version. +versions it supports. We also recommend reviewing documentation of all used API to check if it is explicitly part of the Limited API. Even with ``Py_LIMITED_API`` defined, a few private declarations are exposed for technical reasons (or even unintentionally, as bugs). -Also note that the Limited API is not necessarily stable: compiling with -``Py_LIMITED_API`` with Python 3.8 means that the extension will -run with Python 3.12, but it will not necessarily *compile* with Python 3.12. -In particular, parts of the Limited API may be deprecated and removed, -provided that the Stable ABI stays stable. +Also note that while compiling with ``Py_LIMITED_API`` 3.8 means that the +extension should *load* on Python 3.12, and *compile* with Python 3.12, +the same source will not necessarily compile with ``Py_LIMITED_API`` +set to 3.12. +In general, parts of the Limited API may be deprecated and removed, +provided that Stable ABI stays stable. .. _stable-abi-platform: @@ -199,12 +266,12 @@ Platform Considerations ABI stability depends not only on Python, but also on the compiler used, lower-level libraries and compiler options. For the purposes of -the :ref:`Stable ABI `, these details define a “platform”. They +the :ref:`Stable ABIs `, these details define a “platform”. They usually depend on the OS type and processor architecture It is the responsibility of each particular distributor of Python to ensure that all Python versions on a particular platform are built -in a way that does not break the Stable ABI. +in a way that does not break the Stable ABIs, or the version-specific ABIs. This is the case with Windows and macOS releases from ``python.org`` and many third-party distributors. @@ -312,7 +379,7 @@ The full API is described below for advanced use cases. .. c:macro:: PyABIInfo_STABLE - Specifies that the stable ABI is used. + Specifies that Stable ABI is used. .. c:macro:: PyABIInfo_INTERNAL @@ -323,15 +390,22 @@ The full API is described below for advanced use cases. .. c:macro:: PyABIInfo_FREETHREADED - Specifies ABI compatible with free-threading builds of CPython. + Specifies ABI compatible with :term:`free-threaded builds + ` of CPython. (That is, ones compiled with :option:`--disable-gil`; with ``t`` in :py:data:`sys.abiflags`) .. c:macro:: PyABIInfo_GIL - Specifies ABI compatible with non-free-threading builds of CPython + Specifies ABI compatible with non-free-threaded builds of CPython (ones compiled *without* :option:`--disable-gil`). + .. c:macro:: PyABIInfo_FREETHREADING_AGNOSTIC + + Specifies ABI compatible with both free-threaded and + non-free-threaded builds of CPython, that is, both + ``abi3`` and ``abi3t``. + .. c:member:: uint32_t build_version The version of the Python headers used to build the code, in the format @@ -345,10 +419,11 @@ The full API is described below for advanced use cases. The ABI version. - For the Stable ABI, this field should be the value of - :c:macro:`Py_LIMITED_API` - (except if :c:macro:`Py_LIMITED_API` is ``3``; use - :c:expr:`Py_PACK_VERSION(3, 2)` in that case). + For Stable ABI, this field should be the value of + :c:macro:`Py_LIMITED_API` or :c:macro:`Py_TARGET_ABI3T`. + If both are defined, use the smaller value. + (If :c:macro:`Py_LIMITED_API` is ``3``; use + :c:expr:`Py_PACK_VERSION(3, 2)` instead of ``3``.) Otherwise, it should be set to :c:macro:`PY_VERSION_HEX`. @@ -365,12 +440,13 @@ The full API is described below for advanced use cases. .. versionadded:: 3.15 +.. _limited-c-api: .. _limited-api-list: Contents of Limited API ======================= - -Currently, the :ref:`Limited API ` includes the following items: +This is the definitive list of :ref:`Limited API ` for +Python |version|: .. limited-api-list:: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index c0d2663adefc6b5..1ddde5d136b3bb1 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -33,6 +33,13 @@ under :ref:`reference counting `. The members must not be accessed directly; instead use macros such as :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_refcnt` and :c:member:`!ob_type` fields are available, + but using them directly is discouraged. + .. c:member:: Py_ssize_t ob_refcnt The object's reference count, as returned by :c:macro:`Py_REFCNT`. @@ -72,6 +79,19 @@ under :ref:`reference counting `. instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + In the :ref:`Stable ABI ` for Free-Threaded Builds (``abi3t``), + this struct is opaque; its size and layout may change between + Python versions. + In Stable ABI for non-free-threaded builds (``abi3``), the + :c:member:`!ob_base` and :c:member:`!ob_size` fields are available, + but using them directly is discouraged. + + .. c:member:: PyObject ob_base + + Common object header. + Typically, this field is not accessed directly; instead + :c:type:`!PyVarObject` can be cast to :c:type:`PyObject`. + .. c:member:: Py_ssize_t ob_size A size field, whose contents should be considered an object's internal @@ -434,8 +454,8 @@ method. The method will be passed the type object as the first parameter rather than an instance of the type. This is used to create *class methods*, - similar to what is created when using the :func:`classmethod` built-in - function. + similar to what is created when using the :deco:`classmethod` built-in + decorator. .. c:macro:: METH_STATIC @@ -444,7 +464,7 @@ method. The method will be passed ``NULL`` as the first parameter rather than an instance of the type. This is used to create *static methods*, similar to - what is created when using the :func:`staticmethod` built-in function. + what is created when using the :deco:`staticmethod` built-in decorator. One other constant controls whether a method is loaded in place of another definition with the same method name. diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst index 44e3fc96841aacb..83c3fc3d801e9bd 100644 --- a/Doc/c-api/subinterpreters.rst +++ b/Doc/c-api/subinterpreters.rst @@ -399,6 +399,27 @@ High-level APIs .. versionadded:: 3.9 +.. c:function:: void _PyInterpreterState_SetEvalFrameAllowSpecialization(PyInterpreterState *interp, int allow_specialization) + + Enables or disables specialization why a custom frame evaluator is in place. + + If *allow_specialization* is non-zero, the adaptive specializer will + continue to specialize bytecodes even though a custom eval frame function + is set. When *allow_specialization* is zero, setting a custom eval frame + disables specialization. The standard interpreter loop will continue to deopt + while a frame evaluation API is in place - the frame evaluation function needs + to handle the specialized opcodes to take advantage of this. + + .. versionadded:: 3.15 + +.. c:function:: int _PyInterpreterState_IsSpecializationEnabled(PyInterpreterState *interp) + + Return non-zero if adaptive specialization is enabled for the interpreter. + Specialization is enabled when no custom eval frame function is set, or + when one is set with *allow_specialization* enabled. + + .. versionadded:: 3.15 + Low-level APIs -------------- diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst index 53c9faeae35464d..6f18c047a24a92f 100644 --- a/Doc/c-api/synchronization.rst +++ b/Doc/c-api/synchronization.rst @@ -84,11 +84,6 @@ there is no :c:type:`PyObject` -- for example, when working with a C type that does not extend or wrap :c:type:`PyObject` but still needs to call into the C API in a manner that might lead to deadlocks. -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - .. note:: Operations that need to lock two objects at once must use @@ -114,12 +109,15 @@ section API avoids potential deadlocks due to reentrancy and lock ordering by allowing the runtime to temporarily suspend the critical section if the code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. +.. _critical-section-macros: + .. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) Acquires the per-object lock for the object *op* and begins a critical section. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: { PyCriticalSection _py_cs; @@ -150,7 +148,8 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. Ends the critical section and releases the per-object lock. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: PyCriticalSection_End(&_py_cs); } @@ -179,7 +178,8 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. Locks the mutexes *m1* and *m2* and begins a critical section. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: { PyCriticalSection2 _py_cs2; @@ -196,7 +196,8 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. Ends the critical section and releases the per-object locks. - In the free-threaded build, this macro expands to:: + In the free-threaded build, and when building for the + :ref:`Stable ABI `, this macro expands to:: PyCriticalSection2_End(&_py_cs2); } @@ -205,6 +206,48 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +Low-level critical section API +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions and structs are exposed for cases where C macros +are not available. + +.. c:function:: void PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) + void PyCriticalSection_End(PyCriticalSection *c) + void PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) + void PyCriticalSection2_End(PyCriticalSection2 *c); + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.13 + +.. c:type:: PyCriticalSection + PyCriticalSection2 + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + Note that the contents of the structures are private and their meaning may + change in future Python versions. + + .. versionadded:: 3.13 + +.. c:function:: void PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + void PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + + .. (These need to be in a separate section without a Stable ABI annotation.) + + To be used only as in the macro expansions + listed :ref:`earlier in this section `. + + In non-:term:`free-threaded ` builds of CPython, these + functions do nothing. + + .. versionadded:: 3.14 + Legacy locking APIs ------------------- diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index ee73c1c8adaa7b3..0a446f86e22eaf6 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -433,7 +433,7 @@ accessible to C code. They all work with the current interpreter thread's This function is safe to call before :c:func:`Py_Initialize`. When called after runtime initialization, existing audit hooks are notified and may silently abort the operation by raising an error subclassed from - :class:`Exception` (other errors will not be silenced). + :class:`RuntimeError` (other errors will not be silenced). The hook function is always called with an :term:`attached thread state` by the Python interpreter that raised the event. @@ -447,7 +447,7 @@ accessible to C code. They all work with the current interpreter thread's If the interpreter is initialized, this function raises an auditing event ``sys.addaudithook`` with no arguments. If any existing hooks raise an - exception derived from :class:`Exception`, the new hook will not be + exception derived from :class:`RuntimeError`, the new hook will not be added and the exception is cleared. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. @@ -462,6 +462,11 @@ accessible to C code. They all work with the current interpreter thread's .. versionadded:: 3.8 + .. versionchanged:: 3.8.1 + + Exceptions derived from :class:`Exception` but not :class:`RuntimeError` + are no longer suppressed. + .. _processcontrol: diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst index 3b761d0c657cbd3..f48af654e7422bc 100644 --- a/Doc/c-api/threads.rst +++ b/Doc/c-api/threads.rst @@ -61,9 +61,9 @@ as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. -If it turns out that you do need to create a thread state, call :c:func:`PyThreadState_New` -followed by :c:func:`PyThreadState_Swap`, or use the dangerous -:c:func:`PyGILState_Ensure` function. +If it turns out that you do need to create a thread state, it is recommended to +use :c:func:`PyThreadState_Ensure` or :c:func:`PyThreadState_EnsureFromView`, +which will manage the thread state for you. .. _detaching-thread-state: @@ -178,8 +178,12 @@ example usage in the Python source distribution. declaration. -Non-Python created threads --------------------------- +.. _non-python-created-threads: +.. _c-api-foreign-threads: + + +Using the C API from foreign threads +------------------------------------ When threads are created using the dedicated Python APIs (such as the :mod:`threading` module), a thread state is automatically associated with them, @@ -192,70 +196,330 @@ of a callback API provided by the aforementioned third-party library), you must first register these threads with the interpreter by creating a new thread state and attaching it. -The most robust way to do this is through :c:func:`PyThreadState_New` followed -by :c:func:`PyThreadState_Swap`. +The easiest way to do this is through :c:func:`PyThreadState_Ensure` +or :c:func:`PyThreadState_EnsureFromView`. .. note:: - ``PyThreadState_New`` requires an argument pointing to the desired + These functions require an argument pointing to the desired interpreter; such a pointer can be acquired via a call to - :c:func:`PyInterpreterState_Get` from the code where the thread was - created. + :c:func:`PyInterpreterGuard_FromCurrent` (for ``PyThreadState_Ensure``) or + :c:func:`PyInterpreterView_FromCurrent` (for ``PyThreadState_EnsureFromView``) + from the function that creates the thread. If no pointer is available (such + as when the given native thread library doesn't provide a data argument), + :c:func:`PyInterpreterView_FromMain` can be used to get a view for the main + interpreter, but note that this will make the code incompatible with + subinterpreters. + For example:: - /* The return value of PyInterpreterState_Get() from the - function that created this thread. */ - PyInterpreterState *interp = thread_data->interp; + // The return value of PyInterpreterGuard_FromCurrent() from the + // function that created this thread. + PyInterpreterGuard *guard = thread_data->guard; + + // Create a new thread state for the interpreter. + PyThreadStateToken *token = PyThreadState_Ensure(guard); + if (token == NULL) { + PyInterpreterGuard_Close(guard); + return; + } + + // We have a valid thread state -- perform Python actions here. + result = CallSomeFunction(); + // Evaluate result or handle exceptions. + + // Release the thread state. No calls to the C API are allowed beyond this + // point. + PyThreadState_Release(token); + PyInterpreterGuard_Close(guard); + + +Keep in mind that calling ``PyThreadState_Ensure`` might not always create a new +thread state, and calling ``PyThreadState_Release`` might not always detach it. +These functions may reuse an existing attached thread state, or may re-attach +a thread state that was previously attached for the current thread. + +.. seealso:: + :pep:`788` + +.. _c-api-reuse-thread-state: + +Reusing a thread state across repeated calls +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Creating and destroying a :c:type:`PyThreadState` is not free, and is more +expensive on a :term:`free-threaded build`. A foreign thread that calls into +the interpreter many times -- for example, a worker thread in a native thread +pool -- should avoid creating a fresh thread state on every entry and +destroying it on every exit. Instead, set up one thread state when the thread +starts (or lazily on its first call into Python), attach and detach it around +each call, and tear it down once when the thread exits. + +Manage the thread state explicitly with :c:func:`PyThreadState_New`, attaching +and detaching it with :c:func:`PyEval_RestoreThread` and +:c:func:`PyEval_SaveThread`. This happens in three distinct phases, at +different points in the thread's life. + +When the thread starts, create one thread state for it. ``interp`` is the +target interpreter, captured by the code that created this thread while it held +an attached thread state (for example via :c:func:`PyInterpreterState_Get`):: - /* Create a new thread state for the interpreter. It does not start out - attached. */ PyThreadState *tstate = PyThreadState_New(interp); - /* Attach the thread state, which will acquire the GIL. */ - PyThreadState_Swap(tstate); +Then, on each call into Python -- which may happen many times over the thread's +life -- attach the thread state, make the Python C API calls that require it, +and detach again so the thread does not hold the GIL while off doing non-Python +work:: - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ + PyEval_RestoreThread(tstate); + result = CallSomeFunction(); /* your Python C API calls go here */ + PyEval_SaveThread(); - /* Destroy the thread state. No Python API allowed beyond this point. */ +When the thread is finished calling into Python, destroy the thread state once:: + + PyEval_RestoreThread(tstate); PyThreadState_Clear(tstate); PyThreadState_DeleteCurrent(); -.. warning:: +The general-purpose entry points for calling in from a foreign thread -- +:c:func:`PyThreadState_Ensure` and the older :c:func:`PyGILState_Ensure` -- do +*not* guarantee a persistent thread state: their thread-state lifetime is +deliberately implementation-defined, so a matched acquire/release pair may +create and destroy a thread state each time. Use :c:func:`PyThreadState_New`, +as shown here, whenever you specifically want to reuse one thread state across +calls. + +The code that created the foreign thread must arrange for the shutdown sequence +to run before the thread exits, and before :c:func:`Py_FinalizeEx` is called. +If interpreter finalization begins first, the shutdown +:c:func:`PyEval_RestoreThread` call will hang the thread rather than return (see +:ref:`cautions-regarding-runtime-finalization`). If the thread exits without +running the shutdown sequence, the thread state is leaked for the remainder of +the process. + +.. _c-api-attach-detach: + +Attaching/detaching thread states +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: PyThreadStateToken *PyThreadState_Ensure(PyInterpreterGuard *guard) + + Ensure that the thread has an attached thread state for the + interpreter protected by *guard*, and thus can safely invoke that + interpreter. + + It is OK to call this function if the thread already has an + attached thread state, as long as there is a subsequent call to + :c:func:`PyThreadState_Release` that matches this one (meaning that "nested" + calls to this function are permitted). + + The function's effect (if any) will be reversed by the matching call to + :c:func:`PyThreadState_Release`. + + On error, this function returns ``NULL`` *without* an exception set. + Do not call :c:func:`!PyThreadState_Release` in this case. + + On success, this function returns a pointer value that must be passed + to the matching call to :c:func:`!PyThreadState_Release`. + + The conditions in which this function creates a new :term:`thread state` are + considered unstable and implementation-dependent. If you need to control the + exact lifetime of a thread state, consider using :c:func:`PyThreadState_New`. + However, do not avoid this function solely on the basis that the lifetime + of the thread state may be inconsistent across versions; changes to this + function will be done with caution and in a backwards-compatible manner. + In particular, the saving of thread-local variables and similar state will + be retained across Python versions. + + .. impl-detail:: + + The exact behavior of whether this function creates a new thread state is + described below, but be aware that this may change in the future. + + First, this function checks if an attached thread state is present. + If there is, this function then checks if the interpreter of that + thread state matches the interpreter guarded by *guard*. If that is + the case, this function simply marks the thread state as being used + by a ``PyThreadState_Ensure`` call and returns. + + If there is no attached thread state, then this function checks if any + thread state has been used by the current OS thread. (This is + returned by :c:func:`PyGILState_GetThisThreadState`.) + If there was, then this function checks if that thread state's interpreter + matches *guard*. If it does, it is re-attached and marked as used. + + Otherwise, if both of the above cases fail, a new thread state is created + for *guard*. It is then attached and marked as owned by ``PyThreadState_Ensure``. + + .. versionadded:: 3.15 + + +.. c:function:: PyThreadStateToken *PyThreadState_EnsureFromView(PyInterpreterView *view) + + Get an attached thread state for the interpreter referenced by *view*. + + The behavior and return value are the same as for :c:func:`PyThreadState_Ensure`; + additionally, if the function succeeds, the interpreter referenced by *view* will + be implicitly guarded. The guard will be released upon the corresponding + :c:func:`PyThreadState_Release` call. + + .. versionadded:: 3.15 + + +.. c:function:: void PyThreadState_Release(PyThreadStateToken *token) + + Undo a :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` call. + + This must be called exactly once for each successful *Ensure* call, with + *token* set to that call's return value. + + The state that was attached before the corresponding *Ensure* call + (if any) will be attached when :c:func:`PyThreadState_Release` returns. + + The exact behavior of whether this function deletes a thread state is + considered unstable and implementation-dependent. + + .. impl-detail:: - If the interpreter finalized before ``PyThreadState_Swap`` was called, then - ``interp`` will be a dangling pointer! + Currently, this function will decrement an internal counter on the + attached thread state. If this counter ever reaches below zero, this + function emits a fatal error (via :c:func:`Py_FatalError`). + If the attached thread state is owned by ``PyThreadState_Ensure``, then the + attached thread state will be deallocated and deleted upon the internal counter + reaching zero. Otherwise, nothing happens when the counter reaches zero. + + .. versionadded:: 3.15 + +.. c:type:: PyThreadStateToken + + An opaque token retrieved from a :c:func:`PyThreadState_Ensure` call + and passed to a corresponding :c:func:`PyThreadState_Release` call. + + +.. _legacy-api: .. _gilstate: -Legacy API ----------- +GIL-state APIs +-------------- + +The following APIs are generally not compatible with subinterpreters and +will hang the process during interpreter finalization (see +:ref:`cautions-regarding-runtime-finalization`). As such, these APIs were +:term:`soft deprecated` in Python 3.15 in favor of the :ref:`new APIs +`. + + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED -Another common pattern to call Python code from a non-Python thread is to use -:c:func:`PyGILState_Ensure` followed by a call to :c:func:`PyGILState_Release`. + The GIL was already held when :c:func:`PyGILState_Ensure` was called. -These functions do not work well when multiple interpreters exist in the Python -process. If no Python interpreter has ever been used in the current thread (which -is common for threads created outside Python), ``PyGILState_Ensure`` will create -and attach a thread state for the "main" interpreter (the first interpreter in -the Python process). + .. c:enumerator:: PyGILState_UNLOCKED -Additionally, these functions have thread-safety issues during interpreter -finalization. Using ``PyGILState_Ensure`` during finalization will likely -crash the process. + The GIL was not held when :c:func:`PyGILState_Ensure` was called. -Usage of these functions look like such:: - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); +.. c:function:: PyGILState_STATE PyGILState_Ensure() - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. + + This function has no way to return an error. As such, errors are either fatal + (that is, they send ``SIGABRT`` and crash the process; see + :c:func:`Py_FatalError`), or the thread will be permanently blocked (such as + during interpreter finalization). + + .. warning:: + Calling this function when the interpreter is finalizing will + infinitely hang the thread, which may cause deadlocks. + :ref:`cautions-regarding-runtime-finalization` for more details. + + In addition, this function generally does not work with subinterpreters + when used from foreign threads, because this function has no way of + knowing which interpreter created the thread (and as such, will implicitly + pick the main interpreter). + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Ensure` or + :c:func:`PyThreadState_EnsureFromView` instead. + + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GIL-state API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Release` instead. + + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`thread state` that was most recently :term:`attached + ` for this thread. (If the most recent thread state + has been deleted, this returns ``NULL``.) + + If the caller has an attached thread state, it is returned. + + In other terms, this function returns the thread state that will be used by + :c:func:`PyGILState_Ensure`. If this returns ``NULL``, then + ``PyGILState_Ensure`` will create a new thread state. + + This function cannot fail. + + .. soft-deprecated:: 3.15 + Use :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + instead. + + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread has an :term:`attached thread state` + that matches the thread state returned by + :c:func:`PyGILState_GetThisThreadState`. If the caller has no attached thread + state or it otherwise doesn't match, then this returns ``0``. - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. + + This is mainly a helper/diagnostic function. + + .. versionadded:: 3.4 + + .. soft-deprecated:: 3.15 + Use ``PyThreadState_GetUnchecked() != NULL`` instead. .. _fork-and-threads: @@ -398,101 +662,6 @@ C extensions. thread if the runtime is finalizing. -GIL-state APIs --------------- - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:type:: PyGILState_STATE - - The type of the value returned by :c:func:`PyGILState_Ensure` and passed to - :c:func:`PyGILState_Release`. - - .. c:enumerator:: PyGILState_LOCKED - - The GIL was already held when :c:func:`PyGILState_Ensure` was called. - - .. c:enumerator:: PyGILState_UNLOCKED - - The GIL was not held when :c:func:`PyGILState_Ensure` was called. - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. warning:: - Calling this function when the runtime is finalizing is unsafe. Doing - so will either hang the thread until the program ends, or fully crash - the interpreter in rare cases. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. note:: - This function may return non-``NULL`` even when the :term:`thread state` - is detached. - Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. seealso:: :c:func:`PyThreadState_Get` - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its :term:`thread state ` initialized - via :c:func:`PyGILState_Ensure` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. note:: - If the current Python process has ever created a subinterpreter, this - function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` - for most cases. - - .. versionadded:: 3.4 - - Low-level APIs -------------- @@ -622,10 +791,10 @@ Low-level APIs .. c:function:: PyObject* PyThreadState_GetDict() Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in + information. Each extension should use a unique key to store a state in the dictionary. It is okay to call this function when no :term:`thread state` is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no + ``NULL`` and no exception has been raised, then the caller should assume no thread state is attached. @@ -704,7 +873,7 @@ pointer and a void pointer argument. possible. If the main thread is busy executing a system call, *func* won't be called before the system call returns. This function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. + arbitrary C threads. Instead, use :c:func:`PyThreadState_EnsureFromView`. .. versionadded:: 3.1 @@ -755,7 +924,7 @@ pointer and a void pointer argument. To prevent naive misuse, you must write your own C extension to call this. This function must be called with an :term:`attached thread state`. - This function does not steal any references to *exc*. + This function does not :term:`steal` any references to *exc*. This function does not necessarily interrupt system calls such as :py:func:`~time.sleep`. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index ba4c6b93de4c11e..e8be4762dc33a12 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -104,8 +104,9 @@ Tuple Objects .. note:: - This function "steals" a reference to *o* and discards a reference to - an item already in the tuple at the affected position. + This function ":term:`steals `" a reference to *o* and discards + a reference to an item already in the tuple at the affected position + (unless it was NULL). .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -118,7 +119,7 @@ Tuple Objects .. note:: - This function "steals" a reference to *o*, and, unlike + This function ":term:`steals `" a reference to *o*, and, unlike :c:func:`PyTuple_SetItem`, does *not* discard a reference to any item that is being replaced; any reference in the tuple at position *pos* will be leaked. @@ -263,7 +264,7 @@ type. .. note:: - This function "steals" a reference to *o*. + This function ":term:`steals `" a reference to *o*. .. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index c9bb5c3f09ac186..2e48c56bd2c5a5b 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -3,7 +3,7 @@ .. _typeobjects: Type Objects ------------- +============ .. index:: pair: object; type @@ -37,6 +37,10 @@ Type Objects Clear the internal lookup cache. Return the current version tag. + .. versionchanged:: 3.16 + This function is now a no-op as the type cache is now implemented + per-type. It still returns the current version tag. + .. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type) Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily @@ -110,11 +114,16 @@ Type Objects :c:func:`!_PyType_Lookup` is not called on *type* between the modifications; this is an implementation detail and subject to change.) + The callback is also invoked when a watched heap type is deallocated. + An extension should never call ``PyType_Watch`` with a *watcher_id* that was not returned to it by a previous call to :c:func:`PyType_AddWatcher`. .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback is now also invoked when a watched heap type is deallocated. + .. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) @@ -138,8 +147,17 @@ Type Objects called on *type* or any type in its MRO; violating this rule could cause infinite recursion. + The callback may be called during type deallocation. In this case, the type + object is temporarily resurrected (its reference count is at least 1) and all + its attributes are still valid. However, the callback should not store new + strong references to the type, as this would resurrect the object and prevent + its deallocation. + .. versionadded:: 3.12 + .. versionchanged:: 3.15 + The callback may now be called during deallocation of a watched heap type. + .. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) @@ -384,36 +402,19 @@ Type Objects * :py:mod:`weakref` -Creating Heap-Allocated Types -............................. - -The following functions and structs are used to create -:ref:`heap types `. - -.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) - - Create and return a :ref:`heap type ` from the *spec* - (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). +.. _creating-heap-types: - The metaclass *metaclass* is used to construct the resulting type object. - When *metaclass* is ``NULL``, the metaclass is derived from *bases* - (or *Py_tp_base[s]* slots if *bases* is ``NULL``, see below). +Creating Heap-Allocated Types +----------------------------- - Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not - supported, except if ``tp_new`` is ``NULL``. +The following function is used to create :ref:`heap types `: - The *bases* argument can be used to specify base classes; it can either - be only one class or a tuple of classes. - If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. - If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. - If that also is ``NULL``, the new type derives from :class:`object`. +.. c:function:: PyObject *PyType_FromSlots(const PySlot *slots) - The *module* argument can be used to record the module in which the new - class is defined. It must be a module object or ``NULL``. - If not ``NULL``, the module is associated with the new type and can later be - retrieved with :c:func:`PyType_GetModule`. - The associated module is not inherited by subclasses; it must be specified - for each class individually. + Create and return a :ref:`heap type ` from a :c:type:`!PySlot` + array. + See :ref:`capi-slots` for general information on slots, + and :ref:`pyslot_type_slot_ids` for slots specific to type creation. This function calls :c:func:`PyType_Ready` on the new type. @@ -430,8 +431,407 @@ The following functions and structs are used to create * :py:meth:`~object.__init_subclass__` is not called on any bases. * :py:meth:`~object.__set_name__` is not called on new descriptors. + Slots are typically defined as a global static constant arrays. + However, sometimes slot values are not statically known at compile time. + For example, slots like :c:data:`Py_tp_bases`, :c:data:`Py_tp_metaclass` + and :c:data:`Py_tp_module` require live Python objects. + In this case, it is recommended to put such slots on the stack, + and use :c:macro:`Py_slot_subslots` to refer to an array of static slots. + For example:: + + static const PySlot my_slots[] = { + PySlot_STATIC_DATA(Py_tp_name, "MyClass"), + PySlot_FUNC(Py_tp_repr, my_repr_func), + ... + PySlot_END + }; + + PyObject *make_my_class(PyObject *module) { + PySlot all_slots[] = { + PySlot_STATIC_DATA(Py_slot_subslots, my_slots), + PySlot_DATA(Py_tp_module, module), + PySlot_END + }; + return PyType_FromSlots(all_slots); + } + +Heap types created without the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag may be +modified, for example by setting attributes on them, as with classes defined +in Python code. +Sometimes, such modifications are necessary to fully initialize a type, +but you may wish to prevent users from changing the type after +the initialization is done: + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 + + +.. _pyslot_type_slot_ids: + +Type slot IDs +............. + +Most type slot IDs are named like the field names of the structures +:c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, +:c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and +:c:type:`PyAsyncMethods` with an added ``Py_`` prefix. +For example, use: + +* :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` +* :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` +* :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` + +The following slots need additional considerations when specified as slots: + +* :c:data:`Py_tp_name` +* :c:data:`Py_tp_basicsize` and :c:data:`Py_tp_extra_basicsize` +* :c:data:`Py_tp_itemsize` +* :c:data:`Py_tp_flags` + +Additional slots do not directly correspond to a :c:type:`!PyTypeObject` +struct field: + +* :c:data:`Py_tp_token` +* :c:data:`Py_tp_metaclass` +* :c:data:`Py_tp_module` + +The following “offset” fields cannot be set using :c:type:`PyType_Slot`: + +* :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) +* :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) +* :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in :ref:`PyMemberDef `) + +If it is not possible to switch to a ``MANAGED`` flag (for example, +for vectorcall or to support Python older than 3.12), specify the +offset in :c:data:`Py_tp_members`. +See :ref:`PyMemberDef documentation ` +for details. + +The following internal fields cannot be set at all when creating a heap +type: + +* :c:member:`~PyTypeObject.tp_dict`, + :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, + :c:member:`~PyTypeObject.tp_subclasses`, and + :c:member:`~PyTypeObject.tp_weaklist`. + +The :c:data:`Py_tp_base` slot is equivalent to :c:data:`Py_tp_bases`; +both may be set either to a type or a tuple of types. +If both are specified, the value of :c:data:`Py_tp_bases` +is used. + +Slot values may not be ``NULL``, except for the following: + +* :c:data:`Py_tp_doc` +* :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + +.. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. + +.. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation + for details. + +.. versionchanged:: 3.15 + The :c:data:`Py_tp_bases` slot may be set to a single type object, + making it equivalent to the :c:data:`Py_tp_base` slot. + Previously, a tuple of types was required. + +The following slots correspond to fields in the underlying type structure, +but need extra remarks for use as slots: + +.. c:macro:: Py_tp_name + + :c:member:`Slot ID ` for the name of the type, + used to set :c:member:`PyTypeObject.tp_name`. + + This slot (or :c:member:`PyType_Spec.name`) is required to create a type. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.name` instead. + + .. impl-detail:: + + CPython processes slots in order. + It is recommended to put ``Py_tp_name`` at the beginning of the slots + array, so that if processing of a later slots fails, error messages + can include the name. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_basicsize + + :c:member:`Slot ID ` for the size of the instance in bytes. + It is used to set :c:member:`PyTypeObject.tp_basicsize`. + + The value must be positive. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:member:`PyTypeObject.tp_basicsize` instead if needed, but be aware + that a type's size is often considered an implementation detail. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_extra_basicsize + + :c:member:`Slot ID ` for type data size in bytes, that is, + how much space instances of the class need *in addition* + to space needed for superclasses. + + The value is used, together with the size of superclasses, to set + :c:member:`PyTypeObject.tp_basicsize`. + Python will insert padding as needed to meet + :c:member:`!tp_basicsize`'s alignment requirements. + + Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific + memory reserved this way. + + The value must be positive. + To specify that instances need no additional size (that is, size should be + inherited), omit the :c:macro:`!Py_tp_extra_basicsize` slot rather than + set it to zero. + + Specifying both :c:macro:`Py_tp_basicsize` and + :c:macro:`!Py_tp_extra_basicsize` is an error. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_itemsize + + :c:member:`Slot ID ` for the size of one element of a + variable-size type, in bytes. + Used to set :c:member:`PyTypeObject.tp_itemsize`. + See :c:member:`!tp_itemsize` documentation for caveats. + + The value must be positive. + + If this slot is missing, :c:member:`~PyTypeObject.tp_itemsize` is inherited. + Extending arbitrary variable-sized classes is dangerous, + since some types use a fixed offset for variable-sized memory, + which can then overlap fixed-sized memory used by a subclass. + To help prevent mistakes, inheriting ``itemsize`` is only possible + in the following situations: + + - The base is not variable-sized (its + :c:member:`~PyTypeObject.tp_itemsize` is zero). + - The requested :c:member:`PyType_Spec.basicsize` is positive, + suggesting that the memory layout of the base class is known. + - The requested :c:member:`PyType_Spec.basicsize` is zero, + suggesting that the subclass does not access the instance's memory + directly. + - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:member:`PyType_Spec.itemsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_flags + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_flags`. + + The ``Py_TPFLAGS_HEAPTYPE`` flag is not set, + :c:func:`PyType_FromSpecWithBases` sets it automatically. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use negative :c:member:`PyType_Spec.basicsize` instead. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetFlags` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_bases + + :c:member:`Slot ID ` for type flags, used to set + :c:member:`PyTypeObject.tp_bases`. + + The slot can be set to a tuple of type objects which the newly created + type should inherit from, like the "positional arguments" of + a Python :ref:`class definition `. + + Alternately, the slot can be set to a single type object to specify + a single base. + The effect is the same as specifying a one-element tuple. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_bases` required a tuple of types. + +.. c:macro:: Py_tp_base + + Equivalent to :c:macro:`Py_tp_bases` (with ``s`` at the end). + If both are specified, :c:macro:`!Py_tp_bases` takes priority and + this slot is ignored. + + .. versionchanged:: 3.15 + + Previously, :c:macro:`!Py_tp_base` required a single type, not a tuple. + + .. soft-deprecated:: 3.15 + + When not targeting older Python versions, prefer :c:macro:`!Py_tp_bases`. + +The following slots do not correspond to public fields in the +underlying structures: + +.. c:macro:: Py_tp_metaclass + + :c:member:`Slot ID ` for the metaclass used to construct + the resulting type object. + When omitted the metaclass is derived from bases + (:c:macro:`Py_tp_bases` or the *bases* argument of + :c:func:`PyType_FromMetaclass`). + + Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not + supported, except if ``tp_new`` is ``NULL``. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a metaclass with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`Py_TYPE` on the type object instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_module + + :c:member:`Slot ID ` for recording the module in which + the new class is defined. + + The value must be a module object. + The module is associated with the new type and can later be + retrieved with :c:func:`PyType_GetModule`. + The associated module is not inherited by subclasses; it must be specified + for each class individually. + + This may not be used in :c:member:`PyType_Spec.slots`. + Use :c:func:`PyType_FromMetaclass` to specify a module with + :c:type:`!PyType_Spec`. + + This slot may not be used with :c:func:`PyType_GetSlot`. + Use :c:func:`PyType_GetModule` instead. + + .. versionadded:: 3.15 + +.. c:macro:: Py_tp_token + + :c:member:`Slot ID ` for recording a static memory layout ID + for a class. + + If the class is defined using a :c:type:`PyType_Spec`, and that spec is + statically allocated, the token can be set to the spec using the special + value :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + May only be used for classes defined using :c:type:`!PyType_Spec`. + + Expands to ``NULL``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_tp_slots + + :c:member:`Slot ID ` that works like + :c:macro:`Py_slot_subslots`, except it specifies an array of + :c:type:`PyType_Slot` structures. + + .. versionadded:: 3.15 + + +Soft-deprecated API +------------------- + +The following functions are :term:`soft deprecated`. +They will continue to work, but new features will be added as slots for +:c:func:`PyType_FromSlots`, not as arguments to new ``PyType_From*`` functions. + +.. c:function:: PyObject* PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module, PyType_Spec *spec, PyObject *bases) + + Create and return a :ref:`heap type ` from the *spec* + (see :c:macro:`Py_TPFLAGS_HEAPTYPE`). + + A non-``NULL`` *metaclass* argument corresponds to the + :c:macro:`Py_tp_metaclass` slot. + + A non-``NULL`` *bases* argument corresponds to the :c:data:`Py_tp_bases` + slot, and takes precedence over :c:data:`Py_tp_bases` and + :c:data:`Py_tp_bases` slots. + + A non-``NULL`` *module* argument corresponds to the + :c:macro:`Py_tp_module` slot. + + This function calls :c:func:`PyType_Ready` on the new type. + + Note that this function does *not* fully match the behavior of + calling :py:class:`type() ` or using the :keyword:`class` statement. + See the note in :c:func:`PyType_FromSlots` documentation for details. + .. versionadded:: 3.12 + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) @@ -459,6 +859,10 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -481,6 +885,10 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next + + Prefer :c:func:`PyType_FromSlots` in new code. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -502,20 +910,9 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. soft-deprecated:: next -.. c:function:: int PyType_Freeze(PyTypeObject *type) - - Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. - - All base classes of *type* must be immutable. - - On success, return ``0``. - On error, set an exception and return ``-1``. - - The type must not be used before it's made immutable. For example, type - instances must not be created before the type is made immutable. - - .. versionadded:: 3.14 + Prefer :c:func:`PyType_FromSlots` in new code. .. raw:: html @@ -528,27 +925,23 @@ The following functions and structs are used to create .. c:type:: PyType_Spec - Structure defining a type's behavior. + Structure defining a type's behavior, used for soft-deprecated functions + like :c:func:`PyType_FromMetaclass`. + + This structure contains several members that can instead be specified + as :ref:`slots ` for :c:func:`PyType_FromSlots`, + and an array of slot entries with a simpler structure. .. c:member:: const char* name - Name of the type, used to set :c:member:`PyTypeObject.tp_name`. + Corresponds to :c:macro:`Py_tp_name`. .. c:member:: int basicsize - If positive, specifies the size of the instance in bytes. - It is used to set :c:member:`PyTypeObject.tp_basicsize`. + If positive, corresponds to :c:macro:`Py_tp_basicsize`. - If zero, specifies that :c:member:`~PyTypeObject.tp_basicsize` - should be inherited. - - If negative, the absolute value specifies how much space instances of the - class need *in addition* to the superclass. - Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific - memory reserved this way. - For negative :c:member:`!basicsize`, Python will insert padding when - needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment - requirements. + If negative, corresponds to :c:macro:`Py_tp_extra_basicsize` set to + the absolute value. .. versionchanged:: 3.12 @@ -556,160 +949,53 @@ The following functions and structs are used to create .. c:member:: int itemsize - Size of one element of a variable-size type, in bytes. - Used to set :c:member:`PyTypeObject.tp_itemsize`. - See ``tp_itemsize`` documentation for caveats. - - If zero, :c:member:`~PyTypeObject.tp_itemsize` is inherited. - Extending arbitrary variable-sized classes is dangerous, - since some types use a fixed offset for variable-sized memory, - which can then overlap fixed-sized memory used by a subclass. - To help prevent mistakes, inheriting ``itemsize`` is only possible - in the following situations: - - - The base is not variable-sized (its - :c:member:`~PyTypeObject.tp_itemsize`). - - The requested :c:member:`PyType_Spec.basicsize` is positive, - suggesting that the memory layout of the base class is known. - - The requested :c:member:`PyType_Spec.basicsize` is zero, - suggesting that the subclass does not access the instance's memory - directly. - - With the :c:macro:`Py_TPFLAGS_ITEMS_AT_END` flag. + Corresponds to :c:macro:`Py_tp_itemsize`. .. c:member:: unsigned int flags - Type flags, used to set :c:member:`PyTypeObject.tp_flags`. - - If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, - :c:func:`PyType_FromSpecWithBases` sets it automatically. + Corresponds to :c:macro:`Py_tp_flags`. .. c:member:: PyType_Slot *slots - Array of :c:type:`PyType_Slot` structures. - Terminated by the special slot value ``{0, NULL}``. + Array of :c:type:`PyType_Slot` (not :c:type:`PySlot`) structures. + Terminated by the special slot value ``{0, NULL}``. Each slot ID should be specified at most once. -.. raw:: html - - - - - -.. c:type:: PyType_Slot - - Structure defining optional functionality of a type, containing a slot ID - and a value pointer. - - .. c:member:: int slot - - A slot ID. - - Slot IDs are named like the field names of the structures - :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, - :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and - :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. - For example, use: - - * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` - * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` - * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` - - An additional slot is supported that does not correspond to a - :c:type:`!PyTypeObject` struct field: - - * :c:data:`Py_tp_token` - - The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) - - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:data:`Py_tp_members`. - See :ref:`PyMemberDef documentation ` - for details. - - The following internal fields cannot be set at all when creating a heap - type: - - * :c:member:`~PyTypeObject.tp_dict`, - :c:member:`~PyTypeObject.tp_mro`, - :c:member:`~PyTypeObject.tp_cache`, - :c:member:`~PyTypeObject.tp_subclasses`, and - :c:member:`~PyTypeObject.tp_weaklist`. + .. c:namespace:: NULL - Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be - problematic on some platforms. - To avoid issues, use the *bases* argument of - :c:func:`PyType_FromSpecWithBases` instead. + .. raw:: html - .. versionchanged:: 3.9 - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + + + - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + .. c:type:: PyType_Slot - .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set - using :c:data:`Py_tp_vectorcall`. See the field's documentation - for details. + Structure defining optional functionality of a type, used for + soft-deprecated functions like :c:func:`PyType_FromMetaclass`. - .. c:member:: void *pfunc + Note that a :c:type:`!PyType_Slot` array may be included in a + :c:type:`!PySlot` array using :c:macro:`Py_tp_slots`, + and vice versa using :c:macro:`Py_slot_subslots`. - The desired value of the slot. In most cases, this is a pointer - to a function. + Each :c:type:`!PyType_Slot` structure ``tpslot`` is interpreted + as the following :c:type:`PySlot` structure:: - *pfunc* values may not be ``NULL``, except for the following slots: + (PySlot){ + .sl_id=tpslot.slot, + .sl_flags=PySlot_INTPTR | sub_static, + .sl_ptr=tpslot.func + } - * :c:data:`Py_tp_doc` - * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` - rather than ``NULL``) + where ``sub_static`` is ``PySlot_STATIC`` if the slot requires + the flag (such as for :c:macro:`Py_tp_methods`), or if this flag + is present on the "parent" :c:macro:`!Py_tp_slots` slot (if any). + .. c:member:: int slot -.. c:macro:: Py_tp_token - - A :c:member:`~PyType_Slot.slot` that records a static memory layout ID - for a class. - - If the :c:type:`PyType_Spec` of the class is statically - allocated, the token can be set to the spec using the special value - :c:data:`Py_TP_USE_SPEC`: - - .. code-block:: c - - static PyType_Slot foo_slots[] = { - {Py_tp_token, Py_TP_USE_SPEC}, - - It can also be set to an arbitrary pointer, but you must ensure that: - - * The pointer outlives the class, so it's not reused for something else - while the class exists. - * It "belongs" to the extension module where the class lives, so it will not - clash with other extensions. + Corresponds to :c:member:`PySlot.sl_id`. - Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has - a given token -- that is, check whether the memory layout is compatible. - - To get the token for a given class (without considering superclasses), - use :c:func:`PyType_GetSlot` with ``Py_tp_token``. - - .. versionadded:: 3.14 - - .. c:namespace:: NULL - - .. c:macro:: Py_TP_USE_SPEC - - Used as a value with :c:data:`Py_tp_token` to set the token to the - class's :c:type:`PyType_Spec`. - Expands to ``NULL``. + .. c:member:: void *pfunc - .. versionadded:: 3.14 + Corresponds to :c:member:`PySlot.sl_ptr`. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737deb81c..ec2fba6da8b0438 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index c3960d6ff87ec82..4000f8aa1d4c0c7 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -555,6 +555,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: const char* PyTypeObject.tp_name + See :c:macro:`Py_tp_name` for the corresponding + :c:member:`Slot ID `. + Pointer to a NUL-terminated string containing the name of the type. For types that are accessible as module globals, the string should be the full module name, followed by a dot, followed by the type name; for built-in types, it @@ -594,6 +597,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) These fields allow calculating the size in bytes of instances of the type. + See :c:macro:`Py_tp_basicsize`, :c:macro:`Py_tp_extra_basicsize` and + :c:macro:`Py_tp_itemsize` for the corresponding + :c:member:`Slot IDs `. + There are two kinds of types: types with fixed-length instances have a zero :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero :c:member:`!tp_itemsize` field. For a type with fixed-length instances, all @@ -1133,6 +1140,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: unsigned long PyTypeObject.tp_flags + See :c:macro:`Py_tp_flags` for the corresponding + :c:member:`Slot ID `. + This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via @@ -1391,8 +1401,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionchanged:: 3.9 - Renamed to the current name, without the leading underscore. - The old provisional name is :term:`soft deprecated`. + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1501,11 +1511,13 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG - This is a :term:`soft deprecated` macro that does nothing. + This macro does nothing. Historically, this would indicate that the :c:member:`~PyTypeObject.tp_version_tag` field was available and initialized. + .. soft-deprecated:: 3.13 + .. c:macro:: Py_TPFLAGS_INLINE_VALUES @@ -1924,12 +1936,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base - .. corresponding-type-slot:: Py_tp_base - An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. + For the corresponding slot ID, see :c:macro:`Py_tp_base`. + .. note:: .. from Modules/xxmodule.c @@ -2241,17 +2253,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases - .. corresponding-type-slot:: Py_tp_bases - Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the :c:data:`Py_tp_bases` - :c:type:`slot ` can be used instead of the *bases* argument - of :c:func:`PyType_FromSpecWithBases`. - The argument form is preferred. + For the corresponding slot ID, see :c:macro:`Py_tp_bases`. .. warning:: @@ -2963,13 +2970,13 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. @@ -2995,10 +3002,10 @@ Buffer Object Structures schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -3044,7 +3051,7 @@ Buffer Object Structures * Resource cleanup when the counter reaches zero must be done atomically, as the final release may race with concurrent releases from other - threads and dellocation must only happen once. + threads and deallocation must only happen once. The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain @@ -3052,7 +3059,7 @@ Buffer Object Structures *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 679823948e73b52..9bf801ad608c773 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -676,7 +676,8 @@ APIs: Append the string *right* to the end of *p_left*. *p_left* must point to a :term:`strong reference` to a Unicode object; - :c:func:`!PyUnicode_Append` releases ("steals") this reference. + :c:func:`!PyUnicode_Append` releases (":term:`steals `") + this reference. On error, set *\*p_left* to ``NULL`` and set an exception. @@ -762,7 +763,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -1174,7 +1175,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1196,7 +1197,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs @@ -1881,6 +1882,10 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + To write a :class:`str` subclass which overrides the :meth:`~object.__str__` + method, :c:func:`PyUnicode_FromObject` can be used to get the original + string. + .. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. @@ -1949,7 +1954,7 @@ The following API is deprecated. whether you selected a "narrow" or "wide" Unicode version of Python at build time. - .. deprecated-removed:: 3.13 3.15 + .. deprecated-removed:: 3.13 3.16 .. c:function:: int PyUnicode_READY(PyObject *unicode) diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 6256bf7a1454a9a..bba5c7f8ecf7751 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -343,11 +343,125 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. + .. rubric:: Low-level flags + + The following flags and masks serve narrow needs of the standard + library and interactive interpreters. Code outside the standard + library rarely has a reason to use them. They are considered + implementation details and may change at any time. + + .. c:macro:: PyCF_ALLOW_INCOMPLETE_INPUT + + This flag is a private interface between the compiler and the + :mod:`codeop` module. Do not use it; its behavior is unsupported + and may change without warning. + + With this flag set, when compilation fails because the source text + ends where more input is expected, for example in the middle of an + indented block or an unterminated string literal, the error raised + is the undocumented ``_IncompleteInputError``, a subclass of + :exc:`SyntaxError`. The :mod:`codeop` module sets this flag, + together with :c:macro:`PyCF_DONT_IMPLY_DEDENT`, to tell input + that is incomplete apart from input with a real syntax error, so + that interactive interpreters know when to prompt for another + line instead of reporting an error. + + .. versionadded:: 3.11 + + .. c:macro:: PyCF_DONT_IMPLY_DEDENT + + By default, when compiling with the :c:var:`Py_single_input` start + symbol, reaching the end of the source text implicitly closes any + open indented blocks. With this flag set, open blocks are only + closed if the last line of the source ends with a newline; otherwise, + compilation fails with a :exc:`SyntaxError`: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "if a:\n pass"; + + /* The "if" block is closed implicitly; + this returns a code object: */ + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + /* With the flag, this fails with a SyntaxError, + because the last line does not end with a newline: */ + flags.cf_flags = PyCF_DONT_IMPLY_DEDENT; + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + The :mod:`codeop` module uses this flag to detect incomplete + interactive input. While the user is still typing inside an + indented block, the source does not yet end with a newline, so it + fails to compile and the user is prompted for another line. + + .. c:macro:: PyCF_IGNORE_COOKIE + + Read the source text as UTF-8, ignoring its :pep:`263` encoding + declaration ("coding cookie"), if any: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "# coding: latin-1\ns = '\xe9'\n"; + + /* The coding cookie is honored: byte 0xE9 is decoded as + Latin-1, and this returns a code object that sets s to "é": */ + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + /* With the flag, the cookie is ignored and compilation fails + with a SyntaxError, because 0xE9 is not valid UTF-8: */ + flags.cf_flags = PyCF_IGNORE_COOKIE; + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag when the source is a :class:`str` object, because they + pass the text to the parser encoded as UTF-8. + + .. c:macro:: PyCF_SOURCE_IS_UTF8 + + Mark the source text as known to be UTF-8 encoded. + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag, but it currently has no effect. + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally selectable using :ref:`future statements `. See :ref:`c_codeobject_flags` for a complete list. + The following masks combine several flags: + + .. c:macro:: PyCF_MASK + + Bitmask of all ``CO_FUTURE`` flags (see :ref:`c_codeobject_flags`), + which select features normally enabled by + :ref:`future statements `. + When code compiled with a ``PyCompilerFlags *flags`` argument + contains a ``from __future__ import`` statement, the flag for the + imported feature is added to *flags*, so that code executed later + in the same context inherits it. + + .. c:macro:: PyCF_MASK_OBSOLETE + + Do not use this mask in new code. It is kept only so that old + code passing its flags to :func:`compile` keeps working. + + Bitmask of flags for obsolete future features that no longer + have any effect. + + .. c:macro:: PyCF_COMPILE_MASK + + Bitmask of all ``PyCF`` flags that change how the source is + compiled, such as :c:macro:`PyCF_ONLY_AST`. + The :func:`compile` built-in function uses this mask to validate + its *flags* argument. + .. _start-symbols: diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index db6ae0a9d4ea3d7..8762a003c5218d3 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -43,7 +43,11 @@ as much as it can. should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly @@ -59,7 +63,11 @@ as much as it can. collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a weakly referenceable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + ``NULL``, this will raise :exc:`TypeError` and return ``NULL``. + + .. versionchanged:: next + Raise :exc:`!TypeError` if *callback* is not callable, ``None``, or + ``NULL``. .. seealso:: :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly diff --git a/Doc/conf.py b/Doc/conf.py index e2dff74538a3422..c768e6fd676a5a3 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -43,7 +41,7 @@ # Skip if downstream redistributors haven't installed them _OPTIONAL_EXTENSIONS = ( - 'linklint.ext', + 'sphinx_linklint.ext', 'notfound.extension', 'sphinxext.opengraph', 'sphinxcontrib.rsvgconverter', @@ -78,7 +76,7 @@ # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` @@ -349,8 +347,6 @@ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} } -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} ''', # The paper size ('letterpaper' or 'a4paper'). @@ -557,16 +553,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 2a6e6b963134bb7..60c02aabeb89c51 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2037,6 +2037,11 @@ PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:0: +PySentinel_New:PyObject*::+1: +PySentinel_New:const char*:name:: +PySentinel_New:const char*:module_name:: +PySentinel_New:const char*:repr:: + PySequence_Check:int::: PySequence_Check:PyObject*:o:0: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 4a7fbdf60bfb431..86080fac7163838 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,7 +1,7 @@ role,name,added,ifdef_note,struct_abi_kind macro,METH_CLASS,3.2,, macro,METH_COEXIST,3.2,, -macro,METH_FASTCALL,3.7,, +macro,METH_FASTCALL,3.10,, macro,METH_METHOD,3.7,, macro,METH_NOARGS,3.2,, macro,METH_O,3.2,, @@ -129,6 +129,12 @@ func,PyComplex_FromDoubles,3.2,, func,PyComplex_ImagAsDouble,3.2,, func,PyComplex_RealAsDouble,3.2,, data,PyComplex_Type,3.2,, +type,PyCriticalSection,3.15,,full-abi +type,PyCriticalSection2,3.15,,full-abi +func,PyCriticalSection2_Begin,3.15,, +func,PyCriticalSection2_End,3.15,, +func,PyCriticalSection_Begin,3.15,, +func,PyCriticalSection_End,3.15,, func,PyDescr_NewClassMethod,3.2,, func,PyDescr_NewGetSet,3.2,, func,PyDescr_NewMember,3.2,, @@ -363,6 +369,10 @@ func,PyImport_ImportModuleLevel,3.2,, func,PyImport_ImportModuleLevelObject,3.7,, func,PyImport_ReloadModule,3.2,, func,PyIndex_Check,3.8,, +type,PyInterpreterGuard,3.15,,opaque +func,PyInterpreterGuard_Close,3.15,, +func,PyInterpreterGuard_FromCurrent,3.15,, +func,PyInterpreterGuard_FromView,3.15,, type,PyInterpreterState,3.2,,opaque func,PyInterpreterState_Clear,3.2,, func,PyInterpreterState_Delete,3.2,, @@ -370,6 +380,10 @@ func,PyInterpreterState_Get,3.9,, func,PyInterpreterState_GetDict,3.8,, func,PyInterpreterState_GetID,3.7,, func,PyInterpreterState_New,3.2,, +type,PyInterpreterView,3.15,,opaque +func,PyInterpreterView_Close,3.15,, +func,PyInterpreterView_FromCurrent,3.15,, +func,PyInterpreterView_FromMain,3.15,, func,PyIter_Check,3.8,, func,PyIter_Next,3.2,, func,PyIter_NextItem,3.14,, @@ -470,8 +484,8 @@ func,PyMemoryView_GetContiguous,3.2,, data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi data,PyMethodDescr_Type,3.2,, -type,PyModuleDef,3.2,,full-abi -type,PyModuleDef_Base,3.2,,full-abi +type,PyModuleDef,3.2,,abi3t-opaque +type,PyModuleDef_Base,3.2,,abi3t-opaque func,PyModuleDef_Init,3.5,, type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, @@ -565,6 +579,7 @@ func,PyObject_ASCII,3.2,, func,PyObject_AsFileDescriptor,3.2,, func,PyObject_Bytes,3.2,, func,PyObject_Call,3.2,, +func,PyObject_CallFinalizerFromDealloc,3.15,, func,PyObject_CallFunction,3.2,, func,PyObject_CallFunctionObjArgs,3.2,, func,PyObject_CallMethod,3.2,, @@ -669,6 +684,19 @@ func,PySlice_GetIndicesEx,3.2,, func,PySlice_New,3.2,, data,PySlice_Type,3.2,, func,PySlice_Unpack,3.7,, +type,PySlot,3.15,,full-abi +macro,PySlot_DATA,3.15,, +macro,PySlot_END,3.15,, +macro,PySlot_FUNC,3.15,, +macro,PySlot_INT64,3.15,, +macro,PySlot_INTPTR,3.15,, +macro,PySlot_OPTIONAL,3.15,, +macro,PySlot_PTR,3.15,, +macro,PySlot_PTR_STATIC,3.15,, +macro,PySlot_SIZE,3.15,, +macro,PySlot_STATIC,3.15,, +macro,PySlot_STATIC_DATA,3.15,, +macro,PySlot_UINT64,3.15,, func,PyState_AddModule,3.3,, func,PyState_FindModule,3.2,, func,PyState_RemoveModule,3.3,, @@ -696,14 +724,18 @@ func,PySys_SetObject,3.2,, func,PySys_WriteStderr,3.2,, func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque +type,PyThreadStateToken,3.15,,opaque func,PyThreadState_Clear,3.2,, func,PyThreadState_Delete,3.2,, +func,PyThreadState_Ensure,3.15,, +func,PyThreadState_EnsureFromView,3.15,, func,PyThreadState_Get,3.2,, func,PyThreadState_GetDict,3.2,, func,PyThreadState_GetFrame,3.10,, func,PyThreadState_GetID,3.10,, func,PyThreadState_GetInterpreter,3.10,, func,PyThreadState_New,3.2,, +func,PyThreadState_Release,3.15,, func,PyThreadState_SetAsyncExc,3.2,, func,PyThreadState_Swap,3.2,, func,PyThread_GetInfo,3.3,, @@ -748,6 +780,7 @@ func,PyType_ClearCache,3.2,, func,PyType_Freeze,3.14,, func,PyType_FromMetaclass,3.12,, func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSlots,3.15,, func,PyType_FromSpec,3.2,, func,PyType_FromSpecWithBases,3.3,, func,PyType_GenericAlloc,3.2,, @@ -905,6 +938,8 @@ macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, +macro,Py_BEGIN_CRITICAL_SECTION,3.15,, +macro,Py_BEGIN_CRITICAL_SECTION2,3.15,, macro,Py_BLOCK_THREADS,3.2,, func,Py_BuildValue,3.2,, func,Py_BytesMain,3.8,, @@ -912,6 +947,8 @@ func,Py_CompileString,3.2,, func,Py_DecRef,3.2,, func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, +macro,Py_END_CRITICAL_SECTION,3.15,, +macro,Py_END_CRITICAL_SECTION2,3.15,, func,Py_EncodeLocale,3.7,, func,Py_EndInterpreter,3.2,, func,Py_EnterRecursiveCall,3.9,, @@ -1007,6 +1044,7 @@ macro,Py_mod_gil,3.13,, macro,Py_mod_methods,3.15,, macro,Py_mod_multiple_interpreters,3.12,, macro,Py_mod_name,3.15,, +macro,Py_mod_slots,3.15,, macro,Py_mod_state_clear,3.15,, macro,Py_mod_state_free,3.15,, macro,Py_mod_state_size,3.15,, @@ -1050,6 +1088,9 @@ macro,Py_nb_rshift,3.2,, macro,Py_nb_subtract,3.2,, macro,Py_nb_true_divide,3.2,, macro,Py_nb_xor,3.2,, +macro,Py_slot_end,3.15,, +macro,Py_slot_invalid,3.15,, +macro,Py_slot_subslots,3.15,, macro,Py_sq_ass_item,3.2,, macro,Py_sq_concat,3.2,, macro,Py_sq_contains,3.2,, @@ -1062,6 +1103,7 @@ type,Py_ssize_t,3.2,, macro,Py_tp_alloc,3.2,, macro,Py_tp_base,3.2,, macro,Py_tp_bases,3.2,, +macro,Py_tp_basicsize,3.15,, macro,Py_tp_call,3.2,, macro,Py_tp_clear,3.2,, macro,Py_tp_dealloc,3.2,, @@ -1069,7 +1111,9 @@ macro,Py_tp_del,3.2,, macro,Py_tp_descr_get,3.2,, macro,Py_tp_descr_set,3.2,, macro,Py_tp_doc,3.2,, +macro,Py_tp_extra_basicsize,3.15,, macro,Py_tp_finalize,3.5,, +macro,Py_tp_flags,3.15,, macro,Py_tp_free,3.2,, macro,Py_tp_getattr,3.2,, macro,Py_tp_getattro,3.2,, @@ -1077,15 +1121,20 @@ macro,Py_tp_getset,3.2,, macro,Py_tp_hash,3.2,, macro,Py_tp_init,3.2,, macro,Py_tp_is_gc,3.2,, +macro,Py_tp_itemsize,3.15,, macro,Py_tp_iter,3.2,, macro,Py_tp_iternext,3.2,, macro,Py_tp_members,3.2,, +macro,Py_tp_metaclass,3.15,, macro,Py_tp_methods,3.2,, +macro,Py_tp_module,3.15,, +macro,Py_tp_name,3.15,, macro,Py_tp_new,3.2,, macro,Py_tp_repr,3.2,, macro,Py_tp_richcompare,3.2,, macro,Py_tp_setattr,3.2,, macro,Py_tp_setattro,3.2,, +macro,Py_tp_slots,3.15,, macro,Py_tp_str,3.2,, macro,Py_tp_token,3.14,, macro,Py_tp_traverse,3.2,, diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat index 7c381ecb70c8363..ea5a24a5505e208 100644 --- a/Doc/data/threadsafety.dat +++ b/Doc/data/threadsafety.dat @@ -14,10 +14,71 @@ # The function name must match the C domain identifier used in the documentation. # Synchronization primitives (Doc/c-api/synchronization.rst) -PyMutex_Lock:shared: -PyMutex_Unlock:shared: +PyMutex_Lock:atomic: +PyMutex_Unlock:atomic: PyMutex_IsLocked:atomic: + +# Dictionary objects (Doc/c-api/dict.rst) + +# Type checks - read ob_type pointer, always safe +PyDict_Check:atomic: +PyDict_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyDict_New:atomic: + +# Lock-free lookups - use _Py_dict_lookup_threadsafe(), no locking. +# Atomic with simple types. +PyDict_Contains:shared: +PyDict_ContainsString:atomic: +PyDict_GetItemRef:shared: +PyDict_GetItemStringRef:atomic: +PyDict_Size:atomic: +PyDict_GET_SIZE:atomic: + +# Borrowed-reference lookups - lock-free dict access but returned +# borrowed reference is unsafe in free-threaded builds without +# external synchronization +PyDict_GetItem:compatible: +PyDict_GetItemWithError:compatible: +PyDict_GetItemString:compatible: +PyDict_SetDefault:compatible: + +# Iteration - no locking; returns borrowed refs +PyDict_Next:compatible: + +# Single-item mutations - protected by per-object critical section +PyDict_SetItem:shared: +PyDict_SetItemString:atomic: +PyDict_DelItem:shared: +PyDict_DelItemString:atomic: +PyDict_SetDefaultRef:shared: +PyDict_Pop:shared: +PyDict_PopString:atomic: + +# Bulk reads - hold per-object lock for duration +PyDict_Clear:atomic: +PyDict_Copy:atomic: +PyDict_Keys:atomic: +PyDict_Values:atomic: +PyDict_Items:atomic: + +# Merge/update - lock target dict; also lock source when it is a dict +PyDict_Update:shared: +PyDict_Merge:shared: +PyDict_MergeFromSeq2:shared: + +# Watcher registration - no synchronization on interpreter state +PyDict_AddWatcher:compatible: +PyDict_ClearWatcher:compatible: + +# Per-dict watcher tags - non-atomic RMW on _ma_watcher_tag; +# safe on distinct dicts only +PyDict_Watch:distinct: +PyDict_Unwatch:distinct: + + # List objects (Doc/c-api/list.rst) # Type checks - read ob_type pointer, always safe @@ -125,6 +186,29 @@ PyByteArray_GET_SIZE:atomic: PyByteArray_AsString:compatible: PyByteArray_AS_STRING:compatible: +# Creation - may iterate the iterable argument, calling arbitrary code. +# Atomic for sets, frozensets, dicts, and frozendicts. +PySet_New:shared: +PyFrozenSet_New:shared: + +# Size - uses atomic load on free-threaded builds +PySet_Size:atomic: +PySet_GET_SIZE:atomic: + +# Contains - lock-free, atomic with simple types +PySet_Contains:shared: + +# Mutations - hold per-object lock for duration +# atomic with simple types +PySet_Add:shared: +PySet_Discard:shared: + +# Pop - hold per-object lock for duration +PySet_Pop:atomic: + +# Clear - empties the set before clearing +PySet_Clear:atomic: + # Capsule objects (Doc/c-api/capsule.rst) # Type check - read ob_type pointer, always safe diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index 789ec83d2d957aa..6d0d47403ff2ed4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -46,88 +46,3 @@ Pending removal in Python 3.15 The `pythoncapi-compat project `__ can be used to get :c:func:`PyConfig_Get` on Python 3.13 and older. - -* Functions to configure Python's initialization, deprecated in Python 3.11: - - * :c:func:`!PySys_SetArgvEx()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!PySys_SetArgv()`: - Set :c:member:`PyConfig.argv` instead. - * :c:func:`!Py_SetProgramName()`: - Set :c:member:`PyConfig.program_name` instead. - * :c:func:`!Py_SetPythonHome()`: - Set :c:member:`PyConfig.home` instead. - * :c:func:`!PySys_ResetWarnOptions`: - Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` instead. - -* Global configuration variables: - - * :c:var:`Py_DebugFlag`: - Use :c:member:`PyConfig.parser_debug` or - :c:func:`PyConfig_Get("parser_debug") ` instead. - * :c:var:`Py_VerboseFlag`: - Use :c:member:`PyConfig.verbose` or - :c:func:`PyConfig_Get("verbose") ` instead. - * :c:var:`Py_QuietFlag`: - Use :c:member:`PyConfig.quiet` or - :c:func:`PyConfig_Get("quiet") ` instead. - * :c:var:`Py_InteractiveFlag`: - Use :c:member:`PyConfig.interactive` or - :c:func:`PyConfig_Get("interactive") ` instead. - * :c:var:`Py_InspectFlag`: - Use :c:member:`PyConfig.inspect` or - :c:func:`PyConfig_Get("inspect") ` instead. - * :c:var:`Py_OptimizeFlag`: - Use :c:member:`PyConfig.optimization_level` or - :c:func:`PyConfig_Get("optimization_level") ` instead. - * :c:var:`Py_NoSiteFlag`: - Use :c:member:`PyConfig.site_import` or - :c:func:`PyConfig_Get("site_import") ` instead. - * :c:var:`Py_BytesWarningFlag`: - Use :c:member:`PyConfig.bytes_warning` or - :c:func:`PyConfig_Get("bytes_warning") ` instead. - * :c:var:`Py_FrozenFlag`: - Use :c:member:`PyConfig.pathconfig_warnings` or - :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. - * :c:var:`Py_IgnoreEnvironmentFlag`: - Use :c:member:`PyConfig.use_environment` or - :c:func:`PyConfig_Get("use_environment") ` instead. - * :c:var:`Py_DontWriteBytecodeFlag`: - Use :c:member:`PyConfig.write_bytecode` or - :c:func:`PyConfig_Get("write_bytecode") ` instead. - * :c:var:`Py_NoUserSiteDirectory`: - Use :c:member:`PyConfig.user_site_directory` or - :c:func:`PyConfig_Get("user_site_directory") ` instead. - * :c:var:`Py_UnbufferedStdioFlag`: - Use :c:member:`PyConfig.buffered_stdio` or - :c:func:`PyConfig_Get("buffered_stdio") ` instead. - * :c:var:`Py_HashRandomizationFlag`: - Use :c:member:`PyConfig.use_hash_seed` - and :c:member:`PyConfig.hash_seed` or - :c:func:`PyConfig_Get("hash_seed") ` instead. - * :c:var:`Py_IsolatedFlag`: - Use :c:member:`PyConfig.isolated` or - :c:func:`PyConfig_Get("isolated") ` instead. - * :c:var:`Py_LegacyWindowsFSEncodingFlag`: - Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or - :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. - * :c:var:`Py_LegacyWindowsStdioFlag`: - Use :c:member:`PyConfig.legacy_windows_stdio` or - :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. - * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: - Use :c:member:`PyConfig.filesystem_encoding` or - :c:func:`PyConfig_Get("filesystem_encoding") ` instead. - * :c:var:`!Py_FileSystemDefaultEncodeErrors`: - Use :c:member:`PyConfig.filesystem_errors` or - :c:func:`PyConfig_Get("filesystem_errors") ` instead. - * :c:var:`!Py_UTF8Mode`: - Use :c:member:`PyPreConfig.utf8_mode` or - :c:func:`PyConfig_Get("utf8_mode") ` instead. - (see :c:func:`Py_PreInitialize`) - - The :c:func:`Py_InitializeFromConfig` API should be used with - :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be - used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst index 9453f83799c43d7..fe2d91cf316b18f 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.16.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -1,4 +1,87 @@ Pending removal in Python 3.16 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdec``. +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") ` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") ` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") ` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") ` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") ` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") ` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") ` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") ` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") ` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") ` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") ` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") ` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") ` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") ` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") ` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") ` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") ` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") ` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") ` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") ` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. + +* :c:type:`Py_UNICODE` which was deprecated by :pep:`393`. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 022aee93aa70c47..820334ee43c82e4 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -40,6 +40,10 @@ Pending removal in Python 3.18 * :c:func:`!_PyUnicodeWriter_PrepareKind`: (no replacement). * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + * :c:func:`PyGen_New`: (no replacement). + * :c:func:`PyGen_NewWithQualName`: (no replacement). + * :c:func:`PyCoro_New`: (no replacement). + * :c:func:`PyAsyncGen_New`: (no replacement). The `pythoncapi-compat project `__ can be used to get diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index bb8bfb5c227c2d0..6176169a6f1eba6 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -1,10 +1,6 @@ Deprecations ============ -.. include:: pending-removal-in-3.15.rst - -.. include:: pending-removal-in-3.16.rst - .. include:: pending-removal-in-3.17.rst .. include:: pending-removal-in-3.18.rst @@ -13,15 +9,15 @@ Deprecations .. include:: pending-removal-in-3.20.rst +.. include:: pending-removal-in-3.21.rst + .. include:: pending-removal-in-future.rst +.. include:: soft-deprecations.rst + C API deprecations ------------------ -.. include:: c-api-pending-removal-in-3.15.rst - -.. include:: c-api-pending-removal-in-3.16.rst - .. include:: c-api-pending-removal-in-3.18.rst .. include:: c-api-pending-removal-in-3.19.rst diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index e7f27f73664df39..5c0e592f4caeda1 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -8,11 +8,6 @@ Pending removal in Python 3.15 is deprecated. In Python 3.15, ``__cached__`` will cease to be set or take into consideration by the import system or standard library. (:gh:`97879`) - * Setting :attr:`~module.__package__` on a module while - failing to set :attr:`__spec__.parent ` - is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or - take into consideration by the import system or standard library. (:gh:`97879`) - * :mod:`ctypes`: * The undocumented :func:`!ctypes.SetPointerType` function @@ -60,7 +55,7 @@ Pending removal in Python 3.15 * :mod:`types`: - * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + * :class:`types.CodeType`: Accessing :attr:`!codeobject.co_lnotab` was deprecated in :pep:`626` since 3.10 and was planned to be removed in 3.12, but it only got a proper :exc:`DeprecationWarning` in 3.12. @@ -82,7 +77,7 @@ Pending removal in Python 3.15 Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` to create a TypedDict with zero field. - * The :func:`!typing.no_type_check_decorator` decorator function + * The :deco:`!typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst index b00c7002b03772d..9b19ec3236f92fb 100644 --- a/Doc/deprecations/pending-removal-in-3.16.rst +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -8,6 +8,13 @@ Pending removal in Python 3.16 is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or taken into consideration by the import system or the standard library. + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent ` + is deprecated. In Python 3.16, :attr:`!__package__` will cease to be + taken into consideration by the import system or standard library. (:gh:`97879`) + +* The bundled copy of ``libmpdec``. + * :mod:`array`: * The ``'u'`` format code (:c:type:`wchar_t`) @@ -26,12 +33,12 @@ Pending removal in Python 3.16 * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. In particular, the following classes and functions are deprecated: - * :class:`asyncio.AbstractEventLoopPolicy` - * :class:`asyncio.DefaultEventLoopPolicy` - * :class:`asyncio.WindowsSelectorEventLoopPolicy` - * :class:`asyncio.WindowsProactorEventLoopPolicy` - * :func:`asyncio.get_event_loop_policy` - * :func:`asyncio.set_event_loop_policy` + * :class:`!asyncio.AbstractEventLoopPolicy` + * :class:`!asyncio.DefaultEventLoopPolicy` + * :class:`!asyncio.WindowsSelectorEventLoopPolicy` + * :class:`!asyncio.WindowsProactorEventLoopPolicy` + * :func:`!asyncio.get_event_loop_policy` + * :func:`!asyncio.set_event_loop_policy` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with *loop_factory* to use the desired event loop implementation. @@ -84,12 +91,12 @@ Pending removal in Python 3.16 * :mod:`symtable`: - * The :meth:`Class.get_methods ` method + * The :meth:`!symtable.Class.get_methods` method has been deprecated since Python 3.14. * :mod:`sys`: - * The :func:`~sys._enablelegacywindowsfsencoding` function + * The :func:`!_enablelegacywindowsfsencoding` function has been deprecated since Python 3.13. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. @@ -101,5 +108,5 @@ Pending removal in Python 3.16 * :mod:`tarfile`: - * The undocumented and unused :attr:`!TarFile.tarfile` attribute + * The undocumented and unused :attr:`!TarInfo.tarfile` attribute has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index e769c9d371e133c..5ed674951d90cb8 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -1,6 +1,14 @@ Pending removal in Python 3.17 ------------------------------ +* :mod:`datetime`: + + * :meth:`~datetime.datetime.strptime` calls using a format string containing + ``%e`` (day of month) without a year. + This has been deprecated since Python 3.15. + (Contributed by Stan Ulbrych in :gh:`70647`.) + + * :mod:`collections.abc`: - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. @@ -27,7 +35,19 @@ Pending removal in Python 3.17 - Passing non-ascii *encoding* names to :func:`encodings.normalize_encoding` is deprecated and scheduled for removal in Python 3.17. - (Contributed by Stan Ulbrych in :gh:`136702`) + (Contributed by Stan Ulbrych in :gh:`136702`.) + +* :mod:`profile`: + + - The :mod:`!profile` module is deprecated and will be removed in + Python 3.17. Use :mod:`profiling.tracing` instead, which provides a + compatible API. See :pep:`799` for details. + (Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + +* :mod:`webbrowser`: + + - :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS`. (:gh:`137586`) * :mod:`typing`: @@ -55,3 +75,13 @@ Pending removal in Python 3.17 See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`tkinter`: + + - The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, + :meth:`!trace` (an alias of :meth:`!trace_variable`), + :meth:`!trace_vdelete` and :meth:`!trace_vinfo`, deprecated since + Python 3.14, are scheduled for removal in Python 3.17. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` + instead. + (Contributed by Serhiy Storchaka in :gh:`120220`.) diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst index eb42fe9919eaebd..19113aab981bbc6 100644 --- a/Doc/deprecations/pending-removal-in-3.18.rst +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -10,3 +10,9 @@ Pending removal in Python 3.18 specifier ``'N'``, which is only supported in the :mod:`!decimal` module's C implementation, has been deprecated since Python 3.13. (Contributed by Serhiy Storchaka in :gh:`89902`.) + +* Deprecations defined by :pep:`829`: + + * ``import`` lines in :file:`{name}.pth` files are silently ignored. + + (Contributed by Barry Warsaw in :gh:`148641`.) diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst index 25f9cba390de68e..d7e0cd2d33ddb86 100644 --- a/Doc/deprecations/pending-removal-in-3.19.rst +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -22,3 +22,29 @@ Pending removal in Python 3.19 supported depending on the backend implementation of hash functions. Prefer passing the initial data as a positional argument for maximum backwards compatibility. + +* :mod:`http.cookies`: + + * :meth:`http.cookies.Morsel.js_output` is deprecated and will be + removed in Python 3.19. + + * :meth:`http.cookies.BaseCookie.js_output` is deprecated and will be + removed in Python 3.19. + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does not automatically close the current file. + + Before Python 3.14, this property was used to implement the corresponding + ``read()`` and ``readline()`` methods for :class:`~imaplib.IMAP4` but this + is no longer the case since then. + +* :mod:`tkinter`: + + * :func:`tkinter.filedialog.askopenfiles` has been deprecated since Python + 3.16. Iterate over the names returned by + :func:`~tkinter.filedialog.askopenfilenames` and open them one by one + instead. + (Contributed by Serhiy Storchaka in :gh:`152638`.) diff --git a/Doc/deprecations/pending-removal-in-3.20.rst b/Doc/deprecations/pending-removal-in-3.20.rst index 176e8f3f9f601cf..d771764502c9103 100644 --- a/Doc/deprecations/pending-removal-in-3.20.rst +++ b/Doc/deprecations/pending-removal-in-3.20.rst @@ -38,3 +38,27 @@ Pending removal in Python 3.20 - :mod:`zlib` (Contributed by Hugo van Kemenade and Stan Ulbrych in :gh:`76007`.) + +* Deprecations defined by :pep:`829`: + + * Warnings are produced for ``import`` lines found in :file:`{name}.pth` + files. + + * :file:`{name}.pth` files are no longer decoded in the locale encoding by + default. They **MUST** be encoded in ``utf-8-sig``. + + (Contributed by Barry Warsaw in :gh:`148641`.) + +* :mod:`ast`: + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + +* :mod:`typing`: + + * It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. + This will raise a :exc:`TypeError` in Python 3.20. + + (Contributed by Bartosz Sławecki in :gh:`132604`.) diff --git a/Doc/deprecations/pending-removal-in-3.21.rst b/Doc/deprecations/pending-removal-in-3.21.rst new file mode 100644 index 000000000000000..c2546b7fc451aed --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.21.rst @@ -0,0 +1,30 @@ +Pending removal in Python 3.21 +------------------------------ + +* :mod:`abc` + + * Soft-deprecated since Python 3.3 :class:`abc.abstractclassmethod`, + :class:`abc.abstractstaticmethod`, and :class:`abc.abstractproperty` + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21, instead + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, will be removed in Python 3.21. These types + are not generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` will be removed in Python 3.21. Use + the ``ast.Tuple.elts`` property instead. + +* :mod:`struct`: + + * Soft-deprecated since Python 3.15, using ``'F'`` and ``'D'`` type codes are now + deprecated. These codes will be removed in Python 3.21. Use instead + two-letter forms ``'Zf'`` and ``'Zd'``. + +* :mod:`tempfile`: + + * ``tempfile._TemporaryFileWrapper`` will be removed in Python 3.21. Use the + public :class:`tempfile.TemporaryFileWrapper` instead. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index e8306b8efee1c81..74f98d33a4b61f1 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -47,7 +47,7 @@ although there is currently no date scheduled for their removal. * :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) -* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method +* :attr:`!codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method instead. * :mod:`datetime`: diff --git a/Doc/deprecations/soft-deprecations.rst b/Doc/deprecations/soft-deprecations.rst new file mode 100644 index 000000000000000..a270052788ef2a1 --- /dev/null +++ b/Doc/deprecations/soft-deprecations.rst @@ -0,0 +1,21 @@ +Soft deprecations +----------------- + +There are no plans to remove :term:`soft deprecated` APIs. + +* :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index d33cbd2813d637b..6a807085cc328f0 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -231,10 +231,8 @@ calling the Python callback functions from a C callback. Other uses are also imaginable. Fortunately, the Python interpreter is easily called recursively, and there is a -standard interface to call a Python function. (I won't dwell on how to call the -Python parser with a particular string as input --- if you're interested, have a -look at the implementation of the :option:`-c` command line option in -:file:`Modules/main.c` from the Python source code.) +standard interface to call a Python function. (If you're interested in how to call the +Python parser with a particular string as input, see :ref:`veryhigh`.) Calling a Python function is easy. First, the Python program must somehow pass you the Python function object. You should provide a function (or some other @@ -641,7 +639,7 @@ and the object is freed. An alternative strategy is called :dfn:`automatic garbage collection`. (Sometimes, reference counting is also referred to as a garbage collection -strategy, hence my use of "automatic" to distinguish the two.) The big +strategy, hence the use of "automatic" to distinguish the two.) The big advantage of automatic garbage collection is that the user doesn't need to call :c:func:`free` explicitly. (Another claimed advantage is an improvement in speed or memory usage --- this is no hard fact however.) The disadvantage is that for @@ -1108,7 +1106,7 @@ Finally it should be mentioned that Capsules offer additional functionality, which is especially useful for memory allocation and deallocation of the pointer stored in a Capsule. The details are described in the Python/C API Reference Manual in the section :ref:`capsules` and in the implementation of Capsules (files -:file:`Include/pycapsule.h` and :file:`Objects/pycapsule.c` in the Python source +:file:`Include/pycapsule.h` and :file:`Objects/capsule.c` in the Python source code distribution). .. rubric:: Footnotes diff --git a/Doc/extending/first-extension-module.rst b/Doc/extending/first-extension-module.rst index cd755a98f7f5f4c..55a772e2aca24f5 100644 --- a/Doc/extending/first-extension-module.rst +++ b/Doc/extending/first-extension-module.rst @@ -164,7 +164,7 @@ Then, create ``meson.build`` containing the following: .. note:: - See `meson-python documentation `_ for details on + See the `meson-python documentation `_ for details on configuration. Now, build install the *project in the current directory* (``.``) via ``pip``: @@ -259,21 +259,25 @@ Rather than ``NULL``, the export hook should return the information needed to create a module. Let's start with the basics: the name and docstring. -The information should be defined in a ``static`` array of -:c:type:`PyModuleDef_Slot` entries, which are essentially key-value pairs. +The information should be defined in an array of +:c:type:`PySlot` entries, which are essentially key-value pairs. Define this array just before your export hook: .. code-block:: c PyABIInfo_VAR(abi_info); - static PyModuleDef_Slot spam_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "spam"}, - {Py_mod_doc, "A wonderful module with an example function"}, - {0, NULL} + static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_END }; +The :c:macro:`PySlot_STATIC_DATA` macro is used when the slot value +(here: ``&abi_info``, ``"spam"``, and the docstring) is a pointer to constant, +statically allocated data. + The ``PyABIInfo_VAR(abi_info);`` macro and the :c:data:`Py_mod_abi` slot are a bit of boilerplate that helps prevent extensions compiled for a different version of Python from crashing the interpreter. @@ -281,7 +285,8 @@ a different version of Python from crashing the interpreter. For both :c:data:`Py_mod_name` and :c:data:`Py_mod_doc`, the values are C strings -- that is, NUL-terminated, UTF-8 encoded byte arrays. -Note the zero-filled sentinel entry at the end. +Note ``PySlot_END`` sentinel entry at the end. +This marks the end of the array. If you forget it, you'll trigger undefined behavior. The array is defined as ``static`` -- that is, not visible outside this ``.c`` file. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index ac0aa81e56bb07a..02a3a3a22070dae 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -263,6 +263,8 @@ In general, structured switch statements execute one block of code when an expression has a particular value or set of values. Since Python 3.10 one can easily match literal values, or constants within a namespace, with a ``match ... case`` statement. +See :ref:`the specification ` and :ref:`the tutorial ` +for more information about :keyword:`match` statements. An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, @@ -428,6 +430,8 @@ tuples, but not lists, can be used as keys. Note, however, that a tuple is only hashable if all of its elements are hashable. +.. _how-are-lists-implemented: + How are lists implemented in CPython? ------------------------------------- @@ -443,6 +447,10 @@ cleverness is applied to improve the performance of appending items repeatedly; when the array must be grown, some extra space is allocated so the next few times don't require an actual resize. +See :ref:`time-complexity` for the costs of the various list operations. + + +.. _how-are-dictionaries-implemented: How are dictionaries implemented in CPython? -------------------------------------------- @@ -460,6 +468,8 @@ internal array where the value will be stored. Assuming that you're storing keys that all have different hash values, this means that dictionaries take constant time -- *O*\ (1), in Big-O notation -- to retrieve a key. +See :ref:`time-complexity` for the costs of the various dictionary operations. + Why must dictionary keys be immutable? -------------------------------------- diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 698f2117ff5c649..ff38ba395fba22c 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -152,10 +152,11 @@ Alpha, beta and release candidate versions have an additional suffix: In other words, all versions labeled *2.0aN* precede the versions labeled *2.0bN*, which precede versions labeled *2.0rcN*, and *those* precede 2.0. -You may also find version numbers with a "+" suffix, e.g. "2.2+". These are -unreleased versions, built directly from the CPython development repository. In -practice, after a final minor release is made, the version is incremented to the -next minor version, which becomes the "a0" version, e.g. "2.4a0". +You may also find version numbers with a "+dev" suffix, such as "3.15.0b3+dev". +These are unreleased versions, built directly from the CPython development +repository. After the first beta release is made, the version is +incremented to the next feature version, which becomes the "a0" version, +such as "3.16.0a0". See the `Developer's Guide `__ diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index ff34bb5d71c22ba..4e1157e6ebe7296 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -1136,7 +1136,7 @@ What is the most efficient way to concatenate many strings together? :class:`str` and :class:`bytes` objects are immutable, therefore concatenating many strings together is inefficient as each concatenation creates a new object. In the general case, the total runtime cost is quadratic in the -total string length. +total string length. See :ref:`time-complexity` for more information. To accumulate many :class:`str` objects, the recommended idiom is to place them into a list and call :meth:`str.join` at the end:: @@ -1924,7 +1924,7 @@ correctly using identity tests: .. code-block:: python - _sentinel = object() + _sentinel = sentinel('_sentinel') def pop(self, key, default=_sentinel): if key in self: @@ -2003,7 +2003,7 @@ How do I cache method calls? ---------------------------- The two principal tools for caching methods are -:func:`functools.cached_property` and :func:`functools.lru_cache`. The +:deco:`functools.cached_property` and :deco:`functools.lru_cache`. The former stores results at the instance level and the latter at the class level. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 6151143a97b420d..cd9d38b2fe4af29 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -39,10 +39,11 @@ Glossary ABCs with the :mod:`abc` module. annotate function - A function that can be called to retrieve the :term:`annotations ` - of an object. This function is accessible as the :attr:`~object.__annotate__` - attribute of functions, classes, and modules. Annotate functions are a - subset of :term:`evaluate functions `. + A callable that can be called to retrieve the :term:`annotations ` of + an object. Annotate functions are usually :term:`functions `, + automatically generated as the :attr:`~object.__annotate__` attribute of functions, + classes, and modules. Annotate functions are a subset of + :term:`evaluate functions `. annotation A label associated with a variable, a class @@ -95,21 +96,24 @@ Glossary :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. + Informally used to mean either an :term:`asynchronous generator + function` or an :term:`asynchronous generator iterator`, depending on + context. The formal terms :term:`asynchronous generator function` and + :term:`asynchronous generator iterator` are uncommon in practice; + "asynchronous generator" alone is almost always sufficient. + + asynchronous generator function + A function which returns an :term:`asynchronous generator iterator`. + It looks like a coroutine function defined with :keyword:`async def` + except that it contains :keyword:`yield` expressions for producing a + series of values usable in an :keyword:`async for` loop. See :pep:`525`. An asynchronous generator function may contain :keyword:`await` expressions as well as :keyword:`async for`, and :keyword:`async with` statements. asynchronous generator iterator - An object created by an :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator function`. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -412,7 +416,7 @@ Glossary decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for - decorators are :func:`classmethod` and :func:`staticmethod`. + decorators are :deco:`classmethod` and :deco:`staticmethod`. The decorator syntax is merely syntactic sugar, the following two function definitions are semantically equivalent:: @@ -505,7 +509,7 @@ Glossary A piece of syntax which can be evaluated to some value. In other words, an expression is an accumulation of expression elements like literals, names, attribute access, operators or function calls which all return a - value. In contrast to many other languages, not all language constructs + value. Not all language constructs are expressions. There are also :term:`statement`\s which cannot be used as expressions, such as :keyword:`while`. Assignments are also statements, not expressions. @@ -640,23 +644,33 @@ Glossary .. index:: single: generator generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. + Informally used to mean either a :term:`generator function` or a + :term:`generator iterator`, depending on context. The formal terms + :term:`generator function` and :term:`generator iterator` are uncommon + in practice; "generator" alone is almost always sufficient. - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. + .. index:: single: generator function + + generator function + A function which returns a :term:`generator` object. It looks like a + normal function except that it contains :keyword:`yield` expressions + for producing a series of values usable in a :keyword:`for`\-loop or + that can be retrieved one at a time with the :func:`next` function. + See :ref:`yieldexpr`. generator iterator - An object created by a :term:`generator` function. + An object created by a :term:`generator function` or a + :term:`generator expression`. Each :keyword:`yield` temporarily suspends processing, remembering the - execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). + execution state (including local variables and pending try-statements). + When the *generator iterator* resumes, it picks up where it left off + (in contrast to functions which start fresh on every invocation). + + Generator iterators also implement the :meth:`~generator.send` method + to send a value into the suspended generator, and the + :meth:`~generator.throw` method to raise an exception at the point + where the generator was paused. See :ref:`generator-methods`. .. index:: single: generator expression @@ -675,7 +689,7 @@ Glossary determined by the dispatch algorithm. See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. + :deco:`functools.singledispatch` decorator, and :pep:`443`. generic type A :term:`type` that can be parameterized; typically a @@ -928,7 +942,7 @@ Glossary list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to - elements is *O*\ (1). + elements is *O*\ (1). See :ref:`time-complexity`. list comprehension A compact way to process all or part of the elements in a sequence and @@ -1501,6 +1515,14 @@ Glossary stdlib An abbreviation of :term:`standard library`. + steal + In Python's C API, "*stealing*" an argument means that ownership of the + argument is transferred to the called function. + The caller must not use that reference after the call. + Generally, functions that "steal" an argument do so even if they fail. + + See :ref:`api-refcountdetails` for a full explanation. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst index 3adfedbf410ecc8..7a7a87cb9584001 100644 --- a/Doc/howto/a-conceptual-overview-of-asyncio.rst +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -115,7 +115,7 @@ The terms "coroutine function" and "coroutine object" are often conflated as coroutine. That can be confusing! In this article, coroutine specifically refers to a coroutine object, or more -precisely, an instance of :data:`types.CoroutineType` (native coroutine). +precisely, an instance of :class:`types.CoroutineType` (native coroutine). Note that coroutines can also exist as instances of :class:`collections.abc.Coroutine` -- a distinction that matters for type checking. diff --git a/Doc/howto/abi3t-migration.rst b/Doc/howto/abi3t-migration.rst new file mode 100644 index 000000000000000..ec78a33ed25820e --- /dev/null +++ b/Doc/howto/abi3t-migration.rst @@ -0,0 +1,714 @@ +.. highlight:: c + +.. _abi3t-migration-howto: + +****************************************************** +Migrating to Stable ABI for free threading (``abi3t``) +****************************************************** + +Starting with the 3.15 release, CPython supports a variant of the Stable ABI +that supports :term:`free-threaded ` Python: +the Stable ABI for Free-Threaded Builds, or ``abi3t`` for short. +This document describes how to adapt C API extensions to support free threading. + +Why do this +=========== + +The typical reason to use the Stable ABI is to reduce the number of artifacts +that you need to build and distribute for each version of your library. + +Without the Stable ABI, you must build a separate shared library, and typically +a *wheel* distribution, for each feature version of CPython you wish +to support. +For example, each tag in the following table represents a separate +library/wheel: + ++-----------------+-----------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+=======================+========================+ +| 3.12 | ``cpython-312`` | --- | ++-----------------+-----------------------+------------------------+ +| 3.13 | ``cpython-313`` | ``cpython-313t`` | ++-----------------+-----------------------+------------------------+ +| 3.14 | ``cpython-314`` | ``cpython-314t`` | ++-----------------+-----------------------+------------------------+ +| 3.15 | ``cpython-315`` | ``cpython-315t`` | ++-----------------+-----------------------+------------------------+ +| 3.16 | ``cpython-316`` | ``cpython-316t`` | ++-----------------+-----------------------+------------------------+ +| Later versions | :samp:`cpython-3{XX}` | :samp:`cpython-3{XX}t` | ++-----------------+-----------------------+------------------------+ + +That's a lot of builds, especially when multiplied by the number +of supported platforms. + +With the Stable ABI (``abi3``, introduced in CPython 3.2), a single extension +(per platform) can cover all *non-free-threaded* builds of CPython: + ++-----------------+-------------------+------------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+========================+ +| 3.12 | ``abi3`` | --- | ++-----------------+ +------------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+ +------------------------+ +| 3.15 | | ``cpython-315t`` | ++-----------------+ +------------------------+ +| 3.16 | | ``cpython-316t`` | ++-----------------+ +------------------------+ +| Later versions | | :samp:`cpython-3{XX}t` | ++-----------------+-------------------+------------------------+ + +The Stable ABI for free-threaded builds (``abi3t``), introduced in +CPython 3.15, does the same for free-threaded builds. +And it's compatible with non-free-threaded ones as well: + ++-----------------+-------------------+------------------+ +| CPython version | Non-free-threaded | Free-threaded | ++=================+===================+==================+ +| 3.12 | ``abi3`` * | --- | ++-----------------+ +------------------+ +| 3.13 | | ``cpython-313t`` | ++-----------------+ +------------------+ +| 3.14 | | ``cpython-314t`` | ++-----------------+-------------------+------------------+ +| 3.15 | ``abi3t`` | ++-----------------+ + +| 3.16 | | ++-----------------+ + +| Later versions | | ++-----------------+-------------------+------------------+ + +\* (As above, the ``abi3`` extension is compatible with all non-free-threaded +builds; even the 3.15+ ones that this table "attributes" to ``abi3t``.) + +Why *not* do this +----------------- + +There are two main downsides to the Stable ABI. + +First, your extension may become slower, since the Stable ABI prioritizes +compatibility over performance. +The difference is usually not noticeable, and often can be mitigated by +using the same source to build both a Stable ABI build and a few +version-specific ones for "tier 1" CPython versions. + +Second, not all of the C API is available. +Extensions need to be ported to build for the Stable ABI, which may be difficult +or, in rare cases, impossible. + +Specifically, ``abi3t`` requires APIs added in CPython 3.15. +If you want to build your extension for older versions of CPython from the +same source, you have two main options: + +- Use preprocessor conditionals. + + When following this guide, use ``#ifdef Py_TARGET_ABI3T`` blocks whenever + you are told to do a change that breaks the build on CPython versions you + care about. Keep the pre-existing code in ``#else`` blocks. + + For hand-written C extensions, this approach is reasonable down to + CPython 3.12, due to additions introduced in :pep:`697`. + Keeping compatibility with 3.11 and below may be worth it for code + generators (for example, Cython). + +- Do not port to ``abi3t``, and continue building separate extensions for + each version of CPython, until you can drop support for the older versions. + + This is a valid approach. Not all extensions need to switch to ``abi3t`` + right now. + + +Prerequisites +============= + +This guide assumes that you have an extension written directly in C (or C++), +which you want to port to ``abi3t``. + +If your extension uses a code generator (like Cython) or language binding +(like PyO3), it's best to wait until that tool has support for ``abi3t``. +If you maintain such a tool, you might be able to adapt the instructions +here for your tool. + +Non-free-threaded Stable ABI +---------------------------- + +Your extension should support the non-free-threaded Stable ABI (``abi3``). +If not, either port it first, or follow this guide but be prepared to fix +issues it does not mention. + +Free-threading support +---------------------- + +While it's technically not a hard prerequisite, you will most likely want to +prepare your extension for free threading before you port it to ``abi3t``. +See :ref:`freethreading-extensions-howto` for instructions. + +.. seealso:: + + `Porting Extension Modules to Support Free-Threading + `__: + A community-maintained porting guide for extension authors. + +Isolating extension modules +--------------------------- + +Your module should use :ref:`multi-phase initialization `, +and it should either be isolated or limit itself to be loaded at most once +per process. +If it is not your case, follow :ref:`isolating-extensions-howto` first. +(See the :ref:`opt-out section ` for a shortcut.) + +Avoiding variable-sized types +----------------------------- + +If your extension defines variable-sized types (using :c:macro:`Py_tp_itemsize` +or :c:member:`PyTypeObject.tp_itemsize`), it cannot be ported to +``abi3t`` 3.15. + + +Setting up the build +==================== + +If you use a build tool (such as setuptools, meson-python, scikit-build-core), +search its documentation for a way to select ``abi3t``. +At the time of writing, not all of them have this; but if your tool does, +use it. +You may want to verify that it set the right flag by temporarily adding the +following just after ``#include ``:: + + #if Py_TARGET_ABI3T+0 <= 0x30f0000 + #error "abi3t define is not set!" + #endif + +This should result in a different error than "``abi3t`` define is not set". + +.. note:: + + If your build tool doesn't support ``abi3t`` yet, set the following macro + before including ``Python.h``:: + + #define Py_TARGET_ABI3T 0x30f0000 + + or specify it as a compiler flag, for example:: + + -DPy_TARGET_ABI3T=0x30f0000 + + Once your extension builds with this setting, it will be compatible with + CPython 3.15 and above. + + If you set this macro manually, you will later need to name and tag the + resulting extension manually as well. + This is covered in :ref:`abi3t-migration-tagging` below. + +This guide will ask you to make a series of changes. +After each one, verify that your extension still builds in the original +(non-``abi3t``) configuration, and ideally run tests on all Python +versions you support. +This will ensure that nothing breaks as you are porting. + + +.. _abi3t-howto-modexport: + +Module export hook +================== + +Unless you've done this step already, your extension module defines a +:ref:`module initialization function ` +named :samp:`PyInit_{}`. +You will need to port it to a :ref:`module export hook `, +:samp:`PyModExport_{}`, a feature added in CPython 3.15 in +:pep:`793`. + +Your existing init function should look like this (with your own names +for ```` and ````): + +.. code-block:: + :class: bad + + PyMODINIT_FUNC + PyInit_(void) + { + return PyModuleDef_Init(&); + } + +If there is some code before the ``return``, move it to +a :c:macro:`Py_mod_create` or :c:macro:`Py_mod_exec` slot function. +See the :ref:`PyInit documentation ` for related information. + +The function references a ``PyModuleDef`` object (```` in the code +above). +Its definition should be similar to the following, with different values +and perhaps some fields unnnamed or left out: + +.. code-block:: + :class: bad + + static PyModuleDef = { + PyModuleDef_HEAD_INIT, + .m_name = "my_module", + .m_doc = "my docstring", + .m_size = sizeof(my_state_struct), + .m_methods = my_methods, + .m_slots = my_slots, + .m_traverse = my_traverse, + .m_clear = my_clear, + .m_free = my_free, + }; + +Remove this definition and the ``PyInit`` function (or put them in +an ``#ifndef Py_TARGET_ABI3T`` block, to retain backwards compatibility), +and replace them with the following: + +.. code-block:: + :class: good + + PyABIInfo_VAR(abi_info); + + static PySlot my_slot_array[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "my_module"), + PySlot_STATIC_DATA(Py_mod_doc, "my docstring"), + PySlot_SIZE(Py_mod_state_size, sizeof(my_state_struct)), + PySlot_STATIC_DATA(Py_mod_methods, my_methods), + PySlot_STATIC_DATA(Py_mod_slots, my_slots), + PySlot_FUNC(Py_mod_state_traverse, my_traverse), + PySlot_FUNC(Py_mod_state_clear, my_clear), + PySlot_FUNC(Py_mod_state_free, my_free), + PySlot_END + }; + + PyMODEXPORT_FUNC + PyModExport_(void) + { + return my_slot_array; + } + +Leave out any fields that were missing (except the new :c:macro:`Py_mod_abi`), +and substitute your own values. + +See the :c:type:`PySlot` and :c:ref:`export hook ` +documentation for details on this API. + +As in the example, your ``PyModExport_`` function should *only* return a +pointer to static data. +If you cannot avoid additional code, refer to the +:ref:`caveats in PyModExport documentation `. + + +Existing slots +-------------- + +If you have a ``Py_mod_slots`` slot, check the array it refers to. +It should be a :c:type:`PyModuleDef_Slot` array like the following: + +.. code-block:: + :class: bad + + static PyObject *create_module(PyObject *spec, PyModuleDef *def) { ... } + static int my_first_module_exec(PyObject *module) { ... } + static int my_second_module_exec(PyObject *module) { ... } + + static PyModuleDef_Slot my_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_create, my_module_create}, + {Py_mod_exec, my_first_module_exec}, + {Py_mod_exec, my_second_module_exec}, + {0, NULL} + }; + +``py_mod_create`` +................. + + +If you have a :c:macro:`Py_mod_create` entry, make sure the function can be +called with ``NULL`` as its second argument (instead of the +:c:type:`PyModuleDef`, which you are removing). +Often, this argument isn't used at all; you can check by renaming it: + +.. code-block:: + :class: good + + static PyObject *create_module(PyObject *spec, PyModuleDef *_unused) { ... } + +If the argument is used, find a different way to pass in the data. +Commonly, the information is static and you can refer to it directly. +(If you're reusing a single function for several different modules, consider +defining several functions instead.) + + +Multiple ``py_mod_exec`` +........................ + +If you have *more than one* :c:macro:`Py_mod_exec` entry, consolidate them: +create a new function that calls the others, and replace existing slots +with it. + +.. code-block:: + :class: good + + static int my_module_exec(PyObject *module) { + if (my_first_module_exec(module) < 0) return -1; + if (my_second_module_exec(module) < 0) return -1; + } + + static PyModuleDef_Slot my_slots[] = { + ... + /* (remove other Py_mod_exec slots) */ + ... + {Py_mod_exec, my_module_exec}, + {0, NULL} + }; + +If the functions aren't used elsewhere, you can combine their bodies instead. + + +Merging slot arrays +................... + +Optionally, when you break compatibility with Python 3.14, you may clean up +the code by moving slots into the :c:type:`PySlot` array, and converting the +definitions to :c:macro:`PySlot_DATA` and :c:macro:`PySlot_FUNC`: + +.. code-block:: + :class: good + + static PySlot my_slot_array[] = { + ... + PySlot_DATA(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_DATA(Py_mod_multiple_interpreters, + Py_MOD_PER_INTERPRETER_GIL_SUPPORTED) + PySlot_FUNC(Py_mod_create, my_module_create), + PySlot_FUNC(Py_mod_exec, my_module_exec), + PySlot_END + }; + +If you do this, delete the original :c:type:`PyModuleDef_Slot` array and +its ``Py_mod_slots`` entry. + + +Associated ``PyModuleDef`` +-------------------------- + +Since the new API does not use a :c:type:`!PyModuleDef` structure, a definition +will not be associated with the resulting module. +This changes the behavior of the following functions: + +- :c:func:`PyModule_GetDef` +- :c:func:`PyType_GetModuleByDef` + +Check your code for these. +If you do not use them, you can skip this section. + +These functions are typically used for two purposes: + +1. To get the definition the module was created with. + This is no longer possible using the new API. + Modules no longer keep a reference to the definition, so you will need to + figure out a different way to pass the relevant data around. + +.. _abi3t-migration-module-token: + +2. To check if a given module object is “yours”. + This use case is now served by :ref:`module tokens ` -- + opaque pointers that identify a module. + To use a token, declare (or reuse) a unique static variable, for example: + + .. code-block:: + :class: good + + static char my_token; + + and add a pointer to it in a new entry to your module's ``PySlot`` array: + + .. code-block:: + :class: good + :emphasize-lines: 3 + + static PySlot my_slot_array[] = { + ... + PySlot_STATIC_DATA(Py_mod_token, &my_token), + PySlot_END + } + + Then, switch from :c:func:`PyModule_GetDef` calls such as: + + .. code-block:: + :class: bad + + PyModuleDef *def = PyModule_GetDef(module); + + to :c:func:`PyModule_GetToken` (which uses an output argument and may fail + with an exception): + + .. code-block:: + :class: good + + void *token; + if (PyModule_GetToken(module, &token) < 0) { + /* handle error */ + } + + and from :c:func:`PyType_GetModuleByDef` calls such as: + + .. code-block:: + :class: bad + + PyObject *module = PyType_GetModuleByDef(type, my_def); + /* handle error; use module */ + + to :c:func:`PyType_GetModuleByToken` (which returns a strong reference): + + .. code-block:: + :class: good + + PyObject *module = PyType_GetModuleByToken(type, my_token); + /* handle error; use module */ + Py_XDECREF(module); + +``PyObject`` opaqueness +======================= + +The :c:type:`PyObject` and :c:type:`PyVarObject` structures are opaque +in ``abi3t``. + +Accessing their members is prohibited. +If you do this, switch to getter/setter functions mentioned in +their documentation: + +- :c:member:`PyObject.ob_type` +- :c:member:`PyObject.ob_refcnt` +- :c:member:`PyVarObject.ob_size` + +Also, the *size* of the :c:type:`PyObject` structures is +unknown to the compiler. +It can -- and *does* -- change between different CPython builds. + +.. note:: + + While the size is available at runtime (for example as + ``sys.getsizeof(object())`` in Python code), you should resist the + temptation to calculate pointer offsets from it. + The object memory layout is subject to change in future + ``abi3t`` implementations. + + +Custom type definitions +----------------------- + +Since :c:type:`!PyObject` is opaque, the traditional way of defining +custom types no longer works: + +.. code-block:: + :class: bad + + typedef struct { + PyObject_HEAD // expands to `PyObject ob_base;` which has unknown size + + int my_data; + } CustomObject; + + static PyType_Spec CustomType_spec = { + ... + .basicsize = sizeof(CustomObject), + ... + }; + +Most likely, all your class definitions, *and* all code that accesses +your classes' data, will need to be rewritten. +This will probably be the biggest change you need to support ``abi3t``. + +For each such type, instead of defining a ``struct`` for the entire instance, +define one with only the “additional” fields -- ones specific to your class, +not its superclasses: + +.. code-block:: + :class: good + + typedef struct { + int my_data; + } CustomObjectData; + +Change the name. +Almost all code that uses the struct will need to change +(notably, pointers to the new structure cannot be cast to/from ``PyObject*``), +and changing the name will highlight the usages as compiler errors. +(If you use ``typeof``, C++ ``auto``, or similar ways to avoid +typing the type name, this won't work. Be extra careful, and consider running +tools to detect undefined behavior.) + +Then, to create the class, use *negative* ``basicsize`` to indicate +“extra” storage space rather than *total* instance size: + +.. code-block:: + :class: good + + static PyType_Spec CustomType_spec = { + ... + .basicsize = -sizeof(CustomObjectData), /* note the minus sign */ + ... + }; + +If you use :c:macro:`Py_tp_members`, set the :c:macro:`Py_RELATIVE_OFFSET` +flag on each member and specify the :c:member:`~PyMemberDef.offset` +relative to your new struct. + + +Custom type data access +----------------------- + +Then comes the hard part: in all code that needs to access this struct, +you will need an additional :c:func:`PyObject_GetTypeData` call to +retrieve a ``CustomObjectData *`` pointer from ``PyObject *``: + +.. code-block:: + :class: good + + PyObject *obj = ...; + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + +Note that this call requires the *type object* for your class (``cls``). + +If your class is not subclassable (that is, it does not use the +:c:macro:`Py_TPFLAGS_BASETYPE` flag), ``cls`` will be ``Py_TYPE(obj)``. +Otherwise, **DO NOT USE** ``Py_TYPE`` with :c:func:`!PyObject_GetTypeData`: +it might return memory reserved to an unrelated subclass! +For example, if a user makes a subclass like this: + +.. code-block:: python + + class Sub(YourCustomClass): + __slots__ = ('a', 'b') + +then ``Py_TYPE(obj)`` is ``Sub``, and the underlying memory may +look like this: + +.. code-block:: text + + ╭─ PyObject *obj + │ ╭─ the pointer you want + │ │ ╭─ PyObject_GetTypeData(obj, Py_TYPE(obj)) + ▼ ▼ ▼ + ┌──────────┬───┬────────────────┬───┬─────────────┬───┬─────────────┐ + │ PyObject │...│ CustomTypeData │...│ PyObject *a │...│ PyObject *b │ + └──────────┴───┴────────────────┴───┴─────────────┴───┴─────────────┘ + +(Ellipses indicate possible padding. +Note that this memory layout is not guaranteed: future versions of Python may +add different padding or even switch the order of the structures.) + +There are two main ways to get the right class: + +- In instance methods, your implementation may use the :c:type:`PyCMethod` + signature (and the :c:macro:`METH_METHOD` bit in + :c:member:`PyMethodDef.ml_flags`), + and get the class as the ``defining_class`` argument. +- Otherwise, give your class a unique static token using the + :c:macro:`Py_tp_token` slot, and use: + + .. code-block:: + :class: good + + PyTypeObject cls; + if (PyType_GetBaseByToken(Py_TYPE(obj), my_tp_token, &cls) < 0) { + /* handle error */ + } + CustomObjectData *data = PyObject_GetTypeData(obj, cls); + + Type tokens work similarly to module tokens covered :ref:`earlier in this + guide `. + + + +Avoid build-time conditionals +============================= + +Check your code for API that identifies the version of Python used to +*build* your extension. +This no longer corresponds to the Python your extension runs on, so code +that uses this information often needs changing. +The macros to check for are: + +- :c:macro:`PY_VERSION_HEX`, :c:macro:`PY_MAJOR_VERSION`, + :c:macro:`PY_MINOR_VERSION`: + + - to get the run-time version, use :c:data:`Py_Version`; + - to determine what C API is available, use :c:macro:`Py_TARGET_ABI3T`. + This macro is set to the minimum supported version. + +- :c:macro:`Py_GIL_DISABLED`: under ``abi3t``, this macro is always defined. + Code that works with free-threaded Python *should* also work with + the GIL enabled (since the GIL can be enabled at run time), + and usually *does* (unless it, for some reason, requires more than one + :term:`attached thread state ` at one time). + + +Further code changes +==================== + +If you are still left with compiler errors or warnings, find a way to fix them. +Alas, this guide is limited, and cannot cover all possible code +changes extensions may need. + +If you find a problem that other extension authors might run into, +consider :ref:`reporting an issue ` (or sending +a pull request) for this guide. + +It is possible your issue cannot be fixed for the current version of ``abi3t``. +In that case, reporting it may help it get prioritized for the next version +of CPython. + + +.. _abi3t-migration-tagging: + +Tagging and distribution +======================== + +If you are using a build tool with ``abi3t`` support, your extension is ready, +but you might want to check that it was built correctly. + +Extensions built with ``abi3t`` should have the following extension: + +- On Windows: ``.pyd`` (like any other extension); +- Linux, macOS, and other systems that use the ``.so`` suffix: ``.abi3t.so`` + (**not** ``.cpython-315t.so`` or ``.abi3.so``). + Note that both free-threaded and non-free-threaded builds will + load ``.abi3t.so`` extensions; +- Other systems: consult your distributor, and perhaps update this guide. + +If you distribute the extension as a *wheel*, use the following tags: + +* Python tag: :samp:`cp3{XX}`, where *XX* is the minimum Python version + the extension is built for. + (For example, ``cp315`` if you set ``Py_TARGET_ABI3T`` to ``0x30f0000``. + See :ref:`abi3-compiling` for more values.) +* ABI tag: ``abi3.abi3t``. This is a *compressed tag set* that indicates + support for both non-free-threaded and free-threaded builds. + +For example, the wheel filename may look like this: + +.. code-block:: text + + myproject-1.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl + +.. seealso:: `Platform Compatibility Tags `__ in the PyPA package distribution metadata. + +If the filename or tags are incorrect, fix them. + + +Testing +======= + +Note that when you build an extension compatible with multiple versions of +CPython, you should always *test* it with each version it supports (for +example, 3.15, 3.16, and so on). +The Stable ABI only guarantees *ABI* compatibility; there may also be behavior +changes -- both intentional ones (covered by :pep:`387`) and bugs. + +Be sure to run tests on both free-threaded and non-free-threaded builds +of CPython. + +If they pass, congratulations! You have an ``abi3t`` extension. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index d7deb6c6bc1768f..fcc4fe838c783ba 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -4,8 +4,6 @@ Annotations Best Practices ************************** -:author: Larry Hastings - .. topic:: Abstract This document is designed to encapsulate the best practices @@ -154,7 +152,7 @@ on an arbitrary object ``o``: as the ``globals``, and ``dict(vars(o))`` as the ``locals``, when calling :func:`eval`. * If ``o`` is a wrapped callable using :func:`functools.update_wrapper`, - :func:`functools.wraps`, or :func:`functools.partial`, iteratively + :deco:`functools.wraps`, or :func:`functools.partial`, iteratively unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 902c50de00803c5..0cb8c5cc3ebd369 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -4,8 +4,6 @@ Argparse Tutorial ***************** -:author: Tshepang Mbambo - .. currentmodule:: argparse This tutorial is intended to be a gentle introduction to :mod:`argparse`, the diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 816639552d7cd6d..14f46efc1ec0fea 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -6,9 +6,6 @@ .. currentmodule:: curses -:Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 - .. topic:: Abstract @@ -55,8 +52,8 @@ code, all the functions described here will probably be available. The older versions of curses carried by some proprietary Unixes may not support everything, though. -The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called :pypi:`UniCurses` is available. +The Windows version of Python doesn't include the :mod:`curses` module. +The third-party :pypi:`windows-curses` package provides the same interface on Windows. The Python curses module @@ -298,17 +295,21 @@ underline, reverse code, or in color. They'll be explained in more detail in the next subsection. -The :meth:`~curses.window.addstr` method takes a Python string or -bytestring as the value to be displayed. The contents of bytestrings -are sent to the terminal as-is. Strings are encoded to bytes using -the value of the window's :attr:`~window.encoding` attribute; this defaults to -the default system encoding as returned by :func:`locale.getencoding`. +The :meth:`~curses.window.addstr` method takes a Python string, bytestring +or :class:`~curses.complexstr` as the value to be displayed. The contents +of bytestrings are sent to the terminal as-is. +On a build without wide-character support strings are encoded +using the value of the window's :attr:`~window.encoding` attribute; +this defaults to the default system encoding +as returned by :func:`locale.getencoding`. The :meth:`~curses.window.addch` methods take a character, which can be -either a string of length 1, a bytestring of length 1, or an integer. +either a string of length 1, a bytestring of length 1, an integer, or a +:class:`~curses.complexchar`. -Constants are provided for extension characters; these constants are -integers greater than 255. For example, :const:`ACS_PLMINUS` is a +/- +Constants are provided for the characters of the terminal's alternate +character set. +For example, :const:`ACS_PLMINUS` is a +/- symbol, and :const:`ACS_ULCORNER` is the upper left corner of a box (handy for drawing borders). You can also use the appropriate Unicode character. @@ -322,11 +323,11 @@ won't be distracting; it can be confusing to have the cursor blinking at some apparently random location. If your application doesn't need a blinking cursor at all, you can -call ``curs_set(False)`` to make it invisible. For compatibility -with older curses versions, there's a ``leaveok(bool)`` function -that's a synonym for :func:`~curses.curs_set`. When *bool* is true, the -curses library will attempt to suppress the flashing cursor, and you -won't need to worry about leaving it in odd locations. +call ``curs_set(False)`` to make it invisible. +The window method :meth:`~curses.window.leaveok` does something different: +when its argument is true, +curses leaves the cursor wherever the last update put it, +instead of moving it back to the window's cursor position. Attributes and Color @@ -367,6 +368,14 @@ could code:: curses.A_REVERSE) stdscr.refresh() +A :class:`~curses.complexchar` carries its attributes and color pair +together with the text of one character cell, +and a :class:`~curses.complexstr` is a run of such cells. +They are what :meth:`~curses.window.in_wch` and +:meth:`~curses.window.in_wchstr` return, +so a part of the screen can be read and written back +with its appearance intact. + The curses library also supports color on those terminals that provide it. The most common such terminal is probably the Linux console, followed by color xterms. @@ -432,40 +441,48 @@ The C curses library offers only very simple input mechanisms. Python's :mod:`curses` module adds a basic text-input widget. (Other libraries such as :pypi:`Urwid` have more extensive collections of widgets.) -There are two methods for getting input from a window: +There are three methods for getting input from a window: -* :meth:`~curses.window.getch` refreshes the screen and then waits for +* :meth:`~curses.window.get_wch` refreshes the screen and then waits for the user to hit a key, displaying the key if :func:`~curses.echo` has been called earlier. You can optionally specify a coordinate to which the cursor should be moved before pausing. -* :meth:`~curses.window.getkey` does the same thing but converts the - integer to a string. Individual characters are returned as - 1-character strings, and special keys such as function keys return - longer strings containing a key name such as ``KEY_UP`` or ``^G``. +* :meth:`~curses.window.getch` does the same thing but returns the code of + the key instead of a character. + With ncurses this is a single byte of the key's encoding in the current + locale, so a character encoded with several bytes takes several calls, + one byte per call. + +* :meth:`~curses.window.getkey` does the same as :meth:`!getch` but returns + a string: + an ordinary key as a 1-character string, + and a special key as its name, such as ``KEY_UP``. It's possible to not wait for the user using the :meth:`~curses.window.nodelay` window method. After ``nodelay(True)``, -:meth:`!getch` and :meth:`!getkey` for the window become -non-blocking. To signal that no input is ready, :meth:`!getch` returns -``curses.ERR`` (a value of -1) and :meth:`!getkey` raises an exception. +the reads for the window become non-blocking. +To signal that no input is ready, +:meth:`!get_wch` and :meth:`!getkey` raise an exception, +and :meth:`!getch` returns ``-1``. There's also a :func:`~curses.halfdelay` function, which can be used to (in -effect) set a timer on each :meth:`!getch`; if no input becomes +effect) set a timer on each read; if no input becomes available within a specified delay (measured in tenths of a second), -curses raises an exception. +the read fails the same way. -The :meth:`!getch` method returns an integer; if it's between 0 and 255, it -represents the ASCII code of the key pressed. Values greater than 255 are -special keys such as Page Up, Home, or the cursor keys. You can compare the -value returned to constants such as :const:`curses.KEY_PPAGE`, +Special keys such as Page Up, Home, or the cursor keys are returned by all +three as one of the :ref:`KEY_* constants `, +all larger than 255. +You can compare the value returned to constants such as +:const:`curses.KEY_PPAGE`, :const:`curses.KEY_HOME`, or :const:`curses.KEY_LEFT`. The main loop of your program may look something like this:: while True: - c = stdscr.getch() - if c == ord('p'): + c = stdscr.get_wch() + if c == 'p': PrintDocument() - elif c == ord('q'): + elif c == 'q': break # Exit the while loop elif c == curses.KEY_HOME: x = y = 0 @@ -477,16 +494,17 @@ conversion functions that take either integer or 1-character-string arguments and return the same type. For example, :func:`curses.ascii.ctrl` returns the control character corresponding to its argument. -There's also a method to retrieve an entire string, -:meth:`~curses.window.getstr`. It isn't used very often, because its +There's also a method to retrieve an entire line, +:meth:`~curses.window.get_wstr`. It isn't used very often, because its functionality is quite limited; the only editing keys available are -the backspace key and the Enter key, which terminates the string. It -can optionally be limited to a fixed number of characters. :: +the erase and kill characters, and the Enter key, which terminates the line. +It can optionally be limited to a fixed number of characters; +:meth:`~curses.window.getstr` returns a bytes object instead. :: curses.echo() # Enable echoing of characters - # Get a 15-character string, with the cursor on the top line - s = stdscr.getstr(0,0, 15) + # Get a line of at most 15 characters, with the cursor on the top line + s = stdscr.get_wstr(0,0, 15) The :mod:`curses.textpad` module supplies a text box that supports an Emacs-like set of keybindings. Various methods of the diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 9d5a9ac8b718cb0..7233898860ada14 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -4,9 +4,6 @@ Descriptor Guide ================ -:Author: Raymond Hettinger -:Contact: - .. Contents:: @@ -28,7 +25,7 @@ This guide has four major sections: 4) The last section has pure Python equivalents for built-in descriptors that are written in C. Read this if you're curious about how functions turn into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :deco:`classmethod`, :deco:`staticmethod`, :deco:`property`, and :term:`__slots__`. @@ -317,8 +314,8 @@ Descriptors invert that relationship and allow the data being looked-up to have a say in the matter. Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as +bound methods. Common tools like :deco:`classmethod`, :deco:`staticmethod`, +:deco:`property`, and :deco:`functools.cached_property` are all implemented as descriptors. @@ -594,7 +591,7 @@ a pure Python equivalent: def object_getattribute(obj, name): "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() + null = sentinel('null') objtype = type(obj) cls_var = find_name_in_mro(objtype, name, null) descr_get = getattr(type(cls_var), '__get__', null) @@ -1326,7 +1323,7 @@ example calls are unexciting: 30 Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: +:deco:`staticmethod` would look like this: .. testcode:: @@ -1466,7 +1463,7 @@ Now a new dictionary of unique keys can be constructed like this: {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: +:deco:`classmethod` would look like this: .. testcode:: @@ -1604,7 +1601,7 @@ matters when a large number of instances are going to be created. 4. Improves speed. Reading instance variables is 35% faster with ``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). -5. Blocks tools like :func:`functools.cached_property` which require an +5. Blocks tools like :deco:`functools.cached_property` which require an instance dictionary to function correctly: .. testcode:: @@ -1635,12 +1632,12 @@ by member descriptors: .. testcode:: - null = object() + null = sentinel('null') class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 5260c2ca4add471..adb9dc9a4879eb5 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -256,7 +256,7 @@ Ensuring unique enumeration values ---------------------------------- By default, enumerations allow multiple names as aliases for the same value. -When this behavior isn't desired, you can use the :func:`unique` decorator:: +When this behavior isn't desired, you can use the :deco:`unique` decorator:: >>> from enum import Enum, unique >>> @unique @@ -371,7 +371,7 @@ Equality comparisons are defined though:: >>> Color.BLUE == Color.BLUE True -Comparisons against non-enumeration values will always compare not equal +Equality comparisons against non-enumeration values will always return ``False`` (again, :class:`IntEnum` was explicitly designed to behave differently, see below):: @@ -509,7 +509,7 @@ to use the standard :func:`repr`. .. note:: - Adding :func:`~dataclasses.dataclass` decorator to :class:`Enum` + Adding :deco:`~dataclasses.dataclass` decorator to :class:`Enum` and its subclasses is not supported. It will not raise any errors, but it will produce very strange results at runtime, such as members being equal to each other:: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 2f089a3d89680a9..ad0578df0a27029 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -218,13 +218,15 @@ Thread State and GIL APIs Python provides a set of functions and macros to manage thread state and the GIL, such as: +* :c:func:`PyThreadState_Ensure`, :c:func:`PyThreadState_EnsureFromView`, + and :c:func:`PyThreadState_Release` * :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` * :c:func:`PyEval_SaveThread` and :c:func:`PyEval_RestoreThread` * :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` These functions should still be used in the free-threaded build to manage thread state even when the :term:`GIL` is disabled. For example, if you -create a thread outside of Python, you must call :c:func:`PyGILState_Ensure` +create a thread outside of Python, you must call :c:func:`PyThreadState_Ensure` before calling into the Python API to ensure that the thread has a valid Python thread state. @@ -416,11 +418,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build on Python 3.13 and 3.14. On Python 3.14, free-threaded - wheels will be built by default. On Python 3.13, you will need to set - `CIBW_ENABLE to cpython-freethreading `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 380c2be04957d5e..53bea1db191d76f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -165,3 +165,132 @@ to false. If the flag is true then the :class:`warnings.catch_warnings` context manager uses a context variable for warning filters. If the flag is false then :class:`~warnings.catch_warnings` modifies the global filters list, which is not thread-safe. See the :mod:`warnings` module for more details. + + +Increased memory usage +---------------------- + +The free-threaded build will typically use more memory compared to the default +build. There are multiple reasons for this, mostly due to design decisions. + + +All interned strings are immortal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For modern Python versions (since version 2.3), interning a string (e.g. with +:func:`sys.intern`) does not cause it to become immortal. Instead, if the last +reference to that string disappears, it will be removed from the interned +string table. This is not the case for the free-threaded build and any interned +string will become immortal, surviving until interpreter shutdown. + + +Non-GC objects have a larger object header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The free-threaded build uses a different :c:type:`PyObject` structure. Instead +of having the GC related information allocated before the :c:type:`PyObject` +structure, like in the default build, the GC related info is part of the normal +object header. For example, on the AMD64 platform, ``None`` uses 32 bytes on +the free-threaded build vs 16 bytes for the default build. GC objects (such as +dicts and lists) are the same size for both builds since the free-threaded +build does not use additional space for the GC info. + + +QSBR can delay freeing of memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to safely implement lock-free data structures, a safe memory +reclamation (SMR) scheme is used, known as quiescent state-based reclamation +(QSBR). This means that the memory backing data structures allowing lock-free +access will use QSBR, which defers the free operation, rather than immediately +freeing the memory. Two examples of these data structures are the list object +and the dictionary keys object. See ``InternalDocs/qsbr.md`` in the CPython +source tree for more details on how QSBR is implemented. Running +:func:`gc.collect` should cause all memory being held by QSBR to be actually +freed. Note that even when QSBR frees the memory, the underlying memory +allocator may not immediately return that memory to the OS and so the resident +set size (RSS) of the process might not decrease. + + +mimalloc allocator vs pymalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default build will normally use the "pymalloc" memory allocator for small +allocations (512 bytes or smaller). The free-threaded build does not use +pymalloc and allocates all Python objects using the "mimalloc" allocator. The +pymalloc allocator has the following properties that help keep memory usage +low: small per-allocated-block overhead, effective memory fragmentation +prevention, and quick return of free memory to the operating system. The +mimalloc allocator does quite well in these respects as well but can have some +more overhead. + +In the free-threaded build, mimalloc manages memory in a number of separate +heaps (currently four). For example, all GC supporting objects are allocated +from their own heap. Using separate heaps means that free memory in one heap +cannot be used for an allocation that uses another heap. Also, some heaps are +configured to use QSBR (quiescent-state based reclamation) when freeing the +memory that backs up the heap (known as "pages" in mimalloc terminology). The +use of QSBR creates a delay between all memory blocks for a page being freed +and the memory page being released, either for new allocations or back to the +OS. + +The mimalloc allocator also defers returning freed memory back to the OS. You +can reduce that delay by setting the environment variable +:envvar:`!MIMALLOC_PURGE_DELAY` to ``0``. Note that this will likely reduce +the performance of the allocator. + + +Free-threaded reference counting can cause objects to live longer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the default build, when an object's reference count reaches zero, it is +normally deallocated. The free-threaded build uses "biased reference +counting", with a fast-path for objects "owned" by the current thread and a +slow path for other objects. See :pep:`703` for additional details. Any time +an object's reference count ends up in a "queued" state, deallocation can be +deferred. The queued state is cleared from the "eval breaker" section of the +bytecode evaluator. + +The free-threaded build also allows a different mode of reference counting, +known as "deferred reference counting". This mode is enabled by setting a flag +on a per-object basis. Deferred reference counting is enabled for the +following types: + +* module objects +* module top-level functions +* class methods defined in the class scope +* descriptor objects +* thread-local objects, created by :class:`threading.local` + +When deferred reference counting is enabled, references from Python function +stacks are not added to the reference count. This scheme reduces the overhead +of reference counting, especially for objects used from multiple threads. +Because the stack references are not counted, objects with deferred reference +counting are not immediately freed when their internal reference count goes to +zero. Instead, they are examined by the next GC run and, if no stack +references to them are found, they are freed. This means these objects are +freed by the GC and not when their reference count goes to zero, as is typical. + + +Per-thread reference counting can delay freeing objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid contention on the reference count fields of frequently shared +objects, the free-threaded build also uses "per-thread reference counting" +for a few selected object types. Rather than updating a single shared +reference count, each thread maintains its own local reference count array, +indexed by a unique id assigned to the object. The true reference count is +only computed by summing the per-thread counts when the object's local +count drops to zero. Per-thread reference counting is currently used for: + +* heap type objects (classes created in Python) +* code objects +* the ``__dict__`` of module objects + +Because the per-thread counts must be merged back to the object before it +can be deallocated, objects using per-thread reference counting are +typically freed later than they would be in the default build. In +particular, such an object is usually not freed until the thread that +referenced it reaches a safe point (for example, in the "eval breaker" +section of the bytecode evaluator) or exits. Running :func:`gc.collect` +will merge the per-thread counts and allow these objects to be freed. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 552514063c95ab2..a61fdaee27f6b18 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,9 +4,6 @@ Functional Programming HOWTO ******************************** -:Author: \A. M. Kuchling -:Release: 0.32 - In this document, we'll take a tour of Python's features suitable for implementing programs in a functional style. After an introduction to the concepts of functional programming, we'll look at language features such as @@ -1042,7 +1039,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) @@ -1185,7 +1182,8 @@ about whether this lambda-free style is better. Revision History and Acknowledgements ===================================== -The author would like to thank the following people for offering suggestions, +This HOWTO was originally written by A. M. Kuchling. The author would like to +thank the following people for offering suggestions, corrections and assistance with various drafts of this article: Ian Bicking, Nick Coghlan, Nick Efford, Raymond Hettinger, Jim Jewett, Mike Krell, Leandro Lameiro, Jussi Salmela, Collin Winter, Blake Winton. @@ -1239,9 +1237,9 @@ Text Processing". Mertz also wrote a 3-part series of articles on functional programming for IBM's DeveloperWorks site; see -`part 1 `__, -`part 2 `__, and -`part 3 `__, +`part 1 `__, +`part 2 `__, and +`part 3 `__. Python documentation diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 81fc7e63f35bd79..57e2d6e0752447f 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -37,6 +37,7 @@ Python Library Reference. mro.rst free-threading-python.rst free-threading-extensions.rst + abi3t-migration.rst remote_debugging.rst General: @@ -61,6 +62,7 @@ Advanced development: * :ref:`freethreading-python-howto` * :ref:`freethreading-extensions-howto` * :ref:`isolating-extensions-howto` +* :ref:`abi3t-migration-howto` * :ref:`python_2.3_mro` * :ref:`socket-howto` * :ref:`timerfd-howto` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 06c1ae40da5e67e..8f0b0c41ea48617 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -6,9 +6,6 @@ Instrumenting CPython with DTrace and SystemTap =============================================== -:author: David Malcolm -:author: Łukasz Langa - DTrace and SystemTap are monitoring tools, each providing a way to inspect what the processes on a computer system are doing. They both use domain-specific languages allowing a user to write scripts which: diff --git a/Doc/howto/ipaddress.rst b/Doc/howto/ipaddress.rst index e852db98156facc..646c4c4d9e7ab14 100644 --- a/Doc/howto/ipaddress.rst +++ b/Doc/howto/ipaddress.rst @@ -8,9 +8,6 @@ An introduction to the ipaddress module *************************************** -:author: Peter Moody -:author: Nick Coghlan - .. topic:: Overview This document aims to provide a gentle introduction to the diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 21df6ba858d8c2d..2ff221cf258eda5 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4,8 +4,6 @@ Logging Cookbook ================ -:Author: Vinay Sajip - This page contains a number of recipes related to logging, which have been found useful in the past. For links to tutorial and reference information, please see :ref:`cookbook-ref-links`. @@ -1957,7 +1955,7 @@ Subclass ``QueueListener`` class NNGSocketListener(logging.handlers.QueueListener): def __init__(self, uri, /, *handlers, **kwargs): - # Have a timeout for interruptability, and open a + # Have a timeout for interruptibility, and open a # subscriber socket socket = pynng.Sub0(listen=uri, recv_timeout=500) # The b'' subscription matches all topics @@ -3877,7 +3875,7 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 asctime = dt.datetime.fromtimestamp(record.created).isoformat() - m = self.tz_offset.match(time.strftime('%z')) + m = self.tz_offset.prefixmatch(time.strftime('%z')) has_offset = False if m and time.timezone: hrs, mins = m.groups() diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 454e2f4930e724d..c8ce0df9e937f8c 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -4,8 +4,6 @@ Logging HOWTO ============= -:Author: Vinay Sajip - .. _logging-basic-tutorial: .. currentmodule:: logging diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index 0872bedcd3a2d34..72cc311e30d669c 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -10,7 +10,7 @@ The Python 2.3 Method Resolution Order The Method Resolution Order discussed here was *introduced* in Python 2.3, but it is still used in later versions -- including Python 3. -By `Michele Simionato `__. +By `Michele Simionato `__. :Abstract: diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index fc4772bbccab57a..5565f99b244f11b 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -6,8 +6,6 @@ Python support for the ``perf map`` compatible profilers ======================================================== -:author: Pablo Galindo - `The Linux perf profiler `_ and `samply `_ are powerful tools that allow you to profile and obtain information about the performance of your application. @@ -217,8 +215,9 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: How to obtain the best results ------------------------------ -For best results, Python should be compiled with -``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows +For best results, keep frame pointers enabled. On supported GCC-compatible +toolchains, CPython builds itself with ``-fno-omit-frame-pointer`` and similar +flags (see :option:`--without-frame-pointers` for details). These flags allow profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` support is dynamically generated it doesn't have any DWARF debugging information diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst index 9f73c811cfcbc0d..f19f6006dbac9e7 100644 --- a/Doc/howto/pyporting.rst +++ b/Doc/howto/pyporting.rst @@ -6,8 +6,6 @@ How to port Python 2 Code to Python 3 ************************************* -:author: Brett Cannon - Python 2 reached its official end-of-life at the start of 2020. This means that no new bug reports, fixes, or changes will be made to Python 2 - it's no longer supported: see :pep:`373` and diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 84ec535ca98e973..ecdb35136a99f3b 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -4,8 +4,6 @@ Regular expression HOWTO **************************** -:Author: A.M. Kuchling - .. TODO: Document lookbehind assertions Better way of displaying a RE, a string, and what it matches @@ -362,12 +360,13 @@ for a complete listing. +------------------+-----------------------------------------------+ | Method/Attribute | Purpose | +==================+===============================================+ -| ``match()`` | Determine if the RE matches at the beginning | -| | of the string. | -+------------------+-----------------------------------------------+ | ``search()`` | Scan through a string, looking for any | | | location where this RE matches. | +------------------+-----------------------------------------------+ +| ``prefixmatch()``| Determine if the RE matches at the beginning | +| | of the string. Previously named :ref:`match() | +| | `. | ++------------------+-----------------------------------------------+ | ``findall()`` | Find all substrings where the RE matches, and | | | return them as a list. | +------------------+-----------------------------------------------+ @@ -375,7 +374,7 @@ for a complete listing. | | return them as an :term:`iterator`. | +------------------+-----------------------------------------------+ -:meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If +:meth:`~re.Pattern.search` and :meth:`~re.Pattern.prefixmatch` return ``None`` if no match can be found. If they're successful, a :ref:`match object ` instance is returned, containing information about the match: where it starts and ends, the substring it matched, and more. @@ -393,19 +392,19 @@ Python interpreter, import the :mod:`re` module, and compile a RE:: Now, you can try matching various strings against the RE ``[a-z]+``. An empty string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`~re.Pattern.match` should return ``None`` in this case, which will cause the +:meth:`~re.Pattern.search` should return ``None`` in this case, which will cause the interpreter to print no output. You can explicitly print the result of -:meth:`!match` to make this clear. :: +:meth:`!search` to make this clear. :: - >>> p.match("") - >>> print(p.match("")) + >>> p.search("") + >>> print(p.search("")) None Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`~re.Pattern.match` will return a :ref:`match object `, so you +case, :meth:`~re.Pattern.search` will return a :ref:`match object `, so you should store the result in a variable for later use. :: - >>> m = p.match('tempo') + >>> m = p.search('tempo') >>> m @@ -437,27 +436,28 @@ Trying these methods will soon clarify their meaning:: :meth:`~re.Match.group` returns the substring that was matched by the RE. :meth:`~re.Match.start` and :meth:`~re.Match.end` return the starting and ending index of the match. :meth:`~re.Match.span` -returns both start and end indexes in a single tuple. Since the :meth:`~re.Pattern.match` -method only checks if the RE matches at the start of a string, :meth:`!start` -will always be zero. However, the :meth:`~re.Pattern.search` method of patterns -scans through the string, so the match may not start at zero in that -case. :: +returns both start and end indexes in a single tuple. +The :meth:`~re.Pattern.search` method of patterns +scans through the string, so the match may not start at zero. +However, the :meth:`~re.Pattern.prefixmatch` +method only checks if the RE matches at the start of a string, so :meth:`!start` +will always be zero in that case. :: - >>> print(p.match('::: message')) - None >>> m = p.search('::: message'); print(m) >>> m.group() 'message' >>> m.span() (4, 11) + >>> print(p.prefixmatch('::: message')) + None In actual programs, the most common style is to store the :ref:`match object ` in a variable, and then check if it was ``None``. This usually looks like:: p = re.compile( ... ) - m = p.match( 'string goes here' ) + m = p.search( 'string goes here' ) if m: print('Match found: ', m.group()) else: @@ -495,15 +495,15 @@ Module-level functions ---------------------- You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions +:mod:`re` module also provides top-level functions called :func:`~re.search`, +:func:`~re.prefixmatch`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions take the same arguments as the corresponding pattern method with the RE string added as the first argument, and still return either ``None`` or a :ref:`match object ` instance. :: - >>> print(re.match(r'From\s+', 'Fromage amk')) + >>> print(re.prefixmatch(r'From\s+', 'Fromage amk')) None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS + >>> re.prefixmatch(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS Under the hood, these functions simply create a pattern object for you @@ -812,7 +812,7 @@ of a group with a quantifier, such as ``*``, ``+``, ``?``, or ``ab``. :: >>> p = re.compile('(ab)*') - >>> print(p.match('ababababab').span()) + >>> print(p.search('ababababab').span()) (0, 10) Groups indicated with ``'('``, ``')'`` also capture the starting and ending @@ -825,7 +825,7 @@ argument. Later we'll see how to express groups that don't capture the span of text that they match. :: >>> p = re.compile('(a)b') - >>> m = p.match('ab') + >>> m = p.search('ab') >>> m.group() 'ab' >>> m.group(0) @@ -836,7 +836,7 @@ to determine the number, just count the opening parenthesis characters, going from left to right. :: >>> p = re.compile('(a(b)c)d') - >>> m = p.match('abcd') + >>> m = p.search('abcd') >>> m.group(0) 'abcd' >>> m.group(1) @@ -912,10 +912,10 @@ but aren't interested in retrieving the group's contents. You can make this fact explicit by using a non-capturing group: ``(?:...)``, where you can replace the ``...`` with any other regular expression. :: - >>> m = re.match("([abc])+", "abc") + >>> m = re.search("([abc])+", "abc") >>> m.groups() ('c',) - >>> m = re.match("(?:[abc])+", "abc") + >>> m = re.search("(?:[abc])+", "abc") >>> m.groups() () @@ -949,7 +949,7 @@ given numbers, so you can retrieve information about a group in two ways:: Additionally, you can retrieve named groups as a dictionary with :meth:`~re.Match.groupdict`:: - >>> m = re.match(r'(?P\w+) (?P\w+)', 'Jane Doe') + >>> m = re.search(r'(?P\w+) (?P\w+)', 'Jane Doe') >>> m.groupdict() {'first': 'Jane', 'last': 'Doe'} @@ -1274,21 +1274,26 @@ In short, before turning to the :mod:`re` module, consider whether your problem can be solved with a faster and simpler string method. -match() versus search() ------------------------ +.. _match-versus-search: + +prefixmatch() (aka match) versus search() +----------------------------------------- -The :func:`~re.match` function only checks if the RE matches at the beginning of the -string while :func:`~re.search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`!match` will -only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`!match` will *not* report it. :: +:func:`~re.prefixmatch` was added in Python 3.15 as the :ref:`preferred name +` for :func:`~re.match`. Before this, it was only known +as :func:`!match` and the distinction with :func:`~re.search` was often +misunderstood. - >>> print(re.match('super', 'superstition').span()) +:func:`!prefixmatch` aka :func:`!match` only checks if the RE matches at the +beginning of the string while :func:`!search` scans forward through the +string for a match. :: + + >>> print(re.prefixmatch('super', 'superstition').span()) (0, 5) - >>> print(re.match('super', 'insuperable')) + >>> print(re.prefixmatch('super', 'insuperable')) None -On the other hand, :func:`~re.search` will scan forward through the string, +On the other hand, :func:`~re.search` scans forward through the string, reporting the first match it finds. :: >>> print(re.search('super', 'superstition').span()) @@ -1296,18 +1301,8 @@ reporting the first match it finds. :: >>> print(re.search('super', 'insuperable').span()) (2, 7) -Sometimes you'll be tempted to keep using :func:`re.match`, and just add ``.*`` -to the front of your RE. Resist this temptation and use :func:`re.search` -instead. The regular expression compiler does some analysis of REs in order to -speed up the process of looking for a match. One such analysis figures out what -the first character of a match must be; for example, a pattern starting with -``Crow`` must match starting with a ``'C'``. The analysis lets the engine -quickly scan through the string looking for the starting character, only trying -the full match if a ``'C'`` is found. - -Adding ``.*`` defeats this optimization, requiring scanning to the end of the -string and then backtracking to find a match for the rest of the RE. Use -:func:`re.search` instead. +This distinction is important to remember when using the old :func:`~re.match` +name in code requiring compatibility with older Python versions. Greedy versus non-greedy @@ -1322,9 +1317,9 @@ doesn't work because of the greedy nature of ``.*``. :: >>> s = 'Title' >>> len(s) 32 - >>> print(re.match('<.*>', s).span()) + >>> print(re.prefixmatch('<.*>', s).span()) (0, 32) - >>> print(re.match('<.*>', s).group()) + >>> print(re.prefixmatch('<.*>', s).group()) Title The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of @@ -1340,7 +1335,7 @@ example, the ``'>'`` is tried immediately after the first ``'<'`` matches, and when it fails, the engine advances a character at a time, retrying the ``'>'`` at every step. This produces just the right result:: - >>> print(re.match('<.*?>', s).group()) + >>> print(re.prefixmatch('<.*?>', s).group()) (Note that parsing HTML or XML with regular expressions is painful. diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index cbc49d15a0771b9..b17ab3f4391dad0 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -4,9 +4,6 @@ Socket Programming HOWTO **************************** -:Author: Gordon McMillan - - .. topic:: Abstract Sockets are used nearly everywhere, but are one of the most severely diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst index 70c34cde8a06592..b511aa4e59fe1e7 100644 --- a/Doc/howto/sorting.rst +++ b/Doc/howto/sorting.rst @@ -3,9 +3,6 @@ Sorting Techniques ****************** -:Author: Andrew Dalke and Raymond Hettinger - - Python lists have a built-in :meth:`list.sort` method that modifies the list in-place. There is also a :func:`sorted` built-in function that builds a new sorted list from an iterable. @@ -375,7 +372,7 @@ Odds and Ends :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__` for details on the mechanics). To avoid surprises, :pep:`8` recommends that all six comparison methods be implemented. - The :func:`~functools.total_ordering` decorator is provided to make that + The :deco:`~functools.total_ordering` decorator is provided to make that task easier. * Key functions need not depend directly on the objects being sorted. A key diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 243cc27bac7025d..cbd53d5b619f5a3 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -4,8 +4,6 @@ Unicode HOWTO ***************** -:Release: 1.12 - This HOWTO discusses Python's support for the Unicode specification for representing textual data, and explains various problems that people commonly encounter when trying to work with Unicode. diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 4e77d2cb407f726..e4f218f088ba89d 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -4,9 +4,6 @@ HOWTO Fetch Internet Resources Using The urllib Package *********************************************************** -:Author: `Michael Foord `_ - - Introduction ============ diff --git a/Doc/includes/capi-extension/spammodule-01.c b/Doc/includes/capi-extension/spammodule-01.c index 0bc34ef57445cb9..8ddb416e7d6cd08 100644 --- a/Doc/includes/capi-extension/spammodule-01.c +++ b/Doc/includes/capi-extension/spammodule-01.c @@ -37,12 +37,12 @@ static PyMethodDef spam_methods[] = { PyABIInfo_VAR(abi_info); -static PyModuleDef_Slot spam_slots[] = { - {Py_mod_abi, &abi_info}, - {Py_mod_name, "spam"}, - {Py_mod_doc, "A wonderful module with an example function"}, - {Py_mod_methods, spam_methods}, - {0, NULL} +static PySlot spam_slots[] = { + PySlot_STATIC_DATA(Py_mod_abi, &abi_info), + PySlot_STATIC_DATA(Py_mod_name, "spam"), + PySlot_STATIC_DATA(Py_mod_doc, "A wonderful module with an example function"), + PySlot_STATIC_DATA(Py_mod_methods, spam_methods), + PySlot_END }; /// Export hook prototype diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index 47f5eabb6f2180f..13f463a1e95340e 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -162,58 +157,66 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. - .. versionchanged:: 3.14 - Lock acquires can now be interrupted by signals on Windows. + .. versionchanged:: 3.2 + The *timeout* parameter is new. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. -.. method:: lock.release() + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 8112cfee7d204db..be25a94e7e94f6f 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -166,17 +166,17 @@ The :mod:`!abc` module also provides the following decorator: or is derived from it. A class that has a metaclass derived from :class:`!ABCMeta` cannot be instantiated unless all of its abstract methods and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`!abstractmethod` may be used + of the normal 'super' call mechanisms. :deco:`!abstractmethod` may be used to declare abstract methods for properties and descriptors. Dynamically adding abstract methods to a class, or attempting to modify the abstraction status of a method or class once it is created, are only supported using the :func:`update_abstractmethods` function. The - :func:`!abstractmethod` only affects subclasses derived using regular + :deco:`!abstractmethod` only affects subclasses derived using regular inheritance; "virtual subclasses" registered with the ABC's :meth:`~ABCMeta.register` method are not affected. - When :func:`!abstractmethod` is applied in combination with other method + When :deco:`!abstractmethod` is applied in combination with other method descriptors, it should be applied as the innermost decorator, as shown in the following usage examples:: @@ -214,7 +214,7 @@ The :mod:`!abc` module also provides the following decorator: the descriptor must identify itself as abstract using :attr:`!__isabstractmethod__`. In general, this attribute should be ``True`` if any of the methods used to compose the descriptor are abstract. For - example, Python's built-in :class:`property` does the equivalent of:: + example, Python's built-in :deco:`property` does the equivalent of:: class Descriptor: ... @@ -237,14 +237,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractclassmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`classmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`classmethod`, indicating an abstract - classmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`classmethod`, indicating an abstract + classmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`classmethod` decorator + This special case is deprecated, as the :deco:`classmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -258,14 +258,14 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractstaticmethod .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`staticmethod` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`staticmethod`, indicating an abstract - staticmethod. Otherwise it is similar to :func:`abstractmethod`. + A subclass of the built-in :class:`staticmethod`, indicating an abstract + staticmethod. Otherwise it is similar to :deco:`abstractmethod`. - This special case is deprecated, as the :func:`staticmethod` decorator + This special case is deprecated, as the :deco:`staticmethod` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -278,15 +278,15 @@ The :mod:`!abc` module also supports the following legacy decorators: .. decorator:: abstractproperty - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. + .. deprecated-removed:: 3.3 3.21 + It is now possible to use :deco:`property`, :deco:`property.getter`, + :deco:`property.setter` and :deco:`property.deleter` with + :deco:`abstractmethod`, making this decorator redundant. - A subclass of the built-in :func:`property`, indicating an abstract + A subclass of the built-in :class:`property`, indicating an abstract property. - This special case is deprecated, as the :func:`property` decorator + This special case is deprecated, as the :deco:`property` decorator is now correctly identified as abstract when applied to an abstract method:: diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 40f2a6dc30460b7..5b22353ea08e647 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -235,8 +235,8 @@ Functions annotate functions that support the :attr:`~Format.STRING` format but do not have access to the code creating the annotations. - For example, this is used to implement the :attr:`~Format.STRING` for - :class:`typing.TypedDict` classes created through the functional syntax: + For example, this is used to implement the :attr:`~Format.STRING` format + for :class:`typing.TypedDict` classes created through the functional syntax: .. doctest:: @@ -510,6 +510,81 @@ annotations from the class and puts them in a separate attribute: return typ +Creating a custom callable annotate function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Custom :term:`annotate functions ` may be literal functions like those +automatically generated for functions, classes, and modules. Or, they may wish to utilise +the encapsulation provided by classes, in which case any :term:`callable` can be used as +an :term:`annotate function`. + +To provide the :attr:`~Format.VALUE`, :attr:`~Format.STRING`, or +:attr:`~Format.FORWARDREF` formats directly, an :term:`annotate function` must provide +the following attribute: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with a supported format. + +To provide the :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` format, which is used to +automatically generate :attr:`~Format.STRING` or :attr:`~Format.FORWARDREF` if they are +not supported directly, :term:`annotate functions ` must provide the +following attributes: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with + :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`. +* A :ref:`code object ` ``__code__`` containing the compiled code for the + annotate function. +* Optional: A tuple of the function's positional defaults ``__defaults__``, if the + function represented by ``__code__`` uses any positional defaults. +* Optional: A dict of the function's keyword defaults ``__kwdefaults__``, if the function + represented by ``__code__`` uses any keyword defaults. +* Optional: All other :ref:`function attributes `. + +.. code-block:: python + + class Annotate: + called_formats = [] + + def __call__(self, format=None, /, *, _self=None): + # When called with fake globals, `_self` will be the + # actual self value, and `self` will be the format. + if _self is not None: + self, format = _self, self + + self.called_formats.append(format) + if format <= 2: # VALUE or VALUE_WITH_FAKE_GLOBALS + return {"x": MyType} + raise NotImplementedError + + __code__ = __call__.__code__ + __defaults__ = (None,) + __kwdefaults__ = property(lambda self: dict(_self=self)) + + __globals__ = {} + __builtins__ = {} + __closure__ = None + +This can then be called with: + +.. code-block:: pycon + + >>> from annotationlib import call_annotate_function, Format + >>> call_annotate_function(Annotate(), format=Format.STRING) + {'x': 'MyType'} + +Or used as the annotate function for an object: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> class C: + ... pass + >>> C.__annotate__ = Annotate() + >>> get_annotations(Annotate(), format=Format.STRING) + {'x': 'MyType'} + + Limitations of the ``STRING`` format ------------------------------------ diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 8ba11b7d12d552c..b2318ddace8514b 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -442,9 +442,8 @@ is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. .. note:: - Empty lines are treated as empty strings (``''``), which are allowed as values but - not as arguments. Empty lines that are read as arguments will result in an - "unrecognized arguments" error. + Each line is treated as a single argument, so an empty line is read as an + empty string (``''``). :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -637,25 +636,22 @@ are set. .. versionadded:: 3.14 -To highlight inline code in your description or epilog text, you can use -backticks:: +To highlight inline code in your description, epilog, or argument ``help`` +text, you can use single or double backticks:: >>> parser = argparse.ArgumentParser( ... formatter_class=argparse.RawDescriptionHelpFormatter, + ... description='Run ``python -m myapp`` to start.', ... epilog='''Examples: ... `python -m myapp --verbose` - ... `python -m myapp --config settings.json` + ... ``python -m myapp --config settings.json`` ... ''') + >>> parser.add_argument('--foo', help='set the `foo` value') When colors are enabled, the text inside backticks will be displayed in a distinct color to help examples stand out. When colors are disabled, backticks are preserved as-is, which is readable in plain text. -.. note:: - - Backtick markup only applies to description and epilog text. It does not - apply to individual argument ``help`` strings. - .. versionadded:: 3.15 @@ -1055,6 +1051,10 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +Because ``nargs='*'`` gathers any supplied values into a list, an absent +positional argument yields an empty list (``[]``). Only a non-``None`` +*default* overrides this (so ``default=None`` still gives ``[]``). + For required_ arguments, the ``default`` value is ignored. For example, this applies to positional arguments with nargs_ values other than ``?`` or ``*``, or optional arguments marked as ``required=True``. @@ -1372,6 +1372,11 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +Multiple arguments may share the same ``dest``. By default, the value from the +last such argument given on the command line wins. Use ``action='append'`` to +collect values from all of them into a list instead. For conflicting *option +strings* rather than ``dest`` names, see conflict_handler_. + .. versionchanged:: 3.15 Single-dash long option now takes precedence over short options. @@ -1559,12 +1564,12 @@ it exits and prints the error along with a usage message:: >>> # invalid option >>> parser.parse_args(['--bar']) usage: PROG [-h] [--foo FOO] [bar] - PROG: error: no such option: --bar + PROG: error: unrecognized arguments: --bar >>> # wrong number of arguments >>> parser.parse_args(['spam', 'badger']) usage: PROG [-h] [--foo FOO] [bar] - PROG: error: extra arguments found: badger + PROG: error: unrecognized arguments: badger Arguments containing ``-`` @@ -1601,7 +1606,7 @@ there are no options in the parser that look like negative numbers:: >>> # negative number options present, so -2 is an option >>> parser.parse_args(['-2']) usage: PROG [-h] [-1 ONE] [foo] - PROG: error: no such option: -2 + PROG: error: unrecognized arguments: -2 >>> # negative number options present, so both -1s are options >>> parser.parse_args(['-1', '-1']) @@ -1619,6 +1624,11 @@ argument:: See also :ref:`the argparse howto on ambiguous arguments ` for more details. +.. versionchanged:: 3.14 + Negative-number matching was expanded to include numbers in scientific + notation (``-2.5e-6``), numbers containing underscores (``-1_234.5``), + and complex numbers (``-1.2e-3j``). + .. _prefix-matching: Argument abbreviations (prefix matching) @@ -1674,6 +1684,12 @@ The Namespace object Simple class used by default by :meth:`~ArgumentParser.parse_args` to create an object holding attributes and return it. + :class:`!Namespace` objects support :func:`copy.replace`, + which returns a copy of the object with the specified attributes replaced. + + .. versionchanged:: next + Added support for :func:`copy.replace`. + This class is deliberately simple, just an :class:`object` subclass with a readable string representation. If you prefer to have dict-like view of the attributes, you can use the standard Python idiom, :func:`vars`:: @@ -1780,6 +1796,11 @@ Subcommands present, and when the ``b`` command is specified, only the ``foo`` and ``baz`` attributes are present. + If a subparser defines an argument with the same ``dest`` as the parent + parser, the two share a single namespace attribute, so the parent's value + won't be retained. Users should give them distinct ``dest`` values to + keep both. + Similarly, when a help message is requested from a subparser, only the help for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each @@ -1970,7 +1991,7 @@ FileType objects run and then use the :keyword:`with`-statement to manage the files. .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. .. deprecated:: 3.14 @@ -2032,6 +2053,9 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. + Within each argument group, arguments are displayed in help output in the + order in which they are added. + .. deprecated-removed:: 3.11 3.14 Calling :meth:`add_argument_group` on an argument group now raises an exception. This nesting was never supported, often failed to work @@ -2232,6 +2256,9 @@ Customizing file parsing def convert_arg_line_to_args(self, arg_line): return arg_line.split() + Note that with this override an argument can no longer contain spaces, since + each space-separated word becomes a separate argument. + Exiting methods ^^^^^^^^^^^^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 4468edb6efa6548..c688d54318e707f 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -12,7 +12,7 @@ This module defines an object type which can compactly represent an array of basic values: characters, integers, floating-point numbers, complex numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are +:dfn:`type code`. The following type codes are defined: +-----------+--------------------+-------------------+-----------------------+-------+ @@ -22,9 +22,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'B'`` | unsigned char | int | 1 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | wchar_t | Unicode character | 2 | \(1) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(2) | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -42,35 +40,24 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'Q'`` | unsigned long long | int | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'e'`` | _Float16 | float | 2 | \(3) | +| ``'e'`` | _Float16 | float | 2 | \(2) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'f'`` | float | float | 4 | | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'d'`` | double | float | 8 | | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'F'`` | float complex | complex | 8 | \(4) | +| ``'Zf'`` | float complex | complex | 8 | \(3) | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'D'`` | double complex | complex | 16 | \(4) | +| ``'Zd'`` | double complex | complex | 16 | \(3) | +-----------+--------------------+-------------------+-----------------------+-------+ Notes: (1) - It can be 16 bits or 32 bits depending on the platform. - - .. versionchanged:: 3.9 - ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated - ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. - - .. deprecated-removed:: 3.3 3.16 - Please migrate to ``'w'`` typecode. - -(2) .. versionadded:: 3.13 -(3) +(2) The IEEE 754 binary16 "half precision" type was introduced in the 2008 revision of the `IEEE 754 standard `_. This type is not widely supported by C compilers. It's available @@ -79,8 +66,8 @@ Notes: .. versionadded:: 3.15 -(4) - Complex types (``F`` and ``D``) are available unconditionally, +(3) + Complex types (``Zf`` and ``Zd``) are available unconditionally, regardless on support for complex types (the Annex G of the C11 standard) by the C compiler. As specified in the C11 standard, each complex type is represented by a @@ -91,7 +78,7 @@ Notes: .. seealso:: The :ref:`ctypes ` and - :ref:`struct ` modules, + :ref:`struct ` modules, as well as third-party modules like `numpy `__, use similar -- but slightly different -- type codes. @@ -105,7 +92,10 @@ The module defines the following item: .. data:: typecodes - A string with all available type codes. + A tuple with all available type codes. + + .. versionchanged:: 3.15 + The type changed from :class:`str` to :class:`tuple`. The module defines the following type: @@ -131,6 +121,8 @@ The module defines the following type: :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array @@ -217,7 +209,7 @@ The module defines the following type: .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. - The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. + The array must have type code ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an array of some other type. @@ -285,7 +277,7 @@ The module defines the following type: .. method:: tounicode() - Convert the array to a Unicode string. The array must have a type ``'u'`` or ``'w'``; + Convert the array to a Unicode string. The array must have a type ``'w'``; otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to obtain a Unicode string from an array of some other type. @@ -293,7 +285,7 @@ The module defines the following type: The string representation of array objects has the form ``array(typecode, initializer)``. The *initializer* is omitted if the array is empty, otherwise it is -a Unicode string if the *typecode* is ``'u'`` or ``'w'``, otherwise it is +a Unicode string if the *typecode* is ``'w'``, otherwise it is a list of numbers. The string representation is guaranteed to be able to be converted back to an array with the same type and value using :func:`eval`, so long as the diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 9b4e7ae18348f14..e30dadc9733d37b 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -35,12 +35,14 @@ The abstract grammar is currently defined as follows: :language: asdl +.. _ast_nodes: + Node classes ------------ .. class:: AST - This is the base of all AST node classes. The actual node classes are + This is the abstract base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C module and re-exported in :mod:`!ast`. @@ -164,8 +166,16 @@ Node classes Previous versions of Python allowed the creation of AST nodes that were missing required fields. Similarly, AST node constructors allowed arbitrary keyword arguments that were set as attributes of the AST node, even if they did not - match any of the fields of the AST node. This behavior is deprecated and will - be removed in Python 3.15. + match any of the fields of the AST node. These cases now raise a :exc:`TypeError`. + +.. deprecated-removed:: 3.15 3.20 + + In the :ref:`grammar above `, the AST node classes that + correspond to production rules with variants (aka "sums") are abstract + classes. Previous versions of Python allowed for the creation of direct + instances of these abstract node classes. This behavior is deprecated and + will be removed in Python 3.20. + .. note:: The descriptions of the specific node classes displayed here @@ -270,18 +280,25 @@ Root nodes Literals ^^^^^^^^ -.. class:: Constant(value) +.. class:: Constant(value, kind) A constant value. The ``value`` attribute of the ``Constant`` literal contains the Python object it represents. The values represented can be instances of :class:`str`, :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, and the constants :data:`None` and :data:`Ellipsis`. + The ``kind`` attribute is an optional string. For string literals with a + ``u`` prefix, ``kind`` is set to ``'u'``. For all other + constants, ``kind`` is ``None``. + .. doctest:: >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) Expression( body=Constant(value=123)) + >>> print(ast.dump(ast.parse("u'hello'", mode='eval'), indent=4)) + Expression( + body=Constant(value='hello', kind='u')) .. class:: FormattedValue(value, conversion, format_spec) @@ -790,6 +807,10 @@ Comprehensions comprehensions. ``elt`` (or ``key`` and ``value``) is a single node representing the part that will be evaluated for each item. + For dictionary comprehensions using unpacking, for example + ``{**item for item in items}``, the expression to be expanded goes in + ``key`` and ``value`` is ``None``. + ``generators`` is a list of :class:`comprehension` nodes. .. doctest:: @@ -1349,7 +1370,7 @@ Control flow ``try`` blocks which are followed by ``except*`` clauses. The attributes are the same as for :class:`Try` but the :class:`ExceptHandler` nodes in ``handlers`` - are interpreted as ``except*`` blocks rather then ``except``. + are interpreted as ``except*`` blocks rather than ``except``. .. doctest:: @@ -2233,7 +2254,7 @@ and classes for traversing abstract syntax trees: In addition, if ``mode`` is ``'func_type'``, the input syntax is modified to correspond to :pep:`484` "signature type comments", - e.g. ``(str, int) -> List[str]``. + for example ``(str, int) -> List[str]``. Setting ``feature_version`` to a tuple ``(major, minor)`` will result in a "best-effort" attempt to parse using that Python version's grammar. @@ -2480,7 +2501,7 @@ and classes for traversing abstract syntax trees: node = YourTransformer().visit(node) -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None, show_empty=False) +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, color=False, indent=None, show_empty=False) Return a formatted dump of the tree in *node*. This is mainly useful for debugging purposes. If *annotate_fields* is true (by default), @@ -2490,6 +2511,10 @@ and classes for traversing abstract syntax trees: numbers and column offsets are not dumped by default. If this is wanted, *include_attributes* can be set to true. + If *color* is ``True``, the returned string is syntax highlighted using + ANSI escape sequences. + If ``False`` (the default), colored output is always disabled. + If *indent* is a non-negative integer or string, then the tree will be pretty-printed with that indent level. An indent level of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) @@ -2527,6 +2552,23 @@ and classes for traversing abstract syntax trees: .. versionchanged:: 3.15 Omit optional ``Load()`` values by default. + .. versionchanged:: 3.15 + Added the *color* parameter. + + +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + .. _ast-compiler-flags: @@ -2563,20 +2605,6 @@ effects on the compilation of a program: .. versionadded:: 3.8 -.. function:: compare(a, b, /, *, compare_attributes=False) - - Recursively compares two ASTs. - - *compare_attributes* affects whether AST attributes are considered - in the comparison. If *compare_attributes* is ``False`` (default), then - attributes are ignored. Otherwise they must all be equal. This - option is useful to check whether the ASTs are structurally equal but - differ in whitespace or similar details. Attributes include line numbers - and column offsets. - - .. versionadded:: 3.14 - - .. _ast-cli: Command-line usage @@ -2584,6 +2612,10 @@ Command-line usage .. versionadded:: 3.9 +.. versionchanged:: 3.15 + The output is now syntax highlighted by default. This can be + :ref:`controlled using environment variables `. + The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index f3409bcd2df6489..713b40d746680af 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -304,7 +304,7 @@ generator can occur in an unexpected order:: try: yield 2 finally: - await asyncio.sleep(0.1) # immitate some async work + await asyncio.sleep(0.1) # imitate some async work work_done = True diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 79c9516cda2d60b..41abb2d7d0a53eb 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -48,10 +48,10 @@ an event loop: running event loop. If there is no running event loop set, the function will return - the result of the ``get_event_loop_policy().get_event_loop()`` call. + the loop set by :func:`set_event_loop`, or raise a :exc:`RuntimeError` + if no loop has been set. - Because this function has rather complex behavior (especially - when custom event loop policies are in use), using the + Because this function has rather complex behavior, using the :func:`get_running_loop` function is preferred to :func:`get_event_loop` in coroutines and callbacks. @@ -62,13 +62,6 @@ an event loop: .. versionchanged:: 3.14 Raises a :exc:`RuntimeError` if there is no current event loop. - .. note:: - - The :mod:`!asyncio` policy system is deprecated and will be removed - in Python 3.16; from there on, this function will return the current - running event loop if present else it will return the - loop set by :func:`set_event_loop`. - .. function:: set_event_loop(loop) Set *loop* as the current event loop for the current OS thread. @@ -77,10 +70,6 @@ an event loop: Create and return a new event loop object. -Note that the behaviour of :func:`get_event_loop`, :func:`set_event_loop`, -and :func:`new_event_loop` functions can be altered by -:ref:`setting a custom event loop policy `. - .. rubric:: Contents @@ -849,7 +838,7 @@ Creating network servers *, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ - start_serving=True, cleanup_socket=True) + start_serving=True, cleanup_socket=True, mode=None) :async: Similar to :meth:`loop.create_server` but works with the @@ -864,6 +853,13 @@ Creating network servers be removed from the filesystem when the server is closed, unless the socket has been replaced after the server has been created. + If *mode* is not ``None``, the permissions of the socket file created + for *path* are changed to *mode* (as accepted by :func:`os.chmod`) + right after binding, before the server starts accepting connections, + so a connection can never be accepted while the default, + umask-derived permissions are still in effect. *mode* cannot be + combined with *sock* and is not supported for abstract Unix sockets. + See the documentation of the :meth:`loop.create_server` method for information about arguments to this method. @@ -882,6 +878,10 @@ Creating network servers Added the *cleanup_socket* parameter. + .. versionchanged:: 3.16 + + Added the *mode* parameter. + .. method:: loop.connect_accepted_socket(protocol_factory, \ sock, *, ssl=None, ssl_handshake_timeout=None, \ @@ -1156,12 +1156,21 @@ convenient. *sock* must be a non-blocking socket. + With :class:`SelectorEventLoop`, *address* does not need to be resolved: + for :const:`~socket.AF_INET` and :const:`~socket.AF_INET6` sockets, + ``sock_connect`` first checks whether *address* is already resolved by + calling :func:`socket.inet_pton`, and uses :meth:`loop.getaddrinfo` to + resolve it if it is not. + + :class:`ProactorEventLoop`, the default event loop on Windows, does not + resolve *address*. The host must already be a numeric IP address; passing + a host name raises :exc:`OSError`. Resolve the address with + :meth:`loop.getaddrinfo` first, or use :meth:`loop.create_connection`, + which resolves the address on every platform. + .. versionchanged:: 3.5.2 - ``address`` no longer needs to be resolved. ``sock_connect`` - will try to check if the *address* is already resolved by calling - :func:`socket.inet_pton`. If not, - :meth:`loop.getaddrinfo` will be used to resolve the - *address*. + With :class:`SelectorEventLoop`, ``address`` no longer needs to be + resolved. .. seealso:: @@ -1265,7 +1274,9 @@ Working with pipes *protocol_factory* must be a callable returning an :ref:`asyncio protocol ` implementation. - *pipe* is a :term:`file-like object `. + *pipe* is a :term:`file-like object `. See + :ref:`Supported pipe objects ` for the objects + supported as *pipe*. Return pair ``(transport, protocol)``, where *transport* supports the :class:`ReadTransport` interface and *protocol* is an object @@ -1282,7 +1293,9 @@ Working with pipes *protocol_factory* must be a callable returning an :ref:`asyncio protocol ` implementation. - *pipe* is :term:`file-like object `. + *pipe* is a :term:`file-like object `. See + :ref:`Supported pipe objects ` for the objects + supported as *pipe*. Return pair ``(transport, protocol)``, where *transport* supports :class:`WriteTransport` interface and *protocol* is an object @@ -1291,6 +1304,33 @@ Working with pipes With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. +.. _asyncio-pipe-objects: + +.. rubric:: Supported pipe objects + +These methods only work with objects the operating system can poll for +readiness or perform overlapped I/O on. Regular files on disk are **not** +supported on any platform. There is no asynchronous file I/O in asyncio; +use :meth:`loop.run_in_executor` to read and write regular files without +blocking the event loop. + +On Unix, with :class:`SelectorEventLoop`, *pipe* must wrap one of the +following: + +* a pipe, such as an end of an :func:`os.pipe` pair or a FIFO created with + :func:`os.mkfifo`; +* a socket; +* a character device, such as a terminal. + +On Windows, where only :class:`ProactorEventLoop` implements these methods, +*pipe* must wrap a handle opened for overlapped I/O (that is, created with the +``FILE_FLAG_OVERLAPPED`` flag), since the handle has to be associated with an +I/O completion port. Handles that were not opened for overlapped I/O are +rejected. In particular, the standard streams (:data:`sys.stdin`, +:data:`sys.stdout` and :data:`sys.stderr`), console handles, and the pipes +created by :func:`os.pipe` are **not** opened for overlapped I/O and therefore +cannot be used with these methods. + .. note:: :class:`SelectorEventLoop` does not support the above methods on @@ -1844,6 +1884,12 @@ Do not instantiate the :class:`Server` class directly. Wait until the :meth:`close` method completes and all active connections have finished. + .. versionchanged:: 3.12 + ``wait_closed()`` now waits until the server is closed and + all active connections have finished. Previously, it returned + immediately if the server was already closed, even if + connections were still active. + .. attribute:: sockets List of socket-like objects, ``asyncio.trsock.TransportSocket``, which diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 4b69e569523c58b..195d99123dbd367 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -101,6 +101,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -196,6 +198,10 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + The optional string argument *msg* is passed as the argument to the + :exc:`CancelledError` exception raised when a cancelled Future + is awaited. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 5f642a32bf75c24..1a8cdcf6a9e8cba 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -4,7 +4,7 @@ .. _asyncio-graph: ======================== -Call Graph Introspection +Call graph introspection ======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -17,6 +17,12 @@ a suspended *future*. These utilities and the underlying machinery can be used from within a Python program or by external profilers and debuggers. +.. seealso:: + + :ref:`asyncio-introspection-tools` + Command-line tools for inspecting tasks in another running Python + process. + .. versionadded:: 3.14 diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst index f5af888f31f1863..faf67209a2b0ef5 100644 --- a/Doc/library/asyncio-llapi-index.rst +++ b/Doc/library/asyncio-llapi-index.rst @@ -19,10 +19,10 @@ Obtaining the Event Loop - The **preferred** function to get the running event loop. * - :func:`asyncio.get_event_loop` - - Get an event loop instance (running or current via the current policy). + - Get the running event loop or the event loop set for the current thread. * - :func:`asyncio.set_event_loop` - - Set the event loop as current via the current policy. + - Set the event loop for the current thread. * - :func:`asyncio.new_event_loop` - Create a new event loop. @@ -497,27 +497,3 @@ Protocol classes can implement the following **callback methods**: - Called when the child process has exited. It can be called before :meth:`~SubprocessProtocol.pipe_data_received` and :meth:`~SubprocessProtocol.pipe_connection_lost` methods. - - -Event Loop Policies -=================== - -Policies is a low-level mechanism to alter the behavior of -functions like :func:`asyncio.get_event_loop`. See also -the main :ref:`policies section ` for more -details. - - -.. rubric:: Accessing Policies -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`asyncio.get_event_loop_policy` - - Return the current process-wide policy. - - * - :meth:`asyncio.set_event_loop_policy` - - Set a new process-wide policy. - - * - :class:`AbstractEventLoopPolicy` - - Base class for policy objects. diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index a2a3114ad6e4c5d..0b7c17a2c584795 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -19,6 +19,11 @@ All Platforms * :meth:`loop.add_reader` and :meth:`loop.add_writer` cannot be used to monitor file I/O. +* :meth:`loop.connect_read_pipe` and :meth:`loop.connect_write_pipe` + cannot be used with regular files. See :ref:`Supported pipe objects + ` for the objects that are accepted on each + platform. + Windows ======= @@ -62,6 +67,10 @@ All event loops on Windows do not support the following methods: * The :meth:`loop.add_reader` and :meth:`loop.add_writer` methods are not supported. +* :meth:`loop.connect_read_pipe` and :meth:`loop.connect_write_pipe` only + accept a handle opened for overlapped I/O. + See :ref:`Supported pipe objects ` for which objects are supported. + The resolution of the monotonic clock on Windows is usually around 15.6 milliseconds. The best resolution is 0.5 milliseconds. The resolution depends on the hardware (availability of `HPET diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst deleted file mode 100644 index 57f964912dd6ea2..000000000000000 --- a/Doc/library/asyncio-policy.rst +++ /dev/null @@ -1,173 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-policies: - -======== -Policies -======== - -.. warning:: - - Policies are deprecated and will be removed in Python 3.16. - Users are encouraged to use the :func:`asyncio.run` function - or the :class:`asyncio.Runner` with *loop_factory* to use - the desired loop implementation. - - -An event loop policy is a global object -used to get and set the current :ref:`event loop `, -as well as create new event loops. -The default policy can be :ref:`replaced ` with -:ref:`built-in alternatives ` -to use different event loop implementations, -or substituted by a :ref:`custom policy ` -that can override these behaviors. - -The :ref:`policy object ` -gets and sets a separate event loop per *context*. -This is per-thread by default, -though custom policies could define *context* differently. - -Custom event loop policies can control the behavior of -:func:`get_event_loop`, :func:`set_event_loop`, and :func:`new_event_loop`. - -Policy objects should implement the APIs defined -in the :class:`AbstractEventLoopPolicy` abstract base class. - - -.. _asyncio-policy-get-set: - -Getting and Setting the Policy -============================== - -The following functions can be used to get and set the policy -for the current process: - -.. function:: get_event_loop_policy() - - Return the current process-wide policy. - - .. deprecated:: 3.14 - The :func:`get_event_loop_policy` function is deprecated and - will be removed in Python 3.16. - -.. function:: set_event_loop_policy(policy) - - Set the current process-wide policy to *policy*. - - If *policy* is set to ``None``, the default policy is restored. - - .. deprecated:: 3.14 - The :func:`set_event_loop_policy` function is deprecated and - will be removed in Python 3.16. - - -.. _asyncio-policy-objects: - -Policy Objects -============== - -The abstract event loop policy base class is defined as follows: - -.. class:: AbstractEventLoopPolicy - - An abstract base class for asyncio policies. - - .. method:: get_event_loop() - - Get the event loop for the current context. - - Return an event loop object implementing the - :class:`AbstractEventLoop` interface. - - This method should never return ``None``. - - .. versionchanged:: 3.6 - - .. method:: set_event_loop(loop) - - Set the event loop for the current context to *loop*. - - .. method:: new_event_loop() - - Create and return a new event loop object. - - This method should never return ``None``. - - .. deprecated:: 3.14 - The :class:`AbstractEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. _asyncio-policy-builtin: - -asyncio ships with the following built-in policies: - - -.. class:: DefaultEventLoopPolicy - - The default asyncio policy. Uses :class:`SelectorEventLoop` - on Unix and :class:`ProactorEventLoop` on Windows. - - There is no need to install the default policy manually. asyncio - is configured to use the default policy automatically. - - .. versionchanged:: 3.8 - - On Windows, :class:`ProactorEventLoop` is now used by default. - - .. versionchanged:: 3.14 - The :meth:`get_event_loop` method of the default asyncio policy now - raises a :exc:`RuntimeError` if there is no set event loop. - - .. deprecated:: 3.14 - The :class:`DefaultEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. class:: WindowsSelectorEventLoopPolicy - - An alternative event loop policy that uses the - :class:`SelectorEventLoop` event loop implementation. - - .. availability:: Windows. - - .. deprecated:: 3.14 - The :class:`WindowsSelectorEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. class:: WindowsProactorEventLoopPolicy - - An alternative event loop policy that uses the - :class:`ProactorEventLoop` event loop implementation. - - .. availability:: Windows. - - .. deprecated:: 3.14 - The :class:`WindowsProactorEventLoopPolicy` class is deprecated and - will be removed in Python 3.16. - - -.. _asyncio-custom-policies: - -Custom Policies -=============== - -To implement a new event loop policy, it is recommended to subclass -:class:`DefaultEventLoopPolicy` and override the methods for which -custom behavior is wanted, e.g.:: - - class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): - - def get_event_loop(self): - """Get the event loop. - - This may be None or an instance of EventLoop. - """ - loop = super().get_event_loop() - # Do something with loop ... - return loop - - asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index a9735ae80652df6..5e3f531f4e482e4 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -71,6 +71,8 @@ Queue Return an item if one is immediately available, else raise :exc:`QueueEmpty`. + Raises :exc:`QueueShutDown` if the queue has been shut down and is empty. + .. method:: join() :async: @@ -96,6 +98,8 @@ Queue If no free slot is immediately available, raise :exc:`QueueFull`. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: qsize() Return the number of items in the queue. @@ -188,8 +192,9 @@ Exceptions .. exception:: QueueShutDown - Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is - called on a queue which has been shut down. + Exception raised when :meth:`~Queue.put`, :meth:`~Queue.put_nowait`, + :meth:`~Queue.get` or :meth:`~Queue.get_nowait` is called + on a queue which has been shut down. .. versionadded:: 3.13 diff --git a/Doc/library/asyncio-runner.rst b/Doc/library/asyncio-runner.rst index 48d78099fd3ce7e..cd125b30703c796 100644 --- a/Doc/library/asyncio-runner.rst +++ b/Doc/library/asyncio-runner.rst @@ -43,9 +43,7 @@ Running an asyncio Program otherwise :func:`asyncio.new_event_loop` is used. The loop is closed at the end. This function should be used as a main entry point for asyncio programs, and should ideally only be called once. It is recommended to use - *loop_factory* to configure the event loop instead of policies. - Passing :class:`asyncio.EventLoop` allows running asyncio without the - policy system. + *loop_factory* to configure the event loop. The executor is given a timeout duration of 5 minutes to shutdown. If the executor hasn't finished within that duration, a warning is @@ -76,12 +74,6 @@ Running an asyncio Program *coro* can be any awaitable object. - .. note:: - - The :mod:`!asyncio` policy system is deprecated and will be removed - in Python 3.16; from there on, an explicit *loop_factory* is needed - to configure the event loop. - Runner context manager ====================== diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index 05445219510ca54..d75846163edfddb 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -49,7 +49,7 @@ and work with streams: .. function:: open_connection(host=None, port=None, *, \ - limit=None, ssl=None, family=0, proto=0, \ + limit=65536, ssl=None, family=0, proto=0, \ flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None, \ ssl_shutdown_timeout=None, \ @@ -89,7 +89,7 @@ and work with streams: .. function:: start_server(client_connected_cb, host=None, \ - port=None, *, limit=None, \ + port=None, *, limit=65536, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ @@ -137,7 +137,7 @@ and work with streams: .. rubric:: Unix Sockets -.. function:: open_unix_connection(path=None, *, limit=None, \ +.. function:: open_unix_connection(path=None, *, limit=65536, \ ssl=None, sock=None, server_hostname=None, \ ssl_handshake_timeout=None, ssl_shutdown_timeout=None) :async: @@ -169,9 +169,10 @@ and work with streams: .. function:: start_unix_server(client_connected_cb, path=None, \ - *, limit=None, sock=None, backlog=100, ssl=None, \ + *, limit=65536, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True, cleanup_socket=True) + ssl_shutdown_timeout=None, start_serving=True, \ + cleanup_socket=True, mode=None) :async: Start a Unix socket server. @@ -182,6 +183,9 @@ and work with streams: be removed from the filesystem when the server is closed, unless the socket has been replaced after the server has been created. + If *mode* is not ``None``, the permissions of the Unix socket file + are set to *mode* before the server starts accepting connections. + See also the documentation of :meth:`loop.create_unix_server`. .. note:: @@ -205,6 +209,9 @@ and work with streams: .. versionchanged:: 3.13 Added the *cleanup_socket* parameter. + .. versionchanged:: 3.16 + Added the *mode* parameter. + StreamReader ============ @@ -382,6 +389,16 @@ StreamWriter be resumed. When there is nothing to wait for, the :meth:`drain` returns immediately. + .. note:: + + When the write buffer is below the high watermark, + :meth:`drain` returns immediately without yielding to + the event loop. As a result, code which repeatedly calls + ``write()`` followed by ``await drain()`` may prevent other + tasks from running. To prevent blocking behavior, yield + to the event loop explicitly with ``await asyncio.sleep(0)`` + (see :func:`asyncio.sleep`). + .. method:: start_tls(sslcontext, *, server_hostname=None, \ ssl_handshake_timeout=None, ssl_shutdown_timeout=None) :async: diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index a6514649bf9a0a8..2fdc10db5bfd51e 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -62,7 +62,7 @@ Creating Subprocesses ===================== .. function:: create_subprocess_exec(program, *args, stdin=None, \ - stdout=None, stderr=None, limit=None, **kwds) + stdout=None, stderr=None, limit=65536, **kwds) :async: Create a subprocess. @@ -84,7 +84,7 @@ Creating Subprocesses .. function:: create_subprocess_shell(cmd, stdin=None, \ - stdout=None, stderr=None, limit=None, **kwds) + stdout=None, stderr=None, limit=65536, **kwds) :async: Run the *cmd* shell command. @@ -240,10 +240,34 @@ their completion. Note, that the data read is buffered in memory, so do not use this method if the data size is large or unlimited. + If this coroutine is cancelled (for example, when a timeout is + set with :func:`~asyncio.wait_for`), the output that was already + read is not lost: call :meth:`!communicate` again to read the + remaining output and get the complete data:: + + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(), timeout=5.0) + except TimeoutError: + proc.kill() + stdout, stderr = await proc.communicate() + + Passing *input* after a previous :meth:`!communicate` call was + cancelled raises :exc:`ValueError`; pass ``input=None`` to + continue the communication, the original *input* is used. + .. versionchanged:: 3.12 *stdin* gets closed when ``input=None`` too. + .. versionchanged:: next + + If :meth:`!communicate` is cancelled, the output that was + already read is now preserved and returned by a subsequent + :meth:`!communicate` call. Passing *input* to a + :meth:`!communicate` call following a cancelled one now raises + :exc:`ValueError`. + .. method:: send_signal(signal) Sends the signal *signal* to the child process. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 4e60eee44290afc..1a13ad387f89b22 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -288,6 +288,17 @@ Creating tasks # completion: task.add_done_callback(background_tasks.discard) + Note that this approach never awaits the tasks, so if a task + fails, its exception is never retrieved and asyncio logs a + "Task exception was never retrieved" message when the task is + garbage collected. To avoid this, use :class:`asyncio.TaskGroup` + which keeps a strong reference to each task, awaits them and + propagates their exceptions:: + + async with asyncio.TaskGroup() as tg: + for i in range(10): + tg.create_task(some_coro(param=i)) + .. versionadded:: 3.7 .. versionchanged:: 3.8 @@ -345,7 +356,7 @@ and reliable way to wait for all tasks in the group to finish. The signature matches that of :func:`asyncio.create_task`. If the task group is inactive (e.g. not yet entered, already finished, or in the process of shutting down), - we will close the given ``coro``. + we will close the given ``coro`` and raise :exc:`RuntimeError`. .. versionchanged:: 3.13 @@ -355,6 +366,34 @@ and reliable way to wait for all tasks in the group to finish. Passes on all *kwargs* to :meth:`loop.create_task` + .. method:: cancel() + + Cancel the task group. This is a non-exceptional, early exit of the + task group's lifetime -- useful once the group's goal has been met or + its services no longer needed. + + :meth:`~asyncio.Task.cancel` will be called on any tasks in the group that + aren't yet done, as well as the parent (body) of the group. The task group + context manager will exit *without* :exc:`asyncio.CancelledError` being raised. + + If :meth:`cancel` is called before entering the task group, the group will be + cancelled upon entry. This is useful for patterns where one piece of + code passes an unused :class:`asyncio.TaskGroup` instance to another in order to have + the ability to cancel anything run within the group. + + :meth:`cancel` is idempotent and may be called after the task group has + already exited. + + Some ways to use :meth:`cancel`: + + * call it from the task group body based on some condition or event + * pass the task group instance to child tasks via :meth:`create_task`, allowing a child + task to conditionally cancel the entire group + * pass the task group instance or bound :meth:`cancel` method to some other task *before* + opening the task group, allowing remote cancellation + + .. versionadded:: 3.15 + Example:: async def main(): @@ -363,116 +402,94 @@ Example:: task2 = tg.create_task(another_coro(...)) print(f"Both tasks have completed now: {task1.result()}, {task2.result()}") -The ``async with`` statement will wait for all tasks in the group to finish. -While waiting, new tasks may still be added to the group -(for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). -Once the last task has finished and the ``async with`` block is exited, -no new tasks may be added to the group. - -The first time any of the tasks belonging to the group fails -with an exception other than :exc:`asyncio.CancelledError`, -the remaining tasks in the group are cancelled. -No further tasks can then be added to the group. -At this point, if the body of the ``async with`` statement is still active -(i.e., :meth:`~object.__aexit__` hasn't been called yet), -the task directly containing the ``async with`` statement is also cancelled. -The resulting :exc:`asyncio.CancelledError` will interrupt an ``await``, -but it will not bubble out of the containing ``async with`` statement. - -Once all tasks have finished, if any tasks have failed -with an exception other than :exc:`asyncio.CancelledError`, -those exceptions are combined in an -:exc:`ExceptionGroup` or :exc:`BaseExceptionGroup` -(as appropriate; see their documentation) -which is then raised. - -Two base exceptions are treated specially: -If any task fails with :exc:`KeyboardInterrupt` or :exc:`SystemExit`, -the task group still cancels the remaining tasks and waits for them, -but then the initial :exc:`KeyboardInterrupt` or :exc:`SystemExit` -is re-raised instead of :exc:`ExceptionGroup` or :exc:`BaseExceptionGroup`. - -If the body of the ``async with`` statement exits with an exception -(so :meth:`~object.__aexit__` is called with an exception set), -this is treated the same as if one of the tasks failed: -the remaining tasks are cancelled and then waited for, -and non-cancellation exceptions are grouped into an -exception group and raised. -The exception passed into :meth:`~object.__aexit__`, -unless it is :exc:`asyncio.CancelledError`, -is also included in the exception group. -The same special case is made for -:exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. - -Task groups are careful not to mix up the internal cancellation used to -"wake up" their :meth:`~object.__aexit__` with cancellation requests -for the task in which they are running made by other parties. +A few points to keep in mind when using task groups: + +* The ``async with`` statement will wait for all tasks in the group + to finish. While waiting, new tasks may still be added to the group + (for example, by passing ``tg`` into one of the coroutines and + calling ``tg.create_task()`` in that coroutine); once the last task + has finished and the ``async with`` block is exited, no new tasks + may be added. + +* Termination of the entire task group may be requested with + ``tg.cancel()``, based on some condition. + +* If the group is shut down (e.g. because another task failed) before + a newly created task has started running, the task is cancelled + without its coroutine executing at all, not even to its first + ``await``. To guarantee that the coroutine starts, create the task + eagerly with ``eager_start=True`` or use + :func:`asyncio.eager_task_factory`. For example:: + + async def job(): + print("job started") # never printed + try: + await asyncio.sleep(1) + finally: + print("job cleaned up") # never printed + + async def main(): + async with asyncio.TaskGroup() as tg: + tg.create_task(job()) + raise RuntimeError # shuts down the group before job() runs + + With ``tg.create_task(job(), eager_start=True)``, ``job()`` runs up + to the ``await``, is cancelled there, and both messages are printed. + +When any of the tasks belonging to the group fails with an exception +other than :exc:`asyncio.CancelledError` (or the body of the +``async with`` statement exits with an exception, which is treated +the same way): + +* The first time this happens, the remaining tasks in the group are + cancelled and then waited for, and no further tasks can be added to + the group. If the body of the ``async with`` statement is still + active (i.e., :meth:`~object.__aexit__` hasn't been called yet), + the task directly containing the ``async with`` statement is also + cancelled. The resulting :exc:`asyncio.CancelledError` will + interrupt an ``await``, but it will not bubble out of the containing + ``async with`` statement. + +* Once all tasks have finished, the non-cancellation exceptions -- + including the exception the body exited with, unless it is + :exc:`asyncio.CancelledError` -- are combined in an + :exc:`ExceptionGroup` or :exc:`BaseExceptionGroup` + (as appropriate; see their documentation), which is then raised. + +* Some exceptions are treated specially: if any task fails with + :exc:`KeyboardInterrupt` or :exc:`SystemExit`, the task group still + cancels the remaining tasks and waits for them, but then the initial + :exc:`KeyboardInterrupt` or :exc:`SystemExit` is re-raised instead + of :exc:`ExceptionGroup` or :exc:`BaseExceptionGroup`. + Additionally, if the body of the ``async with`` statement raises + :exc:`GeneratorExit` and none of the other tasks raise exceptions + that would be reported, the :exc:`GeneratorExit` is re-raised. + +Task groups are careful not to mix up the internal cancellation used +to "wake up" their :meth:`~object.__aexit__` with cancellation +requests for the task in which they are running made by other parties. In particular, when one task group is syntactically nested in another, -and both experience an exception in one of their child tasks simultaneously, -the inner task group will process its exceptions, and then the outer task group -will receive another cancellation and process its own exceptions. +and both experience an exception in one of their child tasks +simultaneously, the inner task group will process its exceptions, and +then the outer task group will receive another cancellation and +process its own exceptions. In the case where a task group is cancelled externally and also must raise an :exc:`ExceptionGroup`, it will call the parent task's -:meth:`~asyncio.Task.cancel` method. This ensures that a +:meth:`~asyncio.Task.cancel` method. This ensures that a :exc:`asyncio.CancelledError` will be raised at the next -:keyword:`await`, so the cancellation is not lost. - -Task groups preserve the cancellation count -reported by :meth:`asyncio.Task.cancelling`. +:keyword:`await`, so the cancellation is not lost. Task groups also +preserve the cancellation count reported by +:meth:`asyncio.Task.cancelling`. .. versionchanged:: 3.13 Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a task group ------------------------- - -While terminating a task group is not natively supported by the standard -library, termination can be achieved by adding an exception-raising task -to the task group and ignoring the raised exception: - -.. code-block:: python - - import asyncio - from asyncio import TaskGroup - - class TerminateTaskGroup(Exception): - """Exception raised to terminate a task group.""" - - async def force_terminate_task_group(): - """Used to force termination of a task group.""" - raise TerminateTaskGroup() - - async def job(task_id, sleep_time): - print(f'Task {task_id}: start') - await asyncio.sleep(sleep_time) - print(f'Task {task_id}: done') - - async def main(): - try: - async with TaskGroup() as group: - # spawn some tasks - group.create_task(job(1, 0.5)) - group.create_task(job(2, 1.5)) - # sleep for 1 second - await asyncio.sleep(1) - # add an exception-raising task to force the group to terminate - group.create_task(force_terminate_task_group()) - except* TerminateTaskGroup: - pass +.. versionchanged:: 3.15 - asyncio.run(main()) - -Expected output: - -.. code-block:: text - - Task 1: start - Task 2: start - Task 1: done + Addition of the special case for :exc:`GeneratorExit`. Sleeping ======== @@ -667,12 +684,12 @@ Eager task factory Shielding from cancellation =========================== -.. awaitablefunction:: shield(aw) +.. awaitablefunction:: shield(arg) Protect an :ref:`awaitable object ` from being :meth:`cancelled `. - If *aw* is a coroutine it is automatically scheduled as a Task. + If *arg* is a coroutine it is automatically scheduled as a Task. The statement:: @@ -713,7 +730,7 @@ Shielding from cancellation Removed the *loop* parameter. .. deprecated:: 3.10 - Deprecation warning is emitted if *aw* is not Future-like object + Deprecation warning is emitted if *arg* is not Future-like object and there is no running event loop. @@ -836,29 +853,25 @@ Timeouts .. versionadded:: 3.11 -.. function:: wait_for(aw, timeout) +.. function:: wait_for(fut, timeout) :async: - Wait for the *aw* :ref:`awaitable ` + Wait for the *fut* :ref:`awaitable ` to complete with a timeout. - If *aw* is a coroutine it is automatically scheduled as a Task. - *timeout* can either be ``None`` or a float or int number of seconds to wait for. If *timeout* is ``None``, block until the future completes. - If a timeout occurs, it cancels the task and raises - :exc:`TimeoutError`. + If a timeout occurs, it cancels *fut* and raises :exc:`TimeoutError`. - To avoid the task :meth:`cancellation `, - wrap it in :func:`shield`. + To prevent *fut* from being cancelled, wrap it in :func:`shield`. The function will wait until the future is actually cancelled, so the total wait time may exceed the *timeout*. If an exception happens during cancellation, it is propagated. - If the wait is cancelled, the future *aw* is also cancelled. + If the wait is cancelled, the future *fut* is also cancelled. .. _asyncio_example_waitfor: @@ -883,8 +896,8 @@ Timeouts # timeout! .. versionchanged:: 3.7 - When *aw* is cancelled due to a timeout, ``wait_for`` waits - for *aw* to be cancelled. Previously, it raised + When *fut* is cancelled due to a timeout, ``wait_for`` waits + for *fut* to be cancelled. Previously, it raised :exc:`TimeoutError` immediately. .. versionchanged:: 3.10 @@ -893,24 +906,28 @@ Timeouts .. versionchanged:: 3.11 Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. + .. versionchanged:: 3.12 + Implemented using :func:`asyncio.timeout`, a coroutine passed as *fut* + is no longer wrapped in a :class:`Task` when *timeout* is positive. + Waiting primitives ================== -.. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) +.. function:: wait(fs, *, timeout=None, return_when=ALL_COMPLETED) :async: - Run :class:`~asyncio.Future` and :class:`~asyncio.Task` instances in the *aws* + Run :class:`~asyncio.Future` and :class:`~asyncio.Task` instances in the *fs* iterable concurrently and block until the condition specified by *return_when*. - The *aws* iterable must not be empty. + The *fs* iterable must not be empty. Returns two sets of Tasks/Futures: ``(done, pending)``. Usage:: - done, pending = await asyncio.wait(aws) + done, pending = await asyncio.wait(fs) *timeout* (a float or int), if specified, can be used to control the maximum number of seconds to wait before returning. @@ -942,6 +959,9 @@ Waiting primitives Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. + If ``wait()`` is cancelled, the futures in *fs* are not cancelled + and continue to run. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -952,9 +972,9 @@ Waiting primitives Added support for generators yielding tasks. -.. function:: as_completed(aws, *, timeout=None) +.. function:: as_completed(fs, *, timeout=None) - Run :ref:`awaitable objects ` in the *aws* iterable + Run :ref:`awaitable objects ` in the *fs* iterable concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. @@ -999,11 +1019,15 @@ Waiting primitives are done. This is raised by the ``async for`` loop during asynchronous iteration or by the coroutines yielded during plain iteration. + ``as_completed()`` does not cancel the tasks running the supplied + awaitables: if a timeout occurs or the iteration is cancelled, the + remaining tasks continue to run. + .. versionchanged:: 3.10 Removed the *loop* parameter. .. deprecated:: 3.10 - Deprecation warning is emitted if not all awaitable objects in the *aws* + Deprecation warning is emitted if not all awaitable objects in the *fs* iterable are Future-like objects and there is no running event loop. .. versionchanged:: 3.12 @@ -1248,6 +1272,9 @@ Task object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/asyncio-threading.rst b/Doc/library/asyncio-threading.rst new file mode 100644 index 000000000000000..526901a2e7eb206 --- /dev/null +++ b/Doc/library/asyncio-threading.rst @@ -0,0 +1,154 @@ +.. currentmodule:: asyncio + +.. _asyncio-threading: + +asyncio and free-threaded Python +================================ + +asyncio uses an event loop as a scheduler to enable highly efficient +concurrency by switching between tasks to allow non-blocking I/O +operations. This results in better performance for I/O-bound use +cases. It also allows off-loading CPU-bound work to a thread or +process pool, but that is still limited by the :term:`global +interpreter lock` in CPython. + +However, in :ref:`free-threaded Python `, +the GIL is disabled and Python can run true multi-threaded code. This +means that asyncio can now take advantage of multiple CPU cores without +the limitations imposed by the GIL. + +Since Python 3.14, asyncio has first-class support for free-threaded +Python, and the implementation of asyncio is safe to use in a +multi-threaded environment. + +A single event loop on one core can handle many connections +concurrently, but the Python code that runs to handle each one still +executes serially. Once requests involve a non-trivial amount of +per-request computation, that handling becomes the bottleneck, and a +single core can no longer keep up. Combining asyncio with threads is +most useful here: by running an event loop per thread, the handling of +different requests can run in parallel across multiple CPU cores. It is +also useful when you need to run blocking or CPU-bound code from an +asyncio application. + + +.. seealso:: + + `Scaling asyncio on Free-Threaded Python + `__, + a blog post by Kumar Aditya which explains the internal changes + that make asyncio safe and efficient under free-threaded Python, + together with benchmarks of the resulting improvements. + + +Thread safety considerations +---------------------------- + +While asyncio is designed to be thread-safe in a free-threaded Python +environment, there are still some considerations to keep in mind when +using asyncio with threads: + +1. **Event loop**: Each thread should have its own event loop which + should not be shared across threads. This ensures that the event loop + can manage its own tasks and callbacks without interference from + other threads. + +2. **Task management**: Tasks and futures created in one thread should + not be awaited or manipulated from another thread. + +3. **Thread-safe APIs**: When interacting with asyncio from multiple + threads, it's important to use thread-safe APIs provided by asyncio, + such as :func:`asyncio.run_coroutine_threadsafe` for submitting + coroutines to an event loop from another thread. If you need to + call a callback from a different thread, you can use + :meth:`loop.call_soon_threadsafe` to schedule it safely. + +4. **Synchronization**: The synchronization primitives provided by + asyncio (like :class:`asyncio.Lock` and :class:`asyncio.Event`) + are not designed to be used across threads. If you need to + synchronize between threads, you should use the synchronization + primitives from the :mod:`threading` module instead. + + +Using asyncio with threads +-------------------------- + +asyncio supports running one event loop per thread, which allows you to +take advantage of multiple CPU cores in a free-threaded Python +environment. Each thread can run its own event loop, and tasks can be +scheduled on those loops independently. + +Here's an example of how to use asyncio with threads:: + + import asyncio + import threading + + async def worker(name: str) -> None: + print(f"Worker {name} starting") + await asyncio.sleep(1) + print(f"Worker {name} done") + + def run_loop(name: str) -> None: + asyncio.run(worker(name)) + + threads = [ + threading.Thread(target=run_loop, args=(f"T{i}",)) + for i in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + +In this example, each thread creates its own event loop with +:func:`asyncio.run` and runs a coroutine on it. The threads execute +concurrently, and in a free-threaded build they can run on separate +CPU cores in parallel. + + +Producer/consumer across threads +-------------------------------- + +When a regular (non-asyncio) thread needs to hand work to an asyncio +event loop running in another thread, use a thread-safe primitive such +as :class:`queue.Queue` rather than :class:`asyncio.Queue`, which is +only safe within a single event loop.:: + + import asyncio + import queue + import threading + + def producer(q: queue.Queue[int]) -> None: + for i in range(5): + print(f"Producing {i}") + q.put(i) + q.shutdown() + + async def consumer(q: queue.Queue[int]) -> None: + while True: + try: + item = q.get_nowait() + except queue.Empty: + await asyncio.sleep(0.1) + continue + except queue.ShutDown: + break + print(f"Consumed {item}") + await asyncio.sleep(item) + + q: queue.Queue[int] = queue.Queue() + consumer_thread = threading.Thread( + target=lambda: asyncio.run(consumer(q)) + ) + consumer_thread.start() + producer(q) + consumer_thread.join() + +The producer runs on the main thread while the consumer runs inside an +event loop on its own thread, yet they communicate safely through +``queue.Queue``. When the queue is empty the consumer sleeps briefly +and tries again. When the producer is done it calls +:meth:`~queue.Queue.shutdown`, which causes subsequent +:meth:`~queue.Queue.get_nowait` calls to raise :exc:`queue.ShutDown` +so the consumer can exit cleanly. + diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst new file mode 100644 index 000000000000000..a6353f0618c9416 --- /dev/null +++ b/Doc/library/asyncio-tools.rst @@ -0,0 +1,157 @@ +.. currentmodule:: asyncio + +.. _asyncio-introspection-tools: + +================================ +Command-line introspection tools +================================ + +**Source code:** :source:`Lib/asyncio/tools.py` + +------------------------------------- + +The :mod:`!asyncio` module can be invoked as a script via ``python -m +asyncio`` to inspect the task graph of another running Python process without +modifying it or restarting it. The :mod:`!asyncio.tools` submodule implements +this interface. + +The following commands inspect the process identified by ``PID``: + +.. code-block:: shell-session + + $ python -m asyncio pstree [--retries N] PID + $ python -m asyncio ps [--retries N] PID + +The commands read the target process state without executing any code in it. +They are only available on supported platforms and may require permission to +inspect another process. See the :ref:`permission requirements ` for details. + +.. seealso:: + + :ref:`asyncio-graph` + Programmatic APIs for inspecting the async call graph of a task or + future in the current process. + +The command examples below use this program, which creates a task hierarchy +suitable for inspection and prints its process ID: + +.. code-block:: python + :caption: example.py + + import asyncio + import os + + async def play(track): + await asyncio.sleep(3600) + print(f"🎵 Finished: {track}") + + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) + + async def main(): + print(f"PID: {os.getpid()}") + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), + name="Sundowning", + ) + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), + name="TMBTE", + ) + + asyncio.run(main()) + +Run the program in one terminal and leave it running: + +.. code-block:: shell-session + + $ python example.py + PID: 12345 + +Then pass the printed process ID to the commands from another terminal. +Thread IDs, task IDs, file paths, and line numbers vary between runs and +source layouts. + +.. versionadded:: 3.14 + +Command-line options +==================== + +.. option:: pstree PID + + Display task and coroutine relationships as a tree. Each task is shown + with its full coroutine stack, nested under the task (if any) that is + awaiting it. This subcommand is useful for quickly identifying which branch + of a task hierarchy is blocked and where in its coroutine stack execution + has paused: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + └── (T) Task-1 + └── main example.py:12 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) Sundowning + │ └── album example.py:7 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 + │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) TMBTE + └── album example.py:7 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 + + If the await graph contains a cycle, ``pstree`` reports an error instead + of printing a tree. A cycle in the await graph is unusual and typically + indicates a programming error: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 + +.. option:: ps PID + + Display a flat table of all pending tasks in the process *PID*. Each row + shows the event-loop thread ID, task ID and name, coroutine stack, and the + awaiting task's stack, name, and ID, if any. + + This subcommand prints all tasks regardless of whether the await graph + contains cycles: + + .. code-block:: shell-session + + $ python -m asyncio ps 12345 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 18445801 0x10a456060 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 18445801 0x10a439f60 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a439d70 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a2a3a80 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2a38a0 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2d7150 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + 18445801 0x10a6bdaa0 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + +.. option:: --retries N + + Retry failed attempts to inspect the target process up to *N* times. This + can help when the target process changes while its state is being read. + + .. versionadded:: 3.15 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 0f72e31dee5f1d1..956b00f0873a0d1 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -47,6 +47,13 @@ asyncio provides a set of **high-level** APIs to: * :ref:`synchronize ` concurrent code; +For **introspection**, asyncio provides APIs and tools for: + +* inspecting the :ref:`async call graph ` of tasks and futures; + +* inspecting tasks in another running Python process with + :ref:`command-line tools `; + Additionally, there are **low-level** APIs for *library and framework developers* to: @@ -108,7 +115,13 @@ for full functionality and the latest features. asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + +.. toctree:: + :caption: Introspection APIs + :maxdepth: 1 + asyncio-graph.rst + asyncio-tools.rst .. toctree:: :caption: Low-level APIs @@ -117,7 +130,6 @@ for full functionality and the latest features. asyncio-eventloop.rst asyncio-future.rst asyncio-protocol.rst - asyncio-policy.rst asyncio-platforms.rst asyncio-extending.rst @@ -128,6 +140,7 @@ for full functionality and the latest features. asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + asyncio-threading.rst .. note:: The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index b5caf5502d0e1c6..417611307577fdc 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -6,59 +6,69 @@ -------------- -The :mod:`!atexit` module defines functions to register and unregister cleanup -functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`!atexit` runs these functions in the *reverse* -order in which they were registered; if you register ``A``, ``B``, and ``C``, -at interpreter termination time they will be run in the order ``C``, ``B``, -``A``. - -**Note:** The functions registered via this module are not called when the +The :mod:`!atexit` module defines functions to register and unregister +:dfn:`exit handlers`: functions that are automatically executed +"at exit", that is, upon normal program termination (for instance, +if :func:`sys.exit` is called or the main module's execution completes) +or, more generally, upon :term:`interpreter shutdown`. + +At exit, all registered exit handlers are called +in the *reverse* order in which they were registered. +If you register ``A``, ``B``, and ``C``, at interpreter shutdown time they +will be run in the order ``C``, ``B``, ``A``. +The assumption is that lower level modules will normally be imported before +higher level modules and thus must be cleaned up later. + +If an exception is raised during execution of an exit handler, a traceback is +printed (unless :exc:`SystemExit` is raised) and the exception information is +saved. After all exit handlers have had a chance to run, the last exception to +be raised is re-raised. + +In programs that use multiple interpreters, each interpreter has its own stack +of exit handlers, which are executed when the interpreter shuts down +(for example, with :meth:`concurrent.interpreters.Interpreter.close` or the +C API :c:func:`Py_EndInterpreter`). +Registration functions in this module only affect the interpreter they are +called from. + +**Note:** Exit handlers are not called when the program is killed by a signal not handled by Python, when a Python fatal internal error is detected, or when :func:`os._exit` is called. **Note:** The effect of registering or unregistering functions from within a cleanup function is undefined. -.. versionchanged:: 3.7 - When used with C-API subinterpreters, registered functions - are local to the interpreter they were registered in. +.. warning:: + When writing exit handlers, especially in C API extensions, keep in mind + that other exit handlers may still run arbitrary Python code after you + clean up. + Such code should succeed or fail with an exception, rather than crash. -.. function:: register(func, *args, **kwargs) +.. versionchanged:: 3.12 + Attempts to start a new thread or :func:`os.fork` a new process + in an exit handler now leads to :exc:`RuntimeError`. + Previously, this could cause race conditions between the main Python + runtime thread freeing thread states while internal :mod:`threading` + routines or the new process try to use that state, which could lead to + crashes rather than clean shutdown. - Register *func* as a function to be executed at termination. Any optional - arguments that are to be passed to *func* must be passed as arguments to - :func:`register`. It is possible to register the same function and arguments - more than once. +.. versionchanged:: 3.7 + When used with subinterpreters, registered functions + are local to the interpreter they were registered in. - At normal program termination (for instance, if :func:`sys.exit` is called or - the main module's execution completes), all functions registered are called in - last in, first out order. The assumption is that lower level modules will - normally be imported before higher level modules and thus must be cleaned up - later. +.. function:: register(func, *args, **kwargs) - If an exception is raised during execution of the exit handlers, a traceback is - printed (unless :exc:`SystemExit` is raised) and the exception information is - saved. After all exit handlers have had a chance to run, the last exception to - be raised is re-raised. + Register *func* as an exit handler. + Any optional arguments that are to be passed to *func* must be passed as + arguments to :func:`register`. + It is possible to register the same function and arguments more than once. This function returns *func*, which makes it possible to use it as a decorator. - .. warning:: - Starting new threads or calling :func:`os.fork` from a registered - function can lead to race condition between the main Python - runtime thread freeing thread states while internal :mod:`threading` - routines or the new process try to use that state. This can lead to - crashes rather than clean shutdown. - - .. versionchanged:: 3.12 - Attempts to start a new thread or :func:`os.fork` a new process - in a registered function now leads to :exc:`RuntimeError`. - .. function:: unregister(func) - Remove *func* from the list of functions to be run at interpreter shutdown. + Remove *func* from the list of exit handlers. :func:`unregister` silently does nothing if *func* was not previously registered. If *func* has been registered more than once, every occurrence of that function in the :mod:`!atexit` call stack will be removed. Equality diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 5e08d56fd66186a..8af40a2f8a65e3f 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -16,8 +16,10 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. This includes the :ref:`encodings specified in ` -:rfc:`4648` (Base64, Base32 and Base16) -and the non-standard :ref:`Base85 encodings `. +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -73,8 +75,8 @@ POST request. Added the *padded* and *wrapcol* parameters. -.. function:: b64decode(s, altchars=None, validate=False, *, padded=True) - b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True) +.. function:: b64decode(s, altchars=None, validate=False, *, padded=True, canonical=False) + b64decode(s, altchars=None, validate=True, *, ignorechars, padded=True, canonical=False) Decode the Base64 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -112,10 +114,13 @@ POST request. If *validate* is true, these non-alphabet characters in the input result in a :exc:`binascii.Error`. + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base64` for details. + For more information about the strict base64 check, see :func:`binascii.a2b_base64` .. versionchanged:: 3.15 - Added the *ignorechars* and *padded* parameters. + Added the *canonical*, *ignorechars*, and *padded* parameters. .. deprecated:: 3.15 Accepting the ``+`` and ``/`` characters with an alternative alphabet @@ -179,7 +184,7 @@ POST request. Added the *padded* and *wrapcol* parameters. -.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'') +.. function:: b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', canonical=False) Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and return the decoded :class:`bytes`. @@ -205,12 +210,15 @@ POST request. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-zero padding bits are rejected. + See :func:`binascii.a2b_base32` for details. + A :exc:`binascii.Error` is raised if *s* is incorrectly padded or if there are non-alphabet characters present in the input. .. versionchanged:: 3.15 - Added the *ignorechars* and *padded* parameters. + Added the *canonical*, *ignorechars*, and *padded* parameters. .. function:: b32hexencode(s, *, padded=True, wrapcol=0) @@ -224,7 +232,7 @@ POST request. Added the *padded* and *wrapcol* parameters. -.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'') +.. function:: b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', canonical=False) Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in :rfc:`4648`. @@ -237,7 +245,7 @@ POST request. .. versionadded:: 3.10 .. versionchanged:: 3.15 - Added the *ignorechars* and *padded* parameters. + Added the *canonical*, *ignorechars*, and *padded* parameters. .. function:: b16encode(s, *, wrapcol=0) @@ -278,19 +286,28 @@ POST request. Base85 Encodings ----------------- -Base85 encoding is not formally specified but rather a de facto standard, -thus different systems perform the encoding differently. +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. -The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of -the de facto standard. You should call the function with the Base85 -implementation used by the software you intend to work with. +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. -The two functions present in this module differ in how they handle the following: +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. -* Whether to include enclosing ``<~`` and ``~>`` markers -* Whether to include newline characters -* The set of ASCII characters used for encoding -* Handling of null bytes +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. Refer to the documentation of the individual functions for more information. @@ -301,49 +318,66 @@ Refer to the documentation of the individual functions for more information. *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') +.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', canonical=False) Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and return the decoded :class:`bytes`. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_ascii85` for details. + .. versionadded:: 3.4 + .. versionchanged:: 3.15 + Added the *canonical* parameter. + Single-character final groups are now always rejected as encoding + violations. + .. function:: b85encode(b, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -355,29 +389,34 @@ Refer to the documentation of the individual functions for more information. Added the *wrapcol* parameter. -.. function:: b85decode(b, *, ignorechars=b'') +.. function:: b85decode(b, *, ignorechars=b'', canonical=False) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. + .. versionadded:: 3.4 .. versionchanged:: 3.15 - Added the *ignorechars* parameter. + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. .. function:: z85encode(s, pad=False, *, wrapcol=0) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. If *wrapcol* is non-zero, insert a newline (``b'\n'``) character after at most every *wrapcol* characters. @@ -392,19 +431,23 @@ Refer to the documentation of the individual functions for more information. Added the *wrapcol* parameter. -.. function:: z85decode(s, *, ignorechars=b'') +.. function:: z85decode(s, *, ignorechars=b'', canonical=False) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-canonical encodings are rejected. + See :func:`binascii.a2b_base85` for details. + .. versionadded:: 3.13 .. versionchanged:: 3.15 - Added the *ignorechars* parameter. + Added the *canonical* and *ignorechars* parameters. + Single-character final groups are now always rejected as encoding + violations. .. _base64-legacy: @@ -475,3 +518,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 08a82cc4b5f6d5a..ceb80a35a1a76bb 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -48,8 +48,8 @@ The :mod:`!binascii` module defines the following functions: Added the *backtick* parameter. -.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False) - a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True) +.. function:: a2b_base64(string, /, *, padded=True, alphabet=BASE64_ALPHABET, strict_mode=False, canonical=False) + a2b_base64(string, /, *, ignorechars, padded=True, alphabet=BASE64_ALPHABET, strict_mode=True, canonical=False) Convert a block of base64 data back to binary and return the binary data. More than one line may be passed at a time. @@ -83,11 +83,15 @@ The :mod:`!binascii` module defines the following functions: * Contains no excess data after padding (including excess padding, newlines, etc.). * Does not start with a padding. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. This check is independent of *strict_mode*. + .. versionchanged:: 3.11 Added the *strict_mode* parameter. .. versionchanged:: 3.15 - Added the *alphabet*, *ignorechars* and *padded* parameters. + Added the *alphabet*, *canonical*, *ignorechars*, and *padded* parameters. .. function:: b2a_base64(data, *, padded=True, alphabet=BASE64_ALPHABET, wrapcol=0, newline=True) @@ -113,7 +117,7 @@ The :mod:`!binascii` module defines the following functions: Added the *alphabet*, *padded* and *wrapcol* parameters. -.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'') +.. function:: a2b_ascii85(string, /, *, foldspaces=False, adobe=False, ignorechars=b'', canonical=False) Convert Ascii85 data back to binary and return the binary data. @@ -122,19 +126,29 @@ The :mod:`!binascii` module defines the following functions: characters). Each group encodes 32 bits of binary data in the range from ``0`` to ``2 ** 32 - 1``, inclusive. The special character ``z`` is accepted as a short form of the group ``!!!!!``, which encodes four - consecutive null bytes. + consecutive null bytes. A single-character final group is always rejected + as an encoding violation. *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is not supported by the "standard" Ascii85 encoding. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. If + *adobe* is true, a leading ``<~`` is optionally accepted, while a + trailing ``~>`` is *required*, and :exc:`binascii.Error` is raised + if it is not found. *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. This should only contain whitespace characters. + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_ascii85` would produce: the ``z`` abbreviation must be used + for all-zero groups (rather than ``!!!!!``), and partial final groups + must use the same padding digits as the encoder. + Invalid Ascii85 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 @@ -153,17 +167,21 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - Note that the ``btoa`` implementation always pads. + If *pad* is true, the zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.15 -.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'') +.. function:: a2b_base85(string, /, *, alphabet=BASE85_ALPHABET, ignorechars=b'', canonical=False) Convert Base85 data back to binary and return the binary data. More than one line may be passed at a time. @@ -171,7 +189,8 @@ The :mod:`!binascii` module defines the following functions: Valid Base85 data contains characters from the Base85 alphabet in groups of five (except for the final group, which may have from two to five characters). Each group encodes 32 bits of binary data in the range from - ``0`` to ``2 ** 32 - 1``, inclusive. + ``0`` to ``2 ** 32 - 1``, inclusive. A single-character final group is + always rejected as an encoding violation. Optional *alphabet* must be a :class:`bytes` object of length 85 which specifies an alternative alphabet. @@ -179,6 +198,11 @@ The :mod:`!binascii` module defines the following functions: *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. + If *canonical* is true, non-canonical encodings are rejected with + :exc:`binascii.Error`. Here "canonical" means the encoding that + :func:`b2a_base85` would produce: partial final groups must use the + same padding digits as the encoder. + Invalid Base85 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 @@ -196,13 +220,15 @@ The :mod:`!binascii` module defines the following functions: after at most every *wrapcol* characters. If *wrapcol* is zero (default), do not insert any newlines. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + If *pad* is true, the zero-padding applied to the end of the input + is retained in the output, which will always be a multiple of 5 + bytes, and thus the length of the data may not be preserved on + decoding. .. versionadded:: 3.15 -.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'') +.. function:: a2b_base32(string, /, *, padded=True, alphabet=BASE32_ALPHABET, ignorechars=b'', canonical=False) Convert base32 data back to binary and return the binary data. @@ -231,6 +257,10 @@ The :mod:`!binascii` module defines the following functions: presented before the end of the encoded data and the excess pad characters will be ignored. + If *canonical* is true, non-zero padding bits in the last group are rejected + with :exc:`binascii.Error`, enforcing canonical encoding as defined in + :rfc:`4648` section 3.5. + Invalid base32 data will raise :exc:`binascii.Error`. .. versionadded:: 3.15 @@ -337,9 +367,8 @@ The :mod:`!binascii` module defines the following functions: *ignorechars* should be a :term:`bytes-like object` containing characters to ignore from the input. - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. + Similar functionality (but more liberal towards whitespace) is also accessible + using the :meth:`bytes.fromhex` class method. .. versionchanged:: 3.15 Added the *ignorechars* parameter. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 2c29a5ec992737e..39ab75c0904df9c 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -129,7 +129,7 @@ thoughts in mind: they are used. Consequently, if the search functions are used in a loop, the key function may be called again and again on the same array elements. If the key function isn't fast, consider wrapping it with - :py:func:`functools.cache` to avoid duplicate computations. Alternatively, + :py:deco:`functools.cache` to avoid duplicate computations. Alternatively, consider searching an array of precomputed keys to locate the insertion point (as shown in the examples section below). @@ -200,7 +200,7 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score) + >>> def grade(score): ... i = bisect([60, 70, 80, 90], score) ... return "FDCBA"[i] ... diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 54cafaf4fe47d8f..fb0ffc73b48c40c 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -54,13 +54,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -86,7 +86,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -365,7 +365,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is For simple text calendars this module provides the following functions. -.. function:: setfirstweekday(weekday) +.. function:: setfirstweekday(firstweekday) Sets the weekday (``0`` is Monday, ``6`` is Sunday) to start each week. The values :const:`MONDAY`, :const:`TUESDAY`, :const:`WEDNESDAY`, :const:`THURSDAY`, @@ -400,15 +400,15 @@ For simple text calendars this module provides the following functions. *month* (``1``--``12``), *day* (``1``--``31``). -.. function:: weekheader(n) +.. function:: weekheader(width) - Return a header containing abbreviated weekday names. *n* specifies the width in + Return a header containing abbreviated weekday names. *width* specifies the width in characters for one weekday. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -430,12 +430,12 @@ For simple text calendars this module provides the following functions. of the :class:`TextCalendar` class. -.. function:: prcal(year, w=0, l=0, c=6, m=3) +.. function:: prcal(theyear, w=0, l=0, c=6, m=3) Prints the calendar for an entire year as returned by :func:`calendar`. -.. function:: calendar(year, w=2, l=1, c=6, m=3) +.. function:: calendar(theyear, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. @@ -446,7 +446,7 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. @@ -580,9 +580,14 @@ The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. @@ -751,6 +756,11 @@ The following options are accepted: By default, today's date is highlighted in color and can be :ref:`controlled using environment variables `. +.. versionchanged:: 3.15 + By default, the month is now also highlighted in color, and + the days of the week are also in color. This behavior can be + :ref:`controlled using environment variables `. + *HTML-mode options:* .. option:: --css CSS, -c CSS diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 9259ab10d5850b5..e0e69e71e843e79 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1087,6 +1087,18 @@ On Windows, ``cpXXX`` codecs are available for all code pages. But only codecs listed in the following table are guaranteed to exist on other platforms. +On platforms that provide the C library's :manpage:`iconv(3)` function +(such as those using the GNU C Library), +every encoding known to ``iconv`` for which Python has no built-in codec +is available as well. +Such an encoding is looked up by its ``iconv`` name (for example ``cp1133``). +Prefixing the name with ``iconv:`` forces the use of the ``iconv``-based codec +even when a built-in codec of the same name exists (for example ``iconv:latin1``), +which is mostly useful for testing. + +.. versionchanged:: next + Added support for encodings provided by the C library's ``iconv``. + .. impl-detail:: Some common encodings can bypass the codecs lookup machinery to @@ -1155,7 +1167,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1192,7 +1204,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1399,6 +1411,14 @@ encodings. | punycode | | Implement :rfc:`3492`. | | | | Stateful codecs are not | | | | supported. | +| | | | +| | | .. warning:: | +| | | | +| | | The decoding and | +| | | encoding algorithms | +| | | scale poorly, so | +| | | limit the length of | +| | | untrusted input. | +--------------------+---------+---------------------------+ | raw_unicode_escape | | Latin-1 encoding with | | | | :samp:`\\u{XXXX}` and | @@ -1430,6 +1450,14 @@ encodings. | | | code actually uses UTF-8 | | | | by default. | +--------------------+---------+---------------------------+ +| utf-7-imap | mUTF-7 | Modified UTF-7 encoding | +| | | of :rfc:`3501` for IMAP4 | +| | | mailbox names. Only | +| | | ``errors='strict'`` is | +| | | supported. | +| | | | +| | | .. versionadded:: next | ++--------------------+---------+---------------------------+ .. versionchanged:: 3.8 "unicode_internal" codec is removed. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 51853725b1b297c..10e3790717ed6ed 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -456,7 +456,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index c8dcaef80188cd5..599a898eb2cef83 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -268,6 +268,15 @@ For example:: >>> c['sausage'] = 0 # counter entry with a zero count >>> del c['sausage'] # del actually removes the entry + Counters maintain insertion order internally but display from most common to + least common when possible: + + >>> c = Counter(a=1, b=2, c=3) + >>> c # display most common to least + Counter({'c': 3, 'b': 2, 'a': 1}) + >>> list(c.items()) # original insertion order + [('a', 1), ('b', 2), ('c', 3)] + .. versionadded:: 3.1 .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` @@ -326,7 +335,7 @@ For example:: .. versionadded:: 3.10 The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. + except for these two which work differently for counters: .. method:: fromkeys(iterable) @@ -484,6 +493,8 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: @@ -739,6 +750,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: @@ -1223,12 +1237,12 @@ original insertion position is changed and moved to the end:: self.move_to_end(key) An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`: +variants of :deco:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1243,10 +1257,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result @@ -1346,7 +1360,14 @@ attribute. A real dictionary used to store the contents of the :class:`UserDict` class. + :class:`!UserDict` instances also override the following method: + + .. method:: popitem + Remove and return a ``(key, value)`` pair from the wrapped dictionary. Pairs are + returned in the same order as ``data.popitem()``. (For the default + :meth:`dict.popitem`, this order is :abbr:`LIFO (last-in, first-out)`.) If the + dictionary is empty, raises a :exc:`KeyError`. :class:`UserList` objects ------------------------- diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst index 7ca843f27f5e9a6..a9fc4fa4b8cc36c 100644 --- a/Doc/library/compression.zstd.rst +++ b/Doc/library/compression.zstd.rst @@ -331,10 +331,14 @@ Compressing and decompressing data in memory If *max_length* is non-negative, the method returns at most *max_length* bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to + output can be produced (or EOF is reached), the :attr:`~.needs_input` + attribute will be set to ``False``. In this case, the next call to :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. + more of the output. The full content can thus be read like:: + + process_output(d.decompress(data, max_length)) + while not d.eof and not d.needs_input: + process_output(d.decompress(b"", max_length)) If all of the input data was decompressed and returned (either because this was less than *max_length* bytes, or because @@ -342,7 +346,7 @@ Compressing and decompressing data in memory will be set to ``True``. Attempting to decompress data after the end of a frame will raise a - :exc:`ZstdError`. Any data found after the end of the frame is ignored + :exc:`EOFError`. Any data found after the end of the frame is ignored and saved in the :attr:`~.unused_data` attribute. .. attribute:: eof @@ -499,7 +503,7 @@ Advanced parameter control The :meth:`~.bounds` method can be used on any attribute to get the valid values for that parameter. - Parameters are optional; any omitted parameter will have it's value selected + Parameters are optional; any omitted parameter will have its value selected automatically. Example getting the lower and upper bound of :attr:`~.compression_level`:: @@ -728,7 +732,7 @@ Advanced parameter control An :class:`~enum.IntEnum` containing the advanced decompression parameter keys that can be used when decompressing data. Parameters are optional; any - omitted parameter will have it's value selected automatically. + omitted parameter will have its value selected automatically. The :meth:`~.bounds` method can be used on any attribute to get the valid values for that parameter. diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index a32c38283134545..1f34dc66228e308 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -61,11 +61,18 @@ Executor Objects The returned iterator raises a :exc:`TimeoutError` if :meth:`~iterator.__next__` is called and the result isn't available after *timeout* seconds from the original call to :meth:`Executor.map`. - *timeout* can be an int or a float. If *timeout* is not specified or + *timeout* can be an int or a float. + It cancels all future calls of *fn* and closes the iterator. + If *timeout* is not specified or ``None``, there is no limit to the wait time. If a *fn* call raises an exception, then that exception will be raised when its value is retrieved from the iterator. + It does not cancel future calls of *fn*. + + The returned iterator has method :meth:`!close` which cancels all + future calls of *fn* and discards the results of already finished calls + if they are available. When using :class:`ProcessPoolExecutor`, this method chops *iterables* into a number of chunks which it submits to the pool as separate @@ -82,6 +89,10 @@ Executor Objects .. versionchanged:: 3.14 Added the *buffersize* parameter. + .. versionchanged:: next + The returned iterator is no longer automatically closed if a *fn* + call raises an exception. + .. method:: shutdown(wait=True, *, cancel_futures=False) Signal the executor that it should free any resources that it is using @@ -386,11 +397,6 @@ in a REPL or a lambda should not be expected to work. default in absence of a *mp_context* parameter. This feature is incompatible with the "fork" start method. - .. note:: - Bugs have been reported when using the *max_tasks_per_child* feature that - can result in the :class:`ProcessPoolExecutor` hanging in some - circumstances. Follow its eventual resolution in :gh:`115634`. - .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a :exc:`~concurrent.futures.process.BrokenProcessPool` error is now raised. @@ -426,6 +432,11 @@ in a REPL or a lambda should not be expected to work. require the *fork* start method for :class:`ProcessPoolExecutor` you must explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. versionchanged:: next + Fixed a deadlock (:gh:`115634`) where the executor could hang after + a worker process exited upon reaching its *max_tasks_per_child* + limit while tasks remained queued. + .. method:: terminate_workers() Attempt to terminate all living worker processes immediately by calling diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index a7b115e5f6307dc..2c509ab13a6acfd 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -21,7 +21,7 @@ thread) and calling a function in that execution context. For concurrency, interpreters themselves (and this module) don't provide much more than isolation, which on its own isn't useful. Actual concurrency is available separately through -:mod:`threads ` See `below `_ +:mod:`threads ` -- see `below `_. .. seealso:: @@ -191,7 +191,7 @@ objects are either directly shared or copied efficiently. For example: * :class:`float` * :class:`tuple` (of similarly supported objects) -There is a small number of Python types that actually share mutable +There are a small number of Python types that actually share mutable data between interpreters: * :class:`memoryview` @@ -274,7 +274,7 @@ Interpreter objects .. method:: call(callable, /, *args, **kwargs) - Return the result of calling running the given function in the + Return the result of running the given function in the interpreter (in the current thread). .. _interp-call-in-thread: diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index d058ba206c6cd62..6f005f98bd3ede5 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -68,7 +68,7 @@ A small number of constants live in the built-in namespace. They are: The same as the ellipsis literal "``...``", an object frequently used to indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but assignment to ``...`` raises a :exc:`SyntaxError`. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 5c6403879ab5056..666f85997448881 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -88,11 +88,11 @@ Functions and classes provided: the exception has been handled, and execution will resume with the statement immediately following the :keyword:`!with` statement. - :func:`contextmanager` uses :class:`ContextDecorator` so the context managers + :deco:`contextmanager` uses :class:`ContextDecorator` so the context managers it creates can be used as decorators as well as in :keyword:`with` statements. When used as a decorator, a new generator instance is implicitly created on each function call (this allows the otherwise "one-shot" context managers - created by :func:`contextmanager` to meet the requirement that context + created by :deco:`contextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators). .. versionchanged:: 3.2 @@ -101,7 +101,7 @@ Functions and classes provided: .. decorator:: asynccontextmanager - Similar to :func:`~contextlib.contextmanager`, but creates an + Similar to :deco:`~contextlib.contextmanager`, but creates an :ref:`asynchronous context manager `. This function is a :term:`decorator` that can be used to define a factory @@ -128,7 +128,7 @@ Functions and classes provided: .. versionadded:: 3.7 - Context managers defined with :func:`asynccontextmanager` can be used + Context managers defined with :deco:`asynccontextmanager` can be used either as decorators or with :keyword:`async with` statements:: import time @@ -148,11 +148,11 @@ Functions and classes provided: When used as a decorator, a new generator instance is implicitly created on each function call. This allows the otherwise "one-shot" context managers - created by :func:`asynccontextmanager` to meet the requirement that context + created by :deco:`asynccontextmanager` to meet the requirement that context managers support multiple invocations in order to be used as decorators. .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can + Async context managers created with :deco:`asynccontextmanager` can be used as decorators. @@ -400,7 +400,7 @@ Functions and classes provided: ``__exit__`` retains its optional exception handling even when used as a decorator. - ``ContextDecorator`` is used by :func:`contextmanager`, so you get this + ``ContextDecorator`` is used by :deco:`contextmanager`, so you get this functionality automatically. Example of ``ContextDecorator``:: @@ -467,12 +467,40 @@ Functions and classes provided: statements. If this is not the case, then the original construct with the explicit :keyword:`!with` statement inside the function should be used. + When the decorated callable is a generator function, coroutine function, or + asynchronous generator function, the returned wrapper is of the same kind + and keeps the context manager open for the lifetime of the iteration or + await rather than only for the call that creates the generator or coroutine + object. Wrapped generators and asynchronous generators are explicitly + closed when iteration ends, as if by :func:`closing` or :func:`aclosing`. + + .. note:: + For asynchronous generators the wrapper re-yields each value with + ``async for``; values sent with :meth:`~agen.asend` and exceptions + thrown with :meth:`~agen.athrow` are not forwarded to the wrapped + generator. + .. versionadded:: 3.2 + .. versionchanged:: 3.15 + Decorating a generator function, coroutine function, or asynchronous + generator function now keeps the context manager open across iteration + or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + .. class:: AsyncContextDecorator - Similar to :class:`ContextDecorator` but only for asynchronous functions. + Similar to :class:`ContextDecorator`, but the context manager is entered + and exited with :keyword:`async with`. Decorate coroutine functions and + asynchronous generator functions with this class; the returned wrapper is + of the same kind. + + .. note:: + Synchronous functions and generators are accepted, but the wrapper is + always asynchronous, so the decorated callable must then be awaited or + iterated with ``async for``. If that change of calling convention is + not intended, use :class:`ContextDecorator` instead. Example of ``AsyncContextDecorator``:: @@ -510,6 +538,13 @@ Functions and classes provided: .. versionadded:: 3.10 + .. versionchanged:: 3.15 + Decorating an asynchronous generator function now keeps the context + manager open across iteration. Previously the context manager exited + as soon as the generator object was created. Synchronous functions + and synchronous generator functions are also now accepted, with an + asynchronous wrapper returned. + .. class:: ExitStack() @@ -667,7 +702,7 @@ Functions and classes provided: Similar to :meth:`ExitStack.close` but properly handles awaitables. - Continuing the example for :func:`asynccontextmanager`:: + Continuing the example for :deco:`asynccontextmanager`:: async with AsyncExitStack() as stack: connections = [await stack.enter_async_context(get_connection()) @@ -925,7 +960,7 @@ Files are an example of effectively single use context managers, since the first :keyword:`with` statement will close the file, preventing any further IO operations using that file object. -Context managers created using :func:`contextmanager` are also single use +Context managers created using :deco:`contextmanager` are also single use context managers, and will complain about the underlying generator failing to yield if an attempt is made to use them a second time:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 93d0c0d34bf039d..b0cc0be8e911bf0 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -42,6 +42,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -130,6 +133,9 @@ Context Variables Tokens support the :ref:`context manager protocol ` to automatically reset context variables. See :meth:`ContextVar.set`. + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. + .. versionadded:: 3.14 Added support for usage as a context manager. diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 121c44a16ad43b9..39fc7800d03a916 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,9 +72,13 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 21ecdbcc08f3486..53288e810bffcf6 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -81,6 +81,13 @@ The :mod:`!csv` module defines the following functions: Spam, Spam, Spam, Spam, Spam, Baked Beans Spam, Lovely Spam, Wonderful Spam + where :file:`eggs.csv` contains: + + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + .. function:: writer(csvfile, /, dialect='excel', **fmtparams) @@ -110,6 +117,13 @@ The :mod:`!csv` module defines the following functions: spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) + which writes :file:`eggs.csv` containing: + + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + .. function:: register_dialect(name, /, dialect='excel', **fmtparams) @@ -191,6 +205,14 @@ The :mod:`!csv` module defines the following classes: >>> print(row) {'first_name': 'John', 'last_name': 'Cleese'} + where :file:`names.csv` contains: + + .. code-block:: text + + first_name,last_name + Eric,Idle + John,Cleese + .. class:: DictWriter(f, fieldnames, restval='', extrasaction='raise', \ dialect='excel', *args, **kwds) @@ -228,6 +250,15 @@ The :mod:`!csv` module defines the following classes: writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) + which writes :file:`names.csv` containing: + + .. code-block:: text + + first_name,last_name + Baked,Beans + Lovely,Spam + Wonderful,Spam + .. class:: Dialect @@ -281,6 +312,33 @@ The :mod:`!csv` module defines the following classes: is given, it is interpreted as a string containing possible valid delimiter characters. + The dialect is deduced by parsing the sample with every plausible + combination of parameters + and choosing the combination which splits the sample into rows + with the most consistent number of fields, + so the returned dialect is consistent with how :func:`reader` + will parse the sample. + Raise :exc:`Error` if no combination fits the sample, + in particular if it is a single column, + so there is no delimiter to find. + + If several combinations fit the sample equally well --- + for example if both ``','`` and ``';'`` split every row consistently --- + the delimiters listed in the :attr:`~Sniffer.preferred` attribute + are preferred, in that order, + no matter how many times each of them occurs. + + The *lineterminator* parameter is deduced separately, + by a majority vote among the line endings of the sample. + A tie is broken in the order ``'\r\n'``, ``'\n'``, ``'\r'``, + so a sample without a complete line gives ``'\r\n'``. + + .. versionchanged:: next + The dialect is now deduced by trial parsing + and the results may differ from those of earlier Python versions. + The *escapechar* and *lineterminator* parameters can now be detected, + and the requested *delimiters* are not restricted to ASCII. + .. method:: has_header(sample) @@ -301,6 +359,15 @@ The :mod:`!csv` module defines the following classes: This method is a rough heuristic and may produce both false positives and negatives. + The :class:`Sniffer` class has the following attribute: + + .. attribute:: preferred + + The list of the delimiters preferred for breaking ties, + in the order of preference. + It can be modified. + Its initial value is ``[',', '\t', ';', ' ', ':']``. + An example for :class:`Sniffer` use:: with open('example.csv', newline='') as csvfile: @@ -324,6 +391,8 @@ The :mod:`!csv` module defines the following constants: Instructs :class:`writer` objects to only quote those fields which contain special characters such as *delimiter*, *quotechar*, ``'\r'``, ``'\n'`` or any of the characters in *lineterminator*. + If *doublequote* is :const:`False` and *escapechar* is set, + the *quotechar* is escaped instead of causing the field to be quoted. .. data:: QUOTE_NONNUMERIC @@ -428,6 +497,10 @@ Dialects support the following attributes: On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. + .. versionchanged:: 3.10 + Previously the *escapechar* itself was not escaped, + which lost it on reading. + .. versionchanged:: 3.11 An empty *escapechar* is not allowed. @@ -475,6 +548,13 @@ Dialects support the following attributes: When ``True``, raise exception :exc:`Error` on bad CSV input. The default is ``False``. +Dialects support :func:`copy.replace`, +which returns a copy of the dialect +with the specified formatting parameters replaced. + +.. versionchanged:: next + Added support for :func:`copy.replace`. + .. _reader-objects: Reader Objects diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 571975d46743972..d76c449626fab4e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -14,6 +14,14 @@ used to wrap these libraries in pure Python. .. include:: ../includes/optional-module.rst +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: @@ -198,10 +206,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`!ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -370,15 +376,19 @@ in both C and ``libffi``, the following complex types are available: * - :class:`c_float_complex` - :c:expr:`float complex` - :py:class:`complex` - - ``'F'`` + - ``'Zf'`` * - :class:`c_double_complex` - :c:expr:`double complex` - :py:class:`complex` - - ``'D'`` + - ``'Zd'`` * - :class:`c_longdouble_complex` - :c:expr:`long double complex` - :py:class:`complex` - - ``'G'`` + - ``'Zg'`` + +.. versionchanged:: 3.15 + The :py:attr:`~_SimpleCData._type_` types ``F``, ``D`` and ``G`` have been + replaced with ``Zf``, ``Zd`` and ``Zg``. All these types can be created by calling them with an optional initializer of @@ -541,7 +551,7 @@ object with an :attr:`!_as_parameter_` attribute:: >>> If you don't want to store the instance's data in the :attr:`!_as_parameter_` -instance variable, you could define a :class:`property` which makes the +instance variable, you could define a :deco:`property` which makes the attribute available on request. @@ -684,6 +694,48 @@ through the :attr:`~_CFuncPtr.errcheck` attribute; see the reference manual for details. +Specifying function pointers using type annotations +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. decorator:: wrap_dll_function(dll) + :module: ctypes.util + + A :term:`decorator` that generates :attr:`~ctypes._CFuncPtr.argtypes` and + :attr:`~ctypes._CFuncPtr.restype` from a function signature, using the + :attr:`~function.__name__` of the function and its :term:`type annotations `. + + The decorated function should look like this:: + + @wrap_dll_function(dll_to_wrap) + def function_ptr_name(arg_name: ctypes_type, ...) -> ctypes_type: + """Optional docstring. There should be no function body.""" + + The body of the decorated function is ignored, and any parameters that are + missing type annotations are skipped. The names of the parameters are ignored + and do not have to match the underlying C implementation. + + If the decorated function does not have a return type annotation, a + :exc:`ValueError` is raised. A :exc:`ValueError` is also raised if it has a + keyword-only, ``*args``, or ``**kwargs`` parameter, since + :attr:`~ctypes._CFuncPtr.argtypes` describes positional arguments only. If + the name of the function does not exist in *dll*, an :exc:`AttributeError` + is raised. + + For example:: + + import ctypes + from ctypes.util import wrap_dll_function + + @wrap_dll_function(ctypes.pythonapi) + def PyObject_GetAttrString(op: ctypes.py_object, attr: ctypes.c_char_p) -> ctypes.py_object: + pass + + PyObject_GetAttrString(42, b"real") + + + .. versionadded:: next + + .. _ctypes-passing-pointers: Passing pointers (or: passing parameters by reference) @@ -1735,7 +1787,7 @@ If wrapping a shared library with :mod:`!ctypes`, consider determining the shared library name at development time, and hardcoding it into the wrapper module instead of using :func:`!find_library` to locate the library at runtime. -Also consider addding a configuration option or environment variable to let +Also consider adding a configuration option or environment variable to let users select a library to use, and then perhaps use :func:`!find_library` as a default or fallback. @@ -1756,11 +1808,10 @@ as a default or fallback. (or by) Python. It is recommended to only use this function as a default or fallback, - .. deprecated:: 3.15 + .. soft-deprecated:: 3.15 - This function is :term:`soft deprecated`. - It is kept for use in cases where it works, but not expected to be - updated for additional platforms and configurations. + This function is kept for use in cases where it works, but not expected to + be updated for additional platforms and configurations. On Linux, :func:`!find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the @@ -1818,6 +1869,10 @@ like ``find_library("c")`` will fail and return ``None``. .. availability:: Windows + .. soft-deprecated:: 3.16 + This function now always returns ``None``, as there are no more + VC runtime DLLs that are a single file and supported by Microsoft. + .. _ctypes-listing-loaded-shared-libraries: @@ -2569,8 +2624,7 @@ Fundamental data types .. attribute:: _type_ - Class attribute that contains an internal type code, as a - single-character string. + Class attribute that contains an internal type code, as a string. See :ref:`ctypes-fundamental-data-types` for a summary. Types marked \* in the summary may be (or always are) aliases of a @@ -2584,7 +2638,7 @@ Fundamental data types .. seealso:: - The :mod:`array` and :ref:`struct ` modules, + The :mod:`array` and :ref:`struct ` modules, as well as third-party modules like `numpy `__, use similar -- but slightly different -- type codes. @@ -3153,6 +3207,72 @@ fields, or any other data types containing pointer type fields. that should be merged into a containing structure or union. +.. decorator:: struct(*, align=None, layout, endian='native', pack=None) + :module: ctypes.util + + A :term:`decorator` that allows generating structure types using an + annotation-based syntax, similar to the :mod:`dataclasses` module. + + For example: + + .. code-block:: python + + from ctypes.util import struct + from ctypes import c_int + + @struct + class Point: + x: c_int + y: c_int + + point = Point(1, 2) + + *align*, *layout*, and *pack* supply the value for the :attr:`~ctypes.Structure._align_`, + :attr:`~ctypes.Structure._layout_`, and :attr:`~ctypes.Structure._pack_` + attributes, respectively. + + *endian* controls which structure class will be used as the base. + + - If *endian* is ``'native'``, :class:`~ctypes.Structure` will be used. + - If *endian* is ``'big'``, :class:`~ctypes.BigEndianStructure` will be used. + - If *endian* is ``'little'``, :class:`~ctypes.LittleEndianStructure` will be used. + + Any other value will raise a :class:`ValueError`. + + For controlling field-specific data, wrap the annotation in :class:`typing.Annotated` + with :class:`CFieldInfo` as the second argument, like so: + + .. code-block:: python + + @struct + class PyObject: + ob_refcnt: c_ssize_t + ob_type: c_void_p + + @struct + class PyHovercraftObject: + ob_base: Annotated[PyObject, CFieldInfo(anonymous=True)] + + .. versionadded:: next + + +.. class:: CFieldInfo(anonymous=False, bit_width=None) + :module: ctypes.util + + Information regarding a structure field defined by the :func:`struct` + decorator. This should be used in the second argument of a + :class:`typing.Annotated` wrapping a ctypes type. + + *anonymous* specifies whether the field will be present in the + :attr:`~ctypes.Structure._anonymous_` attribute of the generated class. + + If *bit_width* is non-``None``, the annotated field will be *bit_width* + number of bits in the generated structure. This is equivalent to passing + a third item in :attr:`~ctypes.Structure._fields_`. + + .. versionadded:: next + + .. _ctypes-arrays-pointers: Arrays and pointers @@ -3169,6 +3289,8 @@ Arrays and pointers subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ @@ -3191,8 +3313,8 @@ Arrays and pointers Equivalent to ``type * length``, where *type* is a :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index 9ae82c144655383..fad35973dada185 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -114,7 +114,7 @@ C library: .. function:: isblank(c) - Checks for an ASCII whitespace character; space or horizontal tab. + Checks for an ASCII blank character; space or horizontal tab. .. function:: iscntrl(c) @@ -130,7 +130,7 @@ C library: .. function:: isgraph(c) - Checks for ASCII any printable character except space. + Checks for any ASCII printable character except space. .. function:: islower(c) @@ -145,7 +145,7 @@ C library: .. function:: ispunct(c) - Checks for any printable ASCII character which is not a space or an alphanumeric + Checks for any ASCII printable character which is not a space or an alphanumeric character. @@ -168,22 +168,26 @@ C library: .. function:: isctrl(c) - Checks for an ASCII control character (ordinal values 0 to 31). + Checks for an ASCII control character (ordinal values 0 to 31). Unlike + :func:`iscntrl`, this does not include the delete character (0x7f). .. function:: ismeta(c) Checks for a non-ASCII character (ordinal values 0x80 and above). -These functions accept either integers or single-character strings; when the argument is a -string, it is first converted using the built-in function :func:`ord`. +These functions accept an integer, a single-character string, or a +:class:`curses.complexchar`. +A string is converted using the built-in function :func:`ord`, and a +complexchar by the code of its single character; a complexchar that holds +combining characters is not a single character and matches no class. Note that all these functions check ordinal bit values derived from the character of the string you pass in; they do not actually know anything about the host machine's character encoding. -The following two functions take either a single-character string or integer -byte value; they return a value of the same type. +The following three functions take either a single-character string or an +integer byte value; they return a value of the same type. .. function:: ascii(c) @@ -193,8 +197,13 @@ byte value; they return a value of the same type. .. function:: ctrl(c) - Return the control character corresponding to the given character (the character - bit value is bitwise-anded with 0x1f). + Return the control character corresponding to the given ASCII character (the + character bit value is bitwise-anded with 0x1f). A non-ASCII character has no + control character and is returned unchanged. + + .. versionchanged:: next + A non-ASCII argument is now returned unchanged instead of masked to a + control character. .. function:: alt(c) @@ -202,7 +211,8 @@ byte value; they return a value of the same type. Return the 8-bit character corresponding to the given ASCII character (the character bit value is bitwise-ored with 0x80). -The following function takes either a single-character string or integer value; +The following function takes a single-character string, an integer value, or a +:class:`curses.complexchar`; it returns a string. diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 5bc6b74b7f07cae..50e16847993e148 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -16,6 +16,14 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- +The module :mod:`!curses.panel` defines the following exception: + + +.. exception:: error + + Exception raised when a curses panel library function returns an error. + + The module :mod:`!curses.panel` defines the following functions: @@ -26,7 +34,8 @@ The module :mod:`!curses.panel` defines the following functions: .. function:: new_panel(win) - Returns a panel object, associating it with the given window *win*. Be aware + Returns a panel object, associating it with the given window *win* and + placing the new panel on top of the panel stack. Be aware that you need to keep the returned panel object referenced explicitly. If you don't, the panel object is garbage collected and removed from the panel stack. @@ -44,75 +53,96 @@ The module :mod:`!curses.panel` defines the following functions: .. _curses-panel-objects: -Panel Objects +Panel objects ------------- -Panel objects, as returned by :func:`new_panel` above, are windows with a -stacking order. There's always a window associated with a panel which determines -the content, while the panel methods are responsible for the window's depth in -the panel stack. +.. raw:: html + + + + + + + + + + + + + + + +.. class:: panel + + Panel objects, as returned by :func:`new_panel` above, are windows with a + stacking order. There's always a window associated with a panel which + determines the content, while the panel methods are responsible for the + window's depth in the panel stack. -Panel objects have the following methods: + Panel objects have the following methods: -.. method:: Panel.above() +.. method:: panel.above() Returns the panel above the current panel. -.. method:: Panel.below() +.. method:: panel.below() Returns the panel below the current panel. -.. method:: Panel.bottom() +.. method:: panel.bottom() Push the panel to the bottom of the stack. -.. method:: Panel.hidden() +.. method:: panel.hidden() Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. -.. method:: Panel.hide() +.. method:: panel.hide() Hide the panel. This does not delete the object, it just makes the window on screen invisible. -.. method:: Panel.move(y, x) +.. method:: panel.move(y, x) Move the panel to the screen coordinates ``(y, x)``. -.. method:: Panel.replace(win) +.. method:: panel.replace(win) Change the window associated with the panel to the window *win*. + Raise :exc:`curses.panel.error` if *win* has been detached from its + screen by :meth:`screen.close() `. -.. method:: Panel.set_userptr(obj) +.. method:: panel.set_userptr(obj) Set the panel's user pointer to *obj*. This is used to associate an arbitrary piece of data with the panel, and can be any Python object. -.. method:: Panel.show() +.. method:: panel.show() - Display the panel (which might have been hidden). + Display the panel (which might have been hidden), placing it on top of + the panel stack. -.. method:: Panel.top() +.. method:: panel.top() Push panel to the top of the stack. -.. method:: Panel.userptr() +.. method:: panel.userptr() Returns the user pointer for the panel. This might be any Python object. -.. method:: Panel.window() +.. method:: panel.window() Returns the window object associated with the panel. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 0f1449873fcf735..bcf2ef47d48898a 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -27,9 +27,28 @@ Linux and the BSD variants of Unix. Whenever the documentation mentions a *character* it can be specified as an integer, a one-character Unicode string or a one-byte byte string. + An integer is the code of a single encoded byte, optionally combined with + attributes and a color pair, as returned by :meth:`window.inch`. + Methods that write to a window accept also a character cell: a Unicode + string of a spacing character followed by combining characters, or a + :class:`complexchar`. Whenever the documentation mentions a *character string* it can be specified as a Unicode string or a byte string. + Methods that write to a window accept also a :class:`complexstr`. + +.. note:: + + Whether curses may be used from several threads + depends on the underlying library and how it was built. + In many implementations, including the default build of ncurses, + the screen state is shared and not thread-safe; + since the blocking and refresh methods + (such as :meth:`~window.getch` and :meth:`~window.refresh`) + release the :term:`GIL`, + unsynchronized use from several threads can then crash the interpreter. + Serialize the calls, + or wrap them in :meth:`window.use` and :meth:`screen.use`. .. seealso:: @@ -68,74 +87,87 @@ The module :mod:`!curses` defines the following exception: The module :mod:`!curses` defines the following functions: -.. function:: assume_default_colors(fg, bg, /) +Initialization and termination +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Allow use of default values for colors on terminals supporting this feature. - Use this to support transparency in your application. +.. function:: initscr() - * Assign terminal default foreground/background colors to color number ``-1``. - So ``init_pair(x, COLOR_RED, -1)`` will initialize pair *x* as red - on default background and ``init_pair(x, -1, COLOR_BLUE)`` will - initialize pair *x* as default foreground on blue. + Initialize the library. Return a :ref:`window ` object + which represents the whole screen. - * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + See :func:`setupterm` for a caveat about calling it before this function. - .. versionadded:: 3.14 + .. note:: + If there is an error opening the terminal, the underlying curses library may + cause the interpreter to exit. -.. function:: baudrate() +.. function:: endwin() - Return the output speed of the terminal in bits per second. On software - terminal emulators it will have a fixed high value. Included for historical - reasons; in former times, it was used to write output loops for time delays and - occasionally to change interfaces depending on the line speed. + De-initialize the library, and return terminal to normal status. +.. function:: isendwin() -.. function:: beep() + Return ``True`` if :func:`endwin` has been called (that is, the curses library has + been deinitialized). - Emit a short attention sound. +.. function:: newterm(type=None, fd=None, infd=None, /) + Initialize a new terminal in addition to the one initialized by + :func:`initscr`, + and return a :ref:`screen ` for it. + This allows a program to drive more than one terminal. -.. function:: can_change_color() + *type* is the terminal name, as in :func:`setupterm`; + if ``None``, the value of the :envvar:`TERM` environment variable is used. + *fd* and *infd* are the output and input files for the terminal: + either a file object or a file descriptor. + They default to :data:`sys.stdout` and :data:`sys.stdin`. - Return ``True`` or ``False``, depending on whether the programmer can change the colors - displayed by the terminal. + The new screen becomes the current one. + Use :func:`set_term` to switch between screens. + See :func:`setupterm` for a caveat about calling it before this function. -.. function:: cbreak() + .. versionadded:: next - Enter cbreak mode. In cbreak mode (sometimes called "rare" mode) normal tty - line buffering is turned off and characters are available to be read one by one. - However, unlike raw mode, special characters (interrupt, quit, suspend, and flow - control) retain their effects on the tty driver and calling program. Calling - first :func:`raw` then :func:`cbreak` leaves the terminal in cbreak mode. +.. function:: new_prescr() + Return a new :ref:`screen ` + that can be used to call functions that affect global state + before :func:`initscr` or :func:`newterm` is called. -.. function:: color_content(color_number) + Availability: if the underlying curses library provides ``new_prescr()``. - Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, - containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). + .. versionadded:: next +.. function:: set_term(screen, /) -.. function:: color_pair(pair_number) + Make *screen*, a :ref:`screen ` returned by + :func:`newterm`, the current terminal, + and return the previously current screen. + Returns ``None`` if the previous screen was the one created by + :func:`initscr`. + Raises :exc:`error` if *screen* has no terminal, + as is the case for a screen returned by :func:`new_prescr`. - Return the attribute value for displaying text in the specified color pair. - Only the first 256 color pairs are supported. This - attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, - and the other :const:`!A_\*` attributes. :func:`pair_number` is the counterpart - to this function. + .. versionadded:: next +.. function:: wrapper(func, /, *args, **kwargs) -.. function:: curs_set(visibility) + Initialize curses and call another callable object, *func*, which should be the + rest of your curses-using application. If the application raises an exception, + this function will restore the terminal to a sane state before re-raising the + exception and generating a traceback. The callable object *func* is then passed + the main window 'stdscr' as its first argument, followed by any other arguments + passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on + cbreak mode, turns off echo, enables the terminal keypad, and initializes colors + if the terminal has color support. On exit (whether normally or by exception) + it restores cooked mode, turns on echo, and disables the terminal keypad. - Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, - normal, or very visible. If the terminal supports the visibility requested, return the - previous cursor state; otherwise raise an exception. On many - terminals, the "visible" mode is an underline cursor and the "very visible" mode - is a block cursor. +Terminal mode control +~~~~~~~~~~~~~~~~~~~~~ .. function:: def_prog_mode() @@ -144,7 +176,6 @@ The module :mod:`!curses` defines the following functions: program is not in curses.) Subsequent calls to :func:`reset_prog_mode` will restore this mode. - .. function:: def_shell_mode() Save the current terminal mode as the "shell" mode, the mode when the running @@ -152,17 +183,49 @@ The module :mod:`!curses` defines the following functions: program is using curses capabilities.) Subsequent calls to :func:`reset_shell_mode` will restore this mode. +.. function:: reset_prog_mode() -.. function:: delay_output(ms) + Restore the terminal to "program" mode, as previously saved by + :func:`def_prog_mode`. - Insert an *ms* millisecond pause in output. +.. function:: reset_shell_mode() + + Restore the terminal to "shell" mode, as previously saved by + :func:`def_shell_mode`. + +.. function:: savetty() + + Save the current state of the terminal modes in a buffer, usable by + :func:`resetty`. + +.. function:: resetty() + + Restore the state of the terminal modes to what it was at the last call to + :func:`savetty`. + +.. function:: curs_set(visibility) + + Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, + normal, or very visible. If the terminal supports the visibility requested, return the + previous cursor state; otherwise raise an exception. On many + terminals, the "visible" mode is an underline cursor and the "very visible" mode + is a block cursor. + +.. function:: getsyx() + + Return the current coordinates of the virtual screen cursor as a tuple + ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. + +.. function:: setsyx(y, x) + Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then + :meth:`leaveok ` is set ``True``. .. function:: doupdate() Update the physical screen. The curses library keeps two data structures, one representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the + representing the desired next state. The :func:`doupdate` function updates the physical screen to match the virtual screen. The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write @@ -173,170 +236,178 @@ The module :mod:`!curses` defines the following functions: followed by a single :func:`!doupdate`. -.. function:: echo() +Input options +~~~~~~~~~~~~~ - Enter echo mode. In echo mode, each character input is echoed to the screen as - it is entered. +.. function:: cbreak() + Enter cbreak mode. In cbreak mode (sometimes called "rare" mode) normal tty + line buffering is turned off and characters are available to be read one by one. + However, unlike raw mode, special characters (interrupt, quit, suspend, and flow + control) retain their effects on the tty driver and calling program. Calling + first :func:`raw` then :func:`cbreak` leaves the terminal in cbreak mode. -.. function:: endwin() +.. function:: nocbreak() - De-initialize the library, and return terminal to normal status. + Leave cbreak mode. Return to normal "cooked" mode with line buffering. +.. function:: echo() -.. function:: erasechar() + Enter echo mode. In echo mode, each character input is echoed to the screen as + it is entered. - Return the user's current erase character as a one-byte bytes object. Under Unix operating systems this - is a property of the controlling tty of the curses program, and is not set by - the curses library itself. +.. function:: noecho() + Leave echo mode. Echoing of input characters is turned off. -.. function:: filter() +.. function:: raw() - The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the - capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` - string is set to the value of ``cr``. The effect is that the cursor is confined to - the current line, and so are screen updates. This may be used for enabling - character-at-a-time line editing without touching the rest of the screen. + Enter raw mode. In raw mode, normal line buffering and processing of + interrupt, quit, suspend, and flow control keys are turned off; characters are + presented to curses input functions one by one. +.. function:: noraw() -.. function:: flash() + Leave raw mode. Return to normal "cooked" mode with line buffering. - Flash the screen. That is, change it to reverse-video and then change it back - in a short interval. Some people prefer such as 'visible bell' to the audible - attention signal produced by :func:`beep`. +.. function:: halfdelay(tenths) + Used for half-delay mode, which is similar to cbreak mode in that characters + typed by the user are immediately available to the program. However, after + blocking for *tenths* tenths of seconds, raise an exception if nothing has + been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use + :func:`nocbreak` to leave half-delay mode. -.. function:: flushinp() +.. function:: meta(flag) - Flush all input buffers. This throws away any typeahead that has been typed - by the user and has not yet been processed by the program. + If *flag* is ``True``, allow 8-bit characters to be input. If + *flag* is ``False``, allow only 7-bit chars. +.. function:: nl(flag=True) -.. function:: getmouse() + Enter newline mode. This mode translates the return key into newline on input, + and translates newline into return and line-feed on output. Newline mode is + initially on. - After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be called to retrieve the queued mouse event, represented as a - 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish - multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is - currently unused.) *bstate* is an integer value whose bits will be set to - indicate the type of event, and will be the bitwise OR of one or more of the - following constants, where *n* is the button number from 1 to 5: - :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, - :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, - :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. + If *flag* is ``False``, the effect is the same as calling :func:`nonl`. - .. versionchanged:: 3.10 - The ``BUTTON5_*`` constants are now exposed if they are provided by the - underlying curses library. +.. function:: nonl() + Leave newline mode. Disable translation of return into newline on input, and + disable low-level translation of newline into newline/return on output (but this + does not change the behavior of ``addch('\n')``, which always does the + equivalent of return and line feed on the virtual screen). With translation + off, curses can sometimes speed up vertical motion a little; also, it will be + able to detect the return key on input. -.. function:: getsyx() +.. function:: intrflush(flag) - Return the current coordinates of the virtual screen cursor as a tuple - ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. + If *flag* is ``True``, pressing an interrupt key (interrupt, break, or quit) + will flush all output in the terminal driver queue. If *flag* is ``False``, + no flushing is done. +.. function:: qiflush([flag]) -.. function:: getwin(file) + If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If + *flag* is ``True``, or no argument is provided, the queues will be flushed when + these control characters are read. - Read window related data stored in the file by an earlier :func:`window.putwin` call. - The routine then creates and initializes a new window using that data, returning - the new window object. +.. function:: noqiflush() + When the :func:`!noqiflush` routine is used, normal flush of input and output queues + associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may + want to call :func:`!noqiflush` in a signal handler if you want output to + continue as though the interrupt had not occurred, after the handler exits. -.. function:: has_colors() +.. function:: typeahead(fd) - Return ``True`` if the terminal can display colors; otherwise, return ``False``. + Specify that the file descriptor *fd* be used for typeahead checking. If *fd* + is ``-1``, then no typeahead checking is done. -.. function:: has_extended_color_support() + The curses library does "line-breakout optimization" by looking for typeahead + periodically while updating the screen. If input is found, and it is coming + from a tty, the current update is postponed until refresh or doupdate is called + again, allowing faster response to commands typed in advance. This function + allows specifying a different file descriptor for typeahead checking. - Return ``True`` if the module supports extended colors; otherwise, return - ``False``. Extended color support allows more than 256 color pairs for - terminals that support more than 16 colors (e.g. xterm-256color). +.. function:: is_cbreak() - Extended color support requires ncurses version 6.1 or later. + Return ``True`` if cbreak mode (see :func:`cbreak`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. - .. versionadded:: 3.10 + .. versionadded:: next -.. function:: has_ic() +.. function:: is_echo() - Return ``True`` if the terminal has insert- and delete-character capabilities. - This function is included for historical reasons only, as all modern software - terminal emulators have such capabilities. + Return ``True`` if echo mode (see :func:`echo`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. + .. versionadded:: next -.. function:: has_il() +.. function:: is_nl() - Return ``True`` if the terminal has insert- and delete-line capabilities, or can - simulate them using scrolling regions. This function is included for - historical reasons only, as all modern software terminal emulators have such - capabilities. + Return ``True`` if nl mode (see :func:`nl`) is enabled, ``False`` otherwise. + Availability: ncurses 6.5 or later. + .. versionadded:: next -.. function:: has_key(ch) +.. function:: is_raw() - Take a key value *ch*, and return ``True`` if the current terminal type recognizes - a key with that value. + Return ``True`` if raw mode (see :func:`raw`) is enabled, + ``False`` otherwise. + Availability: ncurses 6.5 or later. + .. versionadded:: next -.. function:: halfdelay(tenths) - Used for half-delay mode, which is similar to cbreak mode in that characters - typed by the user are immediately available to the program. However, after - blocking for *tenths* tenths of seconds, raise an exception if nothing has - been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use - :func:`nocbreak` to leave half-delay mode. +Keyboard input +~~~~~~~~~~~~~~ +.. function:: ungetch(ch) -.. function:: init_color(color_number, r, g, b) + Push *ch* so the next :meth:`~window.getch` or :meth:`~window.get_wch` will + return it. - Change the definition of a color, taking the number of the color to be changed - followed by three RGB values (for the amounts of red, green, and blue - components). The value of *color_number* must be between ``0`` and - ``COLORS - 1``. Each of *r*, *g*, *b*, must be a value between ``0`` and - ``1000``. When :func:`init_color` is used, all occurrences of that color on the - screen immediately change to the new definition. This function is a no-op on - most terminals; it is active only if :func:`can_change_color` returns ``True``. + *ch* may be an integer (a key code or the code of an encoded byte), a byte, + or a string of length 1. A one-character string is pushed like + :func:`unget_wch`; on a narrow build it must encode to a single byte. + .. note:: -.. function:: init_pair(pair_number, fg, bg) + Only one *ch* can be pushed before :meth:`!getch` is called. - Change the definition of a color-pair. It takes three arguments: the number of - the color-pair to be changed, the foreground color number, and the background - color number. The value of *pair_number* must be between ``1`` and - ``COLOR_PAIRS - 1`` (the ``0`` color pair can only be changed by - :func:`use_default_colors` and :func:`assume_default_colors`). - The value of *fg* and *bg* arguments must be between ``0`` and - ``COLORS - 1``, or, after calling :func:`!use_default_colors` or - :func:`!assume_default_colors`, ``-1``. - If the color-pair was previously initialized, the screen is - refreshed and all occurrences of that color-pair are changed to the new - definition. + .. versionchanged:: next + A one-character string argument is no longer required to encode to a single + byte, except on a narrow build. +.. function:: unget_wch(ch) -.. function:: initscr() + Push *ch* so the next :meth:`~window.get_wch` will return it. - Initialize the library. Return a :ref:`window ` object - which represents the whole screen. + *ch* may be an integer (a character code, not a key code) or a string of + length 1. .. note:: - If there is an error opening the terminal, the underlying curses library may - cause the interpreter to exit. - + Only one *ch* can be pushed before :meth:`!get_wch` is called. -.. function:: is_term_resized(nlines, ncols) + .. versionadded:: 3.3 - Return ``True`` if :func:`resize_term` would modify the window structure, - ``False`` otherwise. + .. versionchanged:: next + Also available on a narrow build, where *ch* must encode to a single byte + (an 8-bit locale). +.. function:: flushinp() -.. function:: isendwin() + Flush all input buffers. This throws away any typeahead that has been typed + by the user and has not yet been processed by the program. - Return ``True`` if :func:`endwin` has been called (that is, the curses library has - been deinitialized). +.. function:: has_key(ch) + Take a key value *ch*, and return ``True`` if the current terminal type recognizes + a key with that value. .. function:: keyname(k) @@ -347,316 +418,551 @@ The module :mod:`!curses` defines the following functions: bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. + Raise a :exc:`ValueError` if *k* is negative. -.. function:: killchar() +.. function:: define_key(definition, keycode) - Return the user's current line kill character as a one-byte bytes object. Under Unix operating systems - this is a property of the controlling tty of the curses program, and is not set - by the curses library itself. + Define an escape sequence *definition*, a string, as a key that generates + the key code *keycode*, so that :mod:`curses` interprets it like one of the + keys predefined in the terminal database. + If *definition* is ``None``, any existing binding for *keycode* is removed. + If *keycode* is zero or negative, any existing binding for *definition* is + removed. -.. function:: longname() + .. versionadded:: next - Return a bytes object containing the terminfo long name field describing the current - terminal. The maximum length of a verbose description is 128 characters. It is - defined only after the call to :func:`initscr`. +.. function:: key_defined(definition) + Return the key code bound to the escape sequence *definition*, a string, + ``0`` if no key code is bound to it, or ``-1`` if *definition* is a prefix + of a longer bound sequence (and so is ambiguous). -.. function:: meta(flag) + .. versionadded:: next - If *flag* is ``True``, allow 8-bit characters to be input. If - *flag* is ``False``, allow only 7-bit chars. +.. function:: keyok(keycode, enable) + Enable (if *enable* is true) or disable (otherwise) interpretation of the + key code *keycode*. Unlike :meth:`window.keypad`, this affects a single + key code rather than all of them. -.. function:: mouseinterval(interval) + .. versionadded:: next - Set the maximum time in milliseconds that can elapse between press and release - events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 milliseconds, or one fifth of a second. +Mouse +~~~~~ -.. function:: mousemask(mousemask) +.. function:: getmouse() - Set the mouse events to be reported, and return a tuple ``(availmask, - oldmask)``. *availmask* indicates which of the specified mouse events can be - reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse - events are ever reported. + After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this + method should be called to retrieve the queued mouse event, represented as a + 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish + multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is + currently unused.) *bstate* is an integer value whose bits will be set to + indicate the type of event, and will be the bitwise OR of one or more of the + following constants, where *n* is the button number from 1 to 5: + :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, + :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, + :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. + .. versionchanged:: 3.10 + The ``BUTTON5_*`` constants are now exposed if they are provided by the + underlying curses library. -.. function:: napms(ms) +.. function:: ungetmouse(id, x, y, z, bstate) - Sleep for *ms* milliseconds. + Push a :const:`KEY_MOUSE` event onto the input queue, associating the given + state data with it. +.. function:: mousemask(mousemask) -.. function:: newpad(nlines, ncols) + Set the mouse events to be reported, and return a tuple ``(availmask, + oldmask)``. *availmask* indicates which of the specified mouse events can be + reported; on complete failure it returns ``0``. *oldmask* is the previous value of + the mouse event mask. If this function is never called, no mouse + events are ever reported. - Create and return a pointer to a new pad data structure with the given number - of lines and columns. Return a pad as a window object. +.. function:: mouseinterval(interval) - A pad is like a window, except that it is not restricted by the screen size, and - is not necessarily associated with a particular part of the screen. Pads can be - used when a large window is needed, and only a part of the window will be on the - screen at one time. Automatic refreshes of pads (such as from scrolling or - echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` - methods of a pad require 6 arguments to specify the part of the pad to be - displayed and the location on the screen to be used for the display. The - arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and - the *s* arguments define a clipping box on the screen within which the pad region - is to be displayed. + Set the maximum time in milliseconds that can elapse between press and release + events in order for them to be recognized as a click, and return the previous + interval value. The default value is 166 milliseconds, or one sixth of a second. + Use a negative *interval* to obtain the interval value without changing it. +.. function:: has_mouse() -.. function:: newwin(nlines, ncols) - newwin(nlines, ncols, begin_y, begin_x) + Return ``True`` if the mouse driver has been successfully initialized. - Return a new :ref:`window `, whose left-upper corner - is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. + Availability: if the underlying curses library provides ``has_mouse()``. - By default, the window will extend from the specified position to the lower - right corner of the screen. + .. versionadded:: next -.. function:: nl() +Color +~~~~~ - Enter newline mode. This mode translates the return key into newline on input, - and translates newline into return and line-feed on output. Newline mode is - initially on. +.. function:: start_color() + Must be called if the programmer wants to use colors, and before any other color + manipulation routine is called. It is good practice to call this routine right + after :func:`initscr`. -.. function:: nocbreak() + :func:`start_color` initializes eight basic colors (black, red, green, yellow, + blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` + module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number + of colors and color-pairs the terminal can support. It also restores the colors + on the terminal to the values they had when the terminal was just turned on. - Leave cbreak mode. Return to normal "cooked" mode with line buffering. +.. function:: has_colors() + Return ``True`` if the terminal can display colors; otherwise, return ``False``. -.. function:: noecho() +.. function:: has_extended_color_support() - Leave echo mode. Echoing of input characters is turned off. + Return ``True`` if the module supports extended colors; otherwise, return + ``False``. Extended color support allows more than 256 color pairs for + terminals that support more than 16 colors (for example, xterm-256color). + Extended color support requires ncurses version 6.1 or later. -.. function:: nonl() + .. versionadded:: 3.10 - Leave newline mode. Disable translation of return into newline on input, and - disable low-level translation of newline into newline/return on output (but this - does not change the behavior of ``addch('\n')``, which always does the - equivalent of return and line feed on the virtual screen). With translation - off, curses can sometimes speed up vertical motion a little; also, it will be - able to detect the return key on input. +.. function:: can_change_color() + Return ``True`` or ``False``, depending on whether the programmer can change the colors + displayed by the terminal. -.. function:: noqiflush() +.. function:: init_color(color_number, r, g, b) - When the :func:`!noqiflush` routine is used, normal flush of input and output queues - associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may - want to call :func:`!noqiflush` in a signal handler if you want output to - continue as though the interrupt had not occurred, after the handler exits. + Change the definition of a color, taking the number of the color to be changed + followed by three RGB values (for the amounts of red, green, and blue + components). The value of *color_number* must be between ``0`` and + ``COLORS - 1``. Each of *r*, *g*, *b*, must be a value between ``0`` and + ``1000``. When :func:`init_color` is used, all occurrences of that color on the + screen immediately change to the new definition. This function is a no-op on + most terminals; it is active only if :func:`can_change_color` returns ``True``. +.. function:: color_content(color_number) -.. function:: noraw() + Return the intensity of the red, green, and blue (RGB) components in the color + *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, + containing the R,G,B values for the given color, which will be between + ``0`` (no component) and ``1000`` (maximum amount of component). Raise an + exception if the color is not supported. - Leave raw mode. Return to normal "cooked" mode with line buffering. +.. function:: init_pair(pair_number, fg, bg) + Change the definition of a color-pair. It takes three arguments: the number of + the color-pair to be changed, the foreground color number, and the background + color number. The value of *pair_number* must be between ``1`` and + ``COLOR_PAIRS - 1`` (the ``0`` color pair can only be changed by + :func:`use_default_colors` and :func:`assume_default_colors`). + The value of *fg* and *bg* arguments must be between ``0`` and + ``COLORS - 1``, or, after calling :func:`!use_default_colors` or + :func:`!assume_default_colors`, ``-1``. + If the color-pair was previously initialized, the screen is + refreshed and all occurrences of that color-pair are changed to the new + definition. .. function:: pair_content(pair_number) Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. The value of *pair_number* must be between ``0`` and ``COLOR_PAIRS - 1``. +.. function:: color_pair(pair_number) + + Return the attribute value for displaying text in the specified color pair. + Only color pairs that fit in the color-pair field of the returned value can + be represented (usually the first 256); a larger *pair_number* raises + :exc:`OverflowError` rather than being silently masked to a different pair. + Use :meth:`~window.color_set` or :meth:`~window.attr_set` to display higher + pairs. This attribute value can be combined with :const:`A_STANDOUT`, + :const:`A_REVERSE`, and the other :const:`!A_\*` attributes. + :func:`pair_number` is the counterpart to this function. .. function:: pair_number(attr) Return the number of the color-pair set by the attribute value *attr*. :func:`color_pair` is the counterpart to this function. +.. function:: use_default_colors() -.. function:: putp(str) + Equivalent to ``assume_default_colors(-1, -1)``. - Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified - terminfo capability for the current terminal. Note that the output of :func:`putp` - always goes to standard output. +.. function:: assume_default_colors(fg, bg, /) + Allow use of default values for colors on terminals supporting this feature. + Use this to support transparency in your application. -.. function:: qiflush([flag]) + * Assign terminal default foreground/background colors to color number ``-1``. + So ``init_pair(x, COLOR_RED, -1)`` will initialize pair *x* as red + on default background and ``init_pair(x, -1, COLOR_BLUE)`` will + initialize pair *x* as default foreground on blue. - If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If - *flag* is ``True``, or no argument is provided, the queues will be flushed when - these control characters are read. + * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + This is an ncurses extension. -.. function:: raw() + .. versionadded:: 3.14 - Enter raw mode. In raw mode, normal line buffering and processing of - interrupt, quit, suspend, and flow control keys are turned off; characters are - presented to curses input functions one by one. +.. function:: alloc_pair(fg, bg) + Allocate a color pair for foreground color *fg* and background color *bg*, + and return its number. If a color pair for the same combination of colors + already exists, return its number. Otherwise allocate a new color pair and + return its number. -.. function:: reset_prog_mode() + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). - Restore the terminal to "program" mode, as previously saved by - :func:`def_prog_mode`. + .. versionadded:: next +.. function:: free_pair(pair_number) -.. function:: reset_shell_mode() + Free the color pair *pair_number*, which must have been allocated by + :func:`alloc_pair`. The pair must not be in use. - Restore the terminal to "shell" mode, as previously saved by - :func:`def_shell_mode`. + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). + .. versionadded:: next -.. function:: resetty() +.. function:: find_pair(fg, bg) - Restore the state of the terminal modes to what it was at the last call to - :func:`savetty`. + Return the number of a color pair for foreground color *fg* and background + color *bg*, or ``-1`` if no color pair for this combination of colors has + been allocated. + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). -.. function:: resize_term(nlines, ncols) + .. versionadded:: next - Backend function used by :func:`resizeterm`, performing most of the work; - when resizing the windows, :func:`resize_term` blank-fills the areas that are - extended. The calling application should fill in these areas with - appropriate data. The :func:`!resize_term` function attempts to resize all - windows. However, due to the calling convention of pads, it is not possible - to resize these without additional interaction with the application. +.. function:: reset_color_pairs() + Discard all color-pair definitions, releasing the color pairs allocated by + :func:`init_pair` and :func:`alloc_pair`. -.. function:: resizeterm(nlines, ncols) + This function is only available if Python was built against a wide-character + version of the underlying curses library with extended-color support (see + :func:`has_extended_color_support`). - Resize the standard and current windows to the specified dimensions, and - adjusts other bookkeeping data used by the curses library that record the - window dimensions (in particular the SIGWINCH handler). + .. versionadded:: next -.. function:: savetty() +Windows and pads +~~~~~~~~~~~~~~~~ - Save the current state of the terminal modes in a buffer, usable by - :func:`resetty`. +.. function:: newwin(nlines, ncols) + newwin(nlines, ncols, begin_y, begin_x) -.. function:: get_escdelay() + Return a new :ref:`window `, whose left-upper corner + is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. - Retrieves the value set by :func:`set_escdelay`. + By default, the window will extend from the specified position to the lower + right corner of the screen. - .. versionadded:: 3.9 +.. function:: newpad(nlines, ncols) -.. function:: set_escdelay(ms) + Create and return a pointer to a new pad data structure with the given number + of lines and columns. Return a pad as a window object. - Sets the number of milliseconds to wait after reading an escape character, - to distinguish between an individual escape character entered on the - keyboard from escape sequences sent by cursor and function keys. + A pad is like a window, except that it is not restricted by the screen size, and + is not necessarily associated with a particular part of the screen. Pads can be + used when a large window is needed, and only a part of the window will be on the + screen at one time. Automatic refreshes of pads (such as from scrolling or + echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` + methods of a pad require 6 arguments to specify the part of the pad to be + displayed and the location on the screen to be used for the display. The + arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* + arguments refer to the upper-left corner of the pad region to be displayed and + the *s* arguments define a clipping box on the screen within which the pad region + is to be displayed. - .. versionadded:: 3.9 -.. function:: get_tabsize() +Soft labels +~~~~~~~~~~~ - Retrieves the value set by :func:`set_tabsize`. +.. _curses-slk: - .. versionadded:: 3.9 +The following functions manage *soft-label keys*, a row of labels displayed +along the bottom line of the screen, typically used to label a row of function +keys. :func:`slk_init` must be called before :func:`initscr` or +:func:`newterm`; it takes one screen line away from the standard window for the +labels. -.. function:: set_tabsize(size) - Sets the number of columns used by the curses library when converting a tab - character to spaces as it adds the tab to a window. +.. function:: slk_init(fmt=0) - .. versionadded:: 3.9 + Reserve a screen line for the soft labels and choose their layout. *fmt* + selects the arrangement: ``0`` for 3-2-3 (eight labels), ``1`` for 4-4 + (eight labels). Where the underlying curses library supports them, ``2`` + gives 4-4-4 (twelve labels) and ``3`` gives 4-4-4 with an index line. -.. function:: setsyx(y, x) + Must be called before :func:`initscr` or :func:`newterm`. - Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then - :meth:`leaveok ` is set ``True``. + .. versionadded:: next -.. function:: setupterm(term=None, fd=-1) +.. function:: slk_set(labnum, label, justify) - Initialize the terminal. *term* is a string giving - the terminal name, or ``None``; if omitted or ``None``, the value of the - :envvar:`TERM` environment variable will be used. *fd* is the - file descriptor to which any initialization sequences will be sent; if not - supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + Set the text of soft label number *labnum*, in the range ``1`` through ``8`` + (or ``12`` in a twelve-label layout). *justify* controls how *label* is + placed within the label: ``0`` for left, ``1`` for centered, ``2`` for right. + .. versionadded:: next -.. function:: start_color() - Must be called if the programmer wants to use colors, and before any other color - manipulation routine is called. It is good practice to call this routine right - after :func:`initscr`. +.. function:: slk_label(labnum) - :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` - module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number - of colors and color-pairs the terminal can support. It also restores the colors - on the terminal to the values they had when the terminal was just turned on. + Return the current text of soft label number *labnum*, justified as it was + set, or an empty string if it has no label. + .. versionadded:: next -.. function:: termattrs() - Return a logical OR of all video attributes supported by the terminal. This - information is useful when a curses program needs complete control over the - appearance of the screen. +.. function:: slk_refresh() + Update the soft labels on the physical screen, like + :meth:`~curses.window.refresh` for a window. -.. function:: termname() + .. versionadded:: next - Return the value of the environment variable :envvar:`TERM`, as a bytes object, - truncated to 14 characters. +.. function:: slk_noutrefresh() -.. function:: tigetflag(capname) + Update the soft labels on the virtual screen, like + :meth:`window.noutrefresh`. Use it together with :func:`doupdate` to batch + screen updates. - Return the value of the Boolean capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a - Boolean capability, or ``0`` if it is canceled or absent from the terminal - description. + .. versionadded:: next -.. function:: tigetnum(capname) +.. function:: slk_clear() - Return the value of the numeric capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a - numeric capability, or ``-1`` if it is canceled or absent from the terminal - description. + Remove the soft labels from the screen. + .. versionadded:: next -.. function:: tigetstr(capname) - Return the value of the string capability corresponding to the terminfo - capability name *capname* as a bytes object. Return ``None`` if *capname* - is not a terminfo "string capability", or is canceled or absent from the - terminal description. +.. function:: slk_restore() + Restore the soft labels to the screen after a :func:`slk_clear`. -.. function:: tparm(str[, ...]) + .. versionadded:: next - Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. - ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. +.. function:: slk_touch() -.. function:: typeahead(fd) + Force all the soft labels to be redrawn by the next :func:`slk_refresh` or + :func:`slk_noutrefresh`. - Specify that the file descriptor *fd* be used for typeahead checking. If *fd* - is ``-1``, then no typeahead checking is done. + .. versionadded:: next - The curses library does "line-breakout optimization" by looking for typeahead - periodically while updating the screen. If input is found, and it is coming - from a tty, the current update is postponed until refresh or doupdate is called - again, allowing faster response to commands typed in advance. This function - allows specifying a different file descriptor for typeahead checking. +.. function:: slk_attron(attr) + slk_attroff(attr) + slk_attrset(attr) -.. function:: unctrl(ch) + Add, remove, or set the attributes used to display the soft labels, given as + packed ``A_*`` attributes. - Return a bytes object which is a printable representation of the character *ch*. - Control characters are represented as a caret followed by the character, for - example as ``b'^C'``. Printing characters are left as they are. + .. versionadded:: next -.. function:: ungetch(ch) +.. function:: slk_attr() - Push *ch* so the next :meth:`~window.getch` will return it. + Return the current attributes of the soft labels as packed ``A_*`` + attributes. Availability depends on the underlying curses library. - .. note:: + .. versionadded:: next + + +.. function:: slk_attr_on(attr) + slk_attr_off(attr) + + Turn the given attributes on or off without affecting any others. Like the + ``attr_*`` window methods, these work with the + :ref:`WA_* attributes ` rather than packed ``A_*`` + attributes. + + .. versionadded:: next + + +.. function:: slk_attr_set(attr, pair=0) + + Set the attributes and color pair of the soft labels. *attr* is given as + :ref:`WA_* attributes ` and *pair* as a color pair + number. + + .. versionadded:: next + + +.. function:: slk_color(pair) + + Set the color pair of the soft labels to color pair number *pair*. + + .. versionadded:: next + + +Saving and restoring +~~~~~~~~~~~~~~~~~~~~ + +.. function:: getwin(file) + + Read window-related data stored in the file by an earlier :meth:`window.putwin` call. + The routine then creates and initializes a new window using that data, returning + the new window object. The *file* argument must be a file object opened for + reading in binary mode. + +.. function:: scr_dump(filename) + + Write the current contents of the virtual screen to *filename*, which may be + a string or a :term:`path-like object`. The file can later be read by + :func:`scr_restore`, :func:`scr_init` or :func:`scr_set`. This is the + whole-screen counterpart of :meth:`window.putwin`. + + .. versionadded:: next + +.. function:: scr_restore(filename) + + Set the virtual screen to the contents of *filename*, which must have been + written by :func:`scr_dump`. The next call to :func:`doupdate` or + :meth:`window.refresh` restores the screen to those contents. + + .. versionadded:: next + +.. function:: scr_init(filename) + + Initialize the assumed contents of the terminal from *filename*, which must + have been written by :func:`scr_dump`. Use it when the terminal already + displays those contents, for example after another program has drawn the + screen, so that curses does not redraw what is already there. + + .. versionadded:: next + +.. function:: scr_set(filename) + + Use *filename*, which must have been written by :func:`scr_dump`, as both + the virtual screen and the assumed terminal contents. This combines the + effects of :func:`scr_restore` and :func:`scr_init`. + + .. versionadded:: next + + +Querying the terminal +~~~~~~~~~~~~~~~~~~~~~ + +.. function:: baudrate() + + Return the output speed of the terminal in bits per second. On software + terminal emulators it will have a fixed high value. Included for historical + reasons; in former times, it was used to write output loops for time delays and + occasionally to change interfaces depending on the line speed. + +.. function:: erasechar() + + Return the user's current erase character as a raw byte, a :class:`bytes` + object of length 1. See also :func:`erasewchar`. + +.. function:: erasewchar() + + Return the user's current erase character as a one-character :class:`str`. + Under Unix operating systems this is a property of the controlling tty of the + curses program, and is not set by the curses library itself. + + .. versionadded:: next + +.. function:: killchar() + + Return the user's current line kill character as a raw byte, a :class:`bytes` + object of length 1. See also :func:`killwchar`. + +.. function:: killwchar() + + Return the user's current line kill character as a one-character :class:`str`. + Under Unix operating systems this is a property of the controlling tty of the + curses program, and is not set by the curses library itself. + + .. versionadded:: next + +.. function:: termname() + + Return the value of the environment variable :envvar:`TERM`, as a bytes object, + truncated to 14 characters. + +.. function:: longname() + + Return a bytes object containing the terminfo long name field describing the current + terminal. The maximum length of a verbose description is 128 characters. It is + defined only after the call to :func:`initscr`. + +.. function:: termattrs() + + Return a logical OR of all video attributes supported by the terminal. This + information is useful when a curses program needs complete control over the + appearance of the screen. + +.. function:: term_attrs() + + Like :func:`termattrs`, but return the attributes as :ref:`WA_* + ` values rather than ``A_*`` values. + + .. versionadded:: next + +.. function:: has_ic() + + Return ``True`` if the terminal has insert- and delete-character capabilities. + This function is included for historical reasons only, as all modern software + terminal emulators have such capabilities. + +.. function:: has_il() + + Return ``True`` if the terminal has insert- and delete-line capabilities, or can + simulate them using scrolling regions. This function is included for + historical reasons only, as all modern software terminal emulators have such + capabilities. - Only one *ch* can be pushed before :meth:`!getch` is called. +Bell and screen flash +~~~~~~~~~~~~~~~~~~~~~ + +.. function:: beep() + + Emit a short attention sound. + +.. function:: flash() + + Flash the screen. That is, change it to reverse-video and then change it back + in a short interval. Some people prefer such as 'visible bell' to the audible + attention signal produced by :func:`beep`. + + +Terminal resizing +~~~~~~~~~~~~~~~~~ + +.. function:: resizeterm(nlines, ncols) + + Resize the standard and current windows to the specified dimensions, and + adjusts other bookkeeping data used by the curses library that record the + window dimensions (in particular the SIGWINCH handler). + +.. function:: resize_term(nlines, ncols) + + Backend function used by :func:`resizeterm`, performing most of the work; + when resizing the windows, :func:`resize_term` blank-fills the areas that are + extended. The calling application should fill in these areas with + appropriate data. The :func:`!resize_term` function attempts to resize all + windows. However, due to the calling convention of pads, it is not possible + to resize these without additional interaction with the application. + +.. function:: is_term_resized(nlines, ncols) + + Return ``True`` if :func:`resize_term` would modify the window structure, + ``False`` otherwise. .. function:: update_lines_cols() @@ -666,22 +972,118 @@ The module :mod:`!curses` defines the following functions: .. versionadded:: 3.5 -.. function:: unget_wch(ch) +Terminfo database +~~~~~~~~~~~~~~~~~ - Push *ch* so the next :meth:`~window.get_wch` will return it. +.. function:: setupterm(term=None, fd=-1) + + Initialize the terminal. *term* is a string giving + the terminal name, or ``None``; if omitted or ``None``, the value of the + :envvar:`TERM` environment variable will be used. *fd* is the + file descriptor to which any initialization sequences will be sent; if not + supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + + Raise a :exc:`curses.error` if the terminal could not be found or its + terminfo database entry could not be read. If the terminal has already + been initialized, this function has no effect. .. note:: - Only one *ch* can be pushed before :meth:`!get_wch` is called. + Calling :func:`initscr` or :func:`newterm` after :func:`setupterm` + leaks the terminal that :func:`setupterm` allocated: + the curses library keeps only a single current terminal + and does not free the previously allocated one. - .. versionadded:: 3.3 +.. function:: tigetflag(capname) + Return the value of the Boolean capability corresponding to the terminfo + capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a + Boolean capability, or ``0`` if it is canceled or absent from the terminal + description. -.. function:: ungetmouse(id, x, y, z, bstate) + :func:`setupterm` (or :func:`initscr`) must be called first. - Push a :const:`KEY_MOUSE` event onto the input queue, associating the given - state data with it. +.. function:: tigetnum(capname) + + Return the value of the numeric capability corresponding to the terminfo + capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a + numeric capability, or ``-1`` if it is canceled or absent from the terminal + description. + + :func:`setupterm` (or :func:`initscr`) must be called first. + +.. function:: tigetstr(capname) + + Return the value of the string capability corresponding to the terminfo + capability name *capname* as a bytes object. Return ``None`` if *capname* + is not a terminfo "string capability", or is canceled or absent from the + terminal description. + + :func:`setupterm` (or :func:`initscr`) must be called first. + +.. function:: tparm(str[, ...]) + + Instantiate the bytes object *str* with the supplied parameters, where *str* should + be a parameterized byte string obtained from the terminfo database. For example, + ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact + result depending on terminal type. Up to nine integer parameters may be supplied. + + :func:`setupterm` (or :func:`initscr`) must be called first. + +.. function:: putp(str) + + Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified + terminfo capability, a bytes object, for the current terminal. + Note that the output of :func:`putp` always goes to standard output. + + :func:`setupterm` (or :func:`initscr`) must be called first. + + +Utilities +~~~~~~~~~ + +.. function:: unctrl(ch) + + Return a bytes object which is a printable representation of the character *ch*; + any attributes and color pair are ignored. + Control characters are represented as a caret followed by a character, + for example as ``b'^C'``. + Printing characters are left as they are. + The representation of other characters is defined by the underlying curses + library. + + *ch* must fit in a single byte; use :func:`wunctrl` for other characters. + +.. function:: wunctrl(ch) + + Return a string which is a printable representation of the character *ch*; + any attributes and color pair are ignored. + ASCII control characters are represented as a caret followed by a character, + for example as ``'^C'``. Printing characters, including non-ASCII characters + printable in the locale, are left as they are. The representation of other + characters is defined by the underlying curses library. + + .. versionadded:: next + +.. function:: filter() + + The :func:`.filter` routine, if used, must be called before :func:`initscr` is + called. The effect is that, during the initialization, :envvar:`LINES` is set to ``1``; the + capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` + string is set to the value of ``cr``. The effect is that the cursor is confined to + the current line, and so are screen updates. This may be used for enabling + character-at-a-time line editing without touching the rest of the screen. + +.. function:: nofilter() + + Undo the effect of a previous :func:`.filter` call. + Like :func:`.filter`, it must be called before :func:`initscr` (or + :func:`newterm`) so that the next initialization uses the full screen + again. + Availability: if the underlying curses library provides ``nofilter()``. + + .. versionadded:: next .. function:: use_env(flag) @@ -692,95 +1094,443 @@ The module :mod:`!curses` defines the following functions: window (in which case default behavior would be to use the window size if :envvar:`LINES` and :envvar:`COLUMNS` are not set). +.. function:: get_escdelay() -.. function:: use_default_colors() + Retrieves the value set by :func:`set_escdelay`. - Equivalent to ``assume_default_colors(-1, -1)``. + .. versionadded:: 3.9 + +.. function:: set_escdelay(ms) + + Sets the number of milliseconds to wait after reading an escape character, + to distinguish between an individual escape character entered on the + keyboard from escape sequences sent by cursor and function keys. + + .. versionadded:: 3.9 + +.. function:: get_tabsize() + + Retrieves the value set by :func:`set_tabsize`. + + .. versionadded:: 3.9 + +.. function:: set_tabsize(size) + + Sets the number of columns used by the curses library when converting a tab + character to spaces as it adds the tab to a window. + + .. versionadded:: 3.9 + +.. function:: napms(ms) + + Sleep for *ms* milliseconds. + +.. function:: delay_output(ms) + + Insert an *ms* millisecond pause in output. + +.. _curses-window-objects: + +Window objects +-------------- + +.. class:: window + + Window objects, as returned by :func:`initscr` and :func:`newwin` above, have + the following methods and attributes: + + +Adding and inserting text +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.addch(ch[, attr]) + window.addch(y, x, ch[, attr]) + + Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any + character previously painted at that location. By default, the character + position and attributes are the current settings for the window object. + + *ch* may be a single character, optionally followed by combining + characters, that together occupy one character cell. + + .. note:: + + Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. + Attempting to write to the lower-right corner of a window, subwindow, + or pad will cause an exception to be raised after the character is printed. + + .. versionchanged:: next + A character may now be given as a string of a base character followed + by combining characters, instead of only a single character, or as a + :class:`complexchar` cell. + +.. method:: window.addstr(str[, attr]) + window.addstr(y, x, str[, attr]) + + Paint the character string *str* at ``(y, x)`` with attributes + *attr*, overwriting anything previously on the display. + + *str* may also be a :class:`complexstr`, in which case each cell carries its + own attributes and color pair, so *attr* must not be given. A + :class:`complexstr` obtained from :meth:`in_wchstr` is written back + unchanged. + + .. note:: + + * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. + Attempting to write to the lower-right corner of a window, subwindow, + or pad will cause an exception to be raised after the string is printed. + + * A bug in ncurses, the backend for this Python module, could cause + segfaults when resizing windows. This was fixed in ncurses-6.1-20190511. + If you are stuck with an earlier ncurses, you can avoid triggering it by + not calling :meth:`!addstr` with a *str* that has embedded newlines; + instead, call :meth:`!addstr` separately for each line. + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`, as described above. + +.. method:: window.addnstr(str, n[, attr]) + window.addnstr(y, x, str, n[, attr]) + + Paint at most *n* characters of the character string *str* at + ``(y, x)`` with attributes + *attr*, overwriting anything previously on the display. + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`; see :meth:`addstr`. + +.. method:: window.echochar(ch[, attr]) + + Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` + on the window. + + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. + +.. method:: window.insch(ch[, attr]) + window.insch(y, x, ch[, attr]) + + Insert character *ch* with attributes *attr* before the character under the + cursor, or at ``(y, x)`` if specified. All characters to the right of the + cursor are shifted one position right, with the rightmost character on the + line being lost. The cursor position does not change. + + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. + +.. method:: window.insstr(str[, attr]) + window.insstr(y, x, str[, attr]) + + Insert a character string (as many characters as will fit on the line) before + the character under the cursor. All characters to the right of the cursor are + shifted right, with the rightmost characters on the line being lost. The cursor + position does not change (after moving to *y*, *x*, if specified). + + *str* may also be a :class:`complexstr`, in which case each cell carries its + own attributes and color pair, so *attr* must not be given. + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`, as described above. + +.. method:: window.insnstr(str, n[, attr]) + window.insnstr(y, x, str, n[, attr]) + + Insert a character string (as many characters as will fit on the line) before + the character under the cursor, up to *n* characters. If *n* is zero or + negative, the entire string is inserted. All characters to the right of the + cursor are shifted right, with the rightmost characters on the line being lost. + The cursor position does not change (after moving to *y*, *x*, if specified). + + .. versionchanged:: next + *str* may now also be a :class:`complexstr`; see :meth:`insstr`. + + +Deleting and inserting lines +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.delch([y, x]) + + Delete the character under the cursor, or at ``(y, x)`` if specified. All + characters to the right on the same line are shifted one position left. + +.. method:: window.deleteln() + + Delete the line under the cursor. All following lines are moved up by one line. + +.. method:: window.insertln() + + Insert a blank line under the cursor. All following lines are moved down by one + line. + +.. method:: window.insdelln(nlines) + + Insert *nlines* lines into the specified window above the current line. The + *nlines* bottom lines are lost. For negative *nlines*, delete *nlines* lines + starting with the one under the cursor, and move the remaining lines up. The + bottom *nlines* lines are cleared. The current cursor position remains the + same. + + +Reading input +~~~~~~~~~~~~~ + +.. method:: window.getch([y, x]) + + Read a key press, after moving the cursor to *y*, *x* if specified, + and return it as an integer. + The window is refreshed first if it is not a pad and was modified since + the last refresh. + Wait until a key is pressed, or return ``-1`` if the read is non-blocking + or times out (see :meth:`nodelay` and :meth:`timeout`). + + An ordinary key is returned as the code of a single byte of its encoding + in the current locale, + so a character encoded with several bytes takes several calls. + For example, in a UTF-8 locale ``'é'`` is read as ``195``, then ``169``. + Use :meth:`get_wch` to read it as a single character. + + In keypad mode (see :meth:`keypad`) function keys and other special keys + are returned as one of the :ref:`KEY_* constants `, + which cannot be mistaken for an ordinary key. + Otherwise, or if their escape sequence does not arrive in time + (see :meth:`notimeout` and :func:`set_escdelay`), + their bytes are returned one at a time. + + In echo mode (see :func:`echo`) the key is added to the window as by + :meth:`addch`; special keys are not echoed. + +.. method:: window.get_wch([y, x]) + + Read a key press, after moving the cursor to *y*, *x* if specified, + and return it as a one-character :class:`str`. + The window is refreshed first if it is not a pad and was modified since + the last refresh. + Wait until a key is pressed, or raise :exc:`error` if the read is + non-blocking or times out (see :meth:`nodelay` and :meth:`timeout`). + + In keypad mode (see :meth:`keypad`) function keys and other special keys + are returned as one of the :ref:`KEY_* constants `, + an integer. + Otherwise, or if their escape sequence does not arrive in time + (see :meth:`notimeout` and :func:`set_escdelay`), + their characters are returned one at a time. + + In echo mode (see :func:`echo`) the key is added to the window as by + :meth:`addch`; special keys are not echoed. + + .. versionadded:: 3.3 + + .. versionchanged:: next + Also available on a narrow build, where only a character representable as a + single byte (an 8-bit locale) can be returned. + +.. method:: window.getkey([y, x]) + + Read a key press as :meth:`getch` does, but return it as a :class:`str`: + an ordinary key as a one-character string, the byte decoded as Latin-1, + and a special key as its name, such as ``'KEY_UP'`` (see :func:`keyname`). + Raise :exc:`error` instead of returning ``-1`` if there is no input. + +.. method:: window.getstr() + window.getstr(n) + window.getstr(y, x) + window.getstr(y, x, n) + + Read a line of input from the user, with primitive line editing capacity, + after moving the cursor to *y*, *x* if specified. + Return it as a bytes object, in the encoding of the current locale + and without the terminating newline. + At most *n* bytes are read; + *n* defaults to and cannot exceed 2047. + + Use :meth:`get_wstr` to read the input as a :class:`str`. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. + +.. method:: window.get_wstr() + window.get_wstr(n) + window.get_wstr(y, x) + window.get_wstr(y, x, n) + + Read a line of input from the user, with primitive line editing capacity, + after moving the cursor to *y*, *x* if specified. + Return it as a :class:`str`, without the terminating newline. + At most *n* characters are read; + *n* defaults to and cannot exceed 2047. + + This is the wide-character variant of :meth:`getstr`. + + .. versionadded:: next + +.. method:: window.getdelay() + + Return the window's read timeout in milliseconds, + as set by :meth:`nodelay` or :meth:`timeout`: + ``-1`` for blocking, ``0`` for non-blocking, + or a positive number of milliseconds. + + .. versionadded:: next + + +Reading window contents +~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.inch([y, x]) + + Return the character at the given position in the window. + The bottom 8 bits are the character proper and the upper bits are the attributes; + extract them with the :data:`A_CHARTEXT` and :data:`A_ATTRIBUTES` bit-masks, + and the color pair with :func:`pair_number`. + The character byte is the locale-encoded byte of the cell's character, + consistent with :meth:`instr`. + It cannot represent a cell holding combining characters, a character that does + not fit in a single byte, or a color pair outside the :func:`color_pair` + range; use :meth:`in_wch` for those, which returns it as a :class:`complexchar`. + +.. method:: window.in_wch([y, x]) + + Return the complex character at the given position in the window as a + :class:`complexchar`. Unlike :meth:`inch`, the returned object carries the + cell's text (a spacing character optionally followed by combining characters) + together with its attributes and color pair. + + .. versionadded:: next + +.. method:: window.instr([n]) + window.instr(y, x[, n]) + + Read the text of the window from the current cursor position, + or from *y*, *x* if specified, to the end of the line + or at most *n* bytes if *n* is specified, + and return it as a bytes object, in the encoding of the current locale. + Attributes and color pairs are stripped; + use :meth:`in_wchstr` to read them too. + A character not representable in the encoding cannot be returned; + use :meth:`in_wstr` for those. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. + + .. versionchanged:: next + *n* is no longer limited to 2047. + +.. method:: window.in_wstr([n]) + window.in_wstr(y, x[, n]) + + Read the text of the window from the current cursor position, + or from *y*, *x* if specified, to the end of the line + or at most *n* characters if *n* is specified, + and return it as a :class:`str`. + Attributes and color pairs are stripped; + use :meth:`in_wchstr` to read them too. + + This is the wide-character variant of :meth:`instr`. + + .. versionadded:: next + +.. method:: window.in_wchstr([n]) + window.in_wchstr(y, x[, n]) + + Read the styled cells of the window from the current cursor position, + or from *y*, *x* if specified, to the end of the line + or at most *n* cells if *n* is specified, + and return them as a :class:`complexstr`. + Unlike :meth:`instr` and :meth:`in_wstr`, each cell keeps its attributes + and color pair, so the result can be written back unchanged + with :meth:`addstr`. + + .. versionadded:: next + + +Attributes +~~~~~~~~~~ + +.. method:: window.attroff(attr) + Remove attribute *attr* from the "background" set applied to all writes to the + current window. -.. function:: wrapper(func, /, *args, **kwargs) +.. method:: window.attron(attr) - Initialize curses and call another callable object, *func*, which should be the - rest of your curses-using application. If the application raises an exception, - this function will restore the terminal to a sane state before re-raising the - exception and generating a traceback. The callable object *func* is then passed - the main window 'stdscr' as its first argument, followed by any other arguments - passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on - cbreak mode, turns off echo, enables the terminal keypad, and initializes colors - if the terminal has color support. On exit (whether normally or by exception) - it restores cooked mode, turns on echo, and disables the terminal keypad. + Add attribute *attr* to the "background" set applied to all writes to the + current window. +.. method:: window.attrset(attr) -.. _curses-window-objects: + Set the "background" set of attributes to *attr*. This set is initially + ``0`` (no attributes). -Window Objects --------------- +.. method:: window.attr_get() -.. class:: window + Return the window's current rendition as a ``(attrs, pair)`` tuple, + where *attrs* is the set of attributes and *pair* is the color pair number. - Window objects, as returned by :func:`initscr` and :func:`newwin` above, have - the following methods and attributes: + Unlike :meth:`attron` and friends, which take packed ``A_*`` attributes, + this method and the other ``attr_*`` methods work with the + :ref:`WA_* attributes ` and keep the color pair as a + separate number, which lets them use color pairs that do not fit alongside + the attributes in a single value. + .. versionadded:: next -.. method:: window.addch(ch[, attr]) - window.addch(y, x, ch[, attr]) +.. method:: window.attr_set(attr, pair=0) - Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any - character previously painted at that location. By default, the character - position and attributes are the current settings for the window object. + Set the window's rendition to the attributes *attr* and the color pair *pair*. - .. note:: + .. versionadded:: next - Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the character is printed. +.. method:: window.attr_on(attr) + Turn on the attributes *attr* without affecting any others. -.. method:: window.addnstr(str, n[, attr]) - window.addnstr(y, x, str, n[, attr]) + .. versionadded:: next - Paint at most *n* characters of the character string *str* at - ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. +.. method:: window.attr_off(attr) + Turn off the attributes *attr* without affecting any others. -.. method:: window.addstr(str[, attr]) - window.addstr(y, x, str[, attr]) + .. versionadded:: next - Paint the character string *str* at ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. +.. method:: window.color_set(pair) - .. note:: + Set the window's color pair to *pair*, leaving the other attributes unchanged. - * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the string is printed. + .. versionadded:: next - * A `bug in ncurses `_, the backend - for this Python module, can cause SegFaults when resizing windows. This - is fixed in ncurses-6.1-20190511. If you are stuck with an earlier - ncurses, you can avoid triggering this if you do not call :func:`addstr` - with a *str* that has embedded newlines. Instead, call :func:`addstr` - separately for each line. +.. method:: window.getattrs() + Return the window's current attributes. -.. method:: window.attroff(attr) + .. versionadded:: next - Remove attribute *attr* from the "background" set applied to all writes to the - current window. +.. method:: window.standout() + Turn on attribute *A_STANDOUT*. -.. method:: window.attron(attr) +.. method:: window.standend() - Add attribute *attr* to the "background" set applied to all writes to the - current window. + Turn off the standout attribute. On some terminals this has the side effect of + turning off all attributes. +.. method:: window.chgat(attr) + window.chgat(num, attr) + window.chgat(y, x, attr) + window.chgat(y, x, num, attr) -.. method:: window.attrset(attr) + Set the attributes of *num* characters at the current cursor position, or at + position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, + the attribute will be set on all the characters to the end of the line. This + function moves cursor to position ``(y, x)`` if supplied. The changed line + will be touched using the :meth:`touchline` method so that the contents will + be redisplayed by the next window refresh. - Set the "background" set of attributes to *attr*. This set is initially - ``0`` (no attributes). +Background +~~~~~~~~~~ .. method:: window.bkgd(ch[, attr]) @@ -794,6 +1544,9 @@ Window Objects * Wherever the former background character appears, it is changed to the new background character. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. .. method:: window.bkgdset(ch[, attr]) @@ -804,6 +1557,51 @@ Window Objects characters. The background becomes a property of the character and moves with the character through any scrolling and insert/delete line/character operations. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. + +.. method:: window.getbkgd() + + Return the given window's current background character/attribute pair. + Its components can be extracted like those of :meth:`inch`. + It cannot represent a background set with a wide character or with a color + pair outside the :func:`color_pair` range; use :meth:`getbkgrnd` for those. + +.. method:: window.getbkgrnd() + + Return the given window's current background as a :class:`complexchar`. + Unlike :meth:`getbkgd`, the returned object carries the background character + together with its attributes and color pair, and the color pair is not limited + to the value that fits in a :func:`color_pair`. + + .. versionadded:: next + + +Clearing and erasing +~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.erase() + + Clear the window. + +.. method:: window.clear() + + Like :meth:`erase`, but also cause the whole window to be repainted upon next + call to :meth:`refresh`. + +.. method:: window.clrtobot() + + Erase from cursor to the end of the window: all lines below the cursor are + deleted, and then the equivalent of :meth:`clrtoeol` is performed. + +.. method:: window.clrtoeol() + + Erase from cursor to the end of the line. + + +Borders and lines +~~~~~~~~~~~~~~~~~ .. method:: window.border([ls[, rs[, ts[, bs[, tl[, tr[, bl[, br]]]]]]]]) @@ -817,84 +1615,115 @@ Window Objects that parameter. Keyword parameters can *not* be used. The defaults are listed in this table: - +-----------+---------------------+-----------------------+ - | Parameter | Description | Default value | - +===========+=====================+=======================+ - | *ls* | Left side | :const:`ACS_VLINE` | - +-----------+---------------------+-----------------------+ - | *rs* | Right side | :const:`ACS_VLINE` | - +-----------+---------------------+-----------------------+ - | *ts* | Top | :const:`ACS_HLINE` | - +-----------+---------------------+-----------------------+ - | *bs* | Bottom | :const:`ACS_HLINE` | - +-----------+---------------------+-----------------------+ - | *tl* | Upper-left corner | :const:`ACS_ULCORNER` | - +-----------+---------------------+-----------------------+ - | *tr* | Upper-right corner | :const:`ACS_URCORNER` | - +-----------+---------------------+-----------------------+ - | *bl* | Bottom-left corner | :const:`ACS_LLCORNER` | - +-----------+---------------------+-----------------------+ - | *br* | Bottom-right corner | :const:`ACS_LRCORNER` | - +-----------+---------------------+-----------------------+ - + +-----------+---------------------+-----------------------+------------------------+ + | Parameter | Description | Default value | Wide default value | + +===========+=====================+=======================+========================+ + | *ls* | Left side | :const:`ACS_VLINE` | :const:`WACS_VLINE` | + +-----------+---------------------+-----------------------+------------------------+ + | *rs* | Right side | :const:`ACS_VLINE` | :const:`WACS_VLINE` | + +-----------+---------------------+-----------------------+------------------------+ + | *ts* | Top | :const:`ACS_HLINE` | :const:`WACS_HLINE` | + +-----------+---------------------+-----------------------+------------------------+ + | *bs* | Bottom | :const:`ACS_HLINE` | :const:`WACS_HLINE` | + +-----------+---------------------+-----------------------+------------------------+ + | *tl* | Upper-left corner | :const:`ACS_ULCORNER` | :const:`WACS_ULCORNER` | + +-----------+---------------------+-----------------------+------------------------+ + | *tr* | Upper-right corner | :const:`ACS_URCORNER` | :const:`WACS_URCORNER` | + +-----------+---------------------+-----------------------+------------------------+ + | *bl* | Bottom-left corner | :const:`ACS_LLCORNER` | :const:`WACS_LLCORNER` | + +-----------+---------------------+-----------------------+------------------------+ + | *br* | Bottom-right corner | :const:`ACS_LRCORNER` | :const:`WACS_LRCORNER` | + +-----------+---------------------+-----------------------+------------------------+ + + The wide default value is used when the border is drawn from string + characters or :class:`complexchar` cells. + + If any parameter is a byte character or an integer other than ``0``, the + border is drawn from byte characters, and every string character must be + encodable as a single byte. + + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. A single call cannot mix + :class:`complexchar` cells with integer or byte characters. .. method:: window.box([vertch, horch]) Similar to :meth:`border`, but both *ls* and *rs* are *vertch* and both *ts* and *bs* are *horch*. The default corner characters are always used by this function. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. A single call cannot mix + :class:`complexchar` cells with integer or byte characters. -.. method:: window.chgat(attr) - window.chgat(num, attr) - window.chgat(y, x, attr) - window.chgat(y, x, num, attr) +.. method:: window.hline(ch, n[, attr]) + window.hline(y, x, ch, n[, attr]) - Set the attributes of *num* characters at the current cursor position, or at - position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, - the attribute will be set on all the characters to the end of the line. This - function moves cursor to position ``(y, x)`` if supplied. The changed line - will be touched using the :meth:`touchline` method so that the contents will - be redisplayed by the next window refresh. + Display a horizontal line starting at ``(y, x)`` with length *n* consisting of + the character *ch* with attributes *attr*. The line stops at the right edge + of the window if fewer than *n* cells are available. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. -.. method:: window.clear() +.. method:: window.vline(ch, n[, attr]) + window.vline(y, x, ch, n[, attr]) - Like :meth:`erase`, but also cause the whole window to be repainted upon next - call to :meth:`refresh`. + Display a vertical line starting at ``(y, x)`` with length *n* consisting of the + character *ch* with attributes *attr*. + .. versionchanged:: next + Wide and combining characters, and :class:`complexchar` cells, are now + accepted. -.. method:: window.clearok(flag) - If *flag* is ``True``, the next call to :meth:`refresh` will clear the window - completely. +Cursor position and window geometry +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: window.move(new_y, new_x) -.. method:: window.clrtobot() + Move cursor to ``(new_y, new_x)``. - Erase from cursor to the end of the window: all lines below the cursor are - deleted, and then the equivalent of :meth:`clrtoeol` is performed. +.. method:: window.getyx() + Return a tuple ``(y, x)`` of current cursor position relative to the window's + upper-left corner. -.. method:: window.clrtoeol() +.. method:: window.getbegyx() - Erase from cursor to the end of the line. + Return a tuple ``(y, x)`` of coordinates of upper-left corner. +.. method:: window.getmaxyx() -.. method:: window.cursyncup() + Return a tuple ``(y, x)`` of the height and width of the window. - Update the current cursor position of all the ancestors of the window to - reflect the current cursor position of the window. +.. method:: window.getparyx() + Return the beginning coordinates of this window relative to its parent window + as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no + parent. -.. method:: window.delch([y, x]) +.. method:: window.getparent() - Delete any character at ``(y, x)``. + Return the parent window of this subwindow, + or ``None`` if this window is not a subwindow. + .. versionadded:: next -.. method:: window.deleteln() - Delete the line under the cursor. All following lines are moved up by one line. +Creating and resizing windows +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. method:: window.subwin(begin_y, begin_x) + window.subwin(nlines, ncols, begin_y, begin_x) + + Return a sub-window, whose upper-left corner is at the screen-relative + coordinates ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. + By default, the sub-window will extend from the specified position to the lower + right corner of the window. .. method:: window.derwin(begin_y, begin_x) window.derwin(nlines, ncols, begin_y, begin_x) @@ -904,435 +1733,529 @@ Window Objects of the window, rather than relative to the entire screen. Return a window object for the derived window. +.. method:: window.subpad(begin_y, begin_x) + window.subpad(nlines, ncols, begin_y, begin_x) -.. method:: window.echochar(ch[, attr]) + Return a sub-pad, whose upper-left corner is at ``(begin_y, begin_x)``, and + whose width/height is *ncols*/*nlines*. The coordinates are relative to the + parent pad (unlike :meth:`subwin`, which uses screen coordinates). This + method is only available for pads created with :func:`newpad`. - Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` - on the window. +.. method:: window.dupwin() + Return a new window that is an exact duplicate of the window: it has the same + size, position, contents and attributes. Unlike a window created by + :meth:`subwin` or :meth:`derwin`, the duplicate is independent of the + original -- it has its own cell buffer, so later changes to one do not affect + the other. -.. method:: window.enclose(y, x) + .. versionadded:: next - Test whether the given pair of screen-relative character-cell coordinates are - enclosed by the given window, returning ``True`` or ``False``. It is useful for - determining what subset of the screen windows enclose the location of a mouse - event. +.. method:: window.mvwin(new_y, new_x) - .. versionchanged:: 3.10 - Previously it returned ``1`` or ``0`` instead of ``True`` or ``False``. + Move the window so its upper-left corner is at ``(new_y, new_x)``. + Moving the window so that any part of it would be off the screen is an error: + the window is not moved and :exc:`curses.error` is raised. -.. attribute:: window.encoding +.. method:: window.mvderwin(y, x) - Encoding used to encode method arguments (Unicode strings and characters). - The encoding attribute is inherited from the parent window when a subwindow - is created, for example with :meth:`window.subwin`. - By default, current locale encoding is used (see :func:`locale.getencoding`). + Move the window inside its parent window. The screen-relative parameters of + the window are not changed. This routine is used to display different parts of + the parent window at the same physical position on the screen. - .. versionadded:: 3.3 +.. method:: window.resize(nlines, ncols) + Reallocate storage for a curses window to adjust its dimensions to the + specified values. If either dimension is larger than the current values, the + window's data is filled with blanks that have the current background + rendition (as set by :meth:`bkgdset`) merged into them. -.. method:: window.erase() - Clear the window. +Refreshing and redrawing +~~~~~~~~~~~~~~~~~~~~~~~~ +.. method:: window.refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) -.. method:: window.getbegyx() + Update the display immediately (sync actual screen with previous + drawing/deleting methods). - Return a tuple ``(y, x)`` of coordinates of upper-left corner. + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`. The additional parameters are needed to indicate what part + of the pad and screen are involved. *pminrow* and *pmincol* specify the + upper-left corner of the rectangle to be displayed in the pad. *sminrow*, + *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be + displayed on the screen. The lower-right corner of the rectangle to be + displayed in the pad is calculated from the screen coordinates, since the + rectangles must be the same size. Both rectangles must be entirely contained + within their respective structures. Negative values of *pminrow*, *pmincol*, + *sminrow*, or *smincol* are treated as if they were zero. +.. method:: window.noutrefresh() + window.noutrefresh(pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol) -.. method:: window.getbkgd() + Mark for refresh but wait. This function updates the data structure + representing the desired state of the window, but does not force an update of + the physical screen. To accomplish that, call :func:`doupdate`. - Return the given window's current background character/attribute pair. + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`; they have the same meaning as for + :meth:`refresh`. +.. method:: window.redrawln(beg, num) -.. method:: window.getch([y, x]) + Indicate that the *num* screen lines, starting at line *beg*, are corrupted and + should be completely redrawn on the next :meth:`refresh` call. - Get a character. Note that the integer returned does *not* have to be in ASCII - range: function keys, keypad keys and so on are represented by numbers higher - than 255. In no-delay mode, return ``-1`` if there is no input, otherwise - wait until a key is pressed. +.. method:: window.redrawwin() + Touch the entire window, causing it to be completely redrawn on the next + :meth:`refresh` call. -.. method:: window.get_wch([y, x]) - Get a wide character. Return a character for most keys, or an integer for - function keys, keypad keys, and other special keys. - In no-delay mode, raise an exception if there is no input. +Output options +~~~~~~~~~~~~~~ - .. versionadded:: 3.3 +.. method:: window.clearok(flag) + If *flag* is ``True``, the next call to :meth:`refresh` will clear the window + completely. -.. method:: window.getkey([y, x]) +.. method:: window.idlok(flag) - Get a character, returning a string instead of an integer, as :meth:`getch` - does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, raise an exception if - there is no input. + If *flag* is ``True``, :mod:`!curses` will try to use hardware line + editing facilities. Otherwise, curses will not use them. +.. method:: window.idcok(flag) -.. method:: window.getmaxyx() + If *flag* is ``False``, curses no longer considers using the hardware insert/delete + character feature of the terminal; if *flag* is ``True``, use of character insertion + and deletion is enabled. When curses is first initialized, use of character + insert/delete is enabled by default. - Return a tuple ``(y, x)`` of the height and width of the window. +.. method:: window.immedok(flag) + If *flag* is ``True``, any change in the window image automatically causes the + window to be refreshed; you no longer have to call :meth:`refresh` yourself. + However, it may degrade performance considerably, due to repeated calls to + wrefresh. This option is disabled by default. -.. method:: window.getparyx() +.. method:: window.leaveok(flag) - Return the beginning coordinates of this window relative to its parent window - as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no - parent. + If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor + position." This reduces cursor movement where possible. + If *flag* is ``False``, cursor will always be at "cursor position" after an update. -.. method:: window.getstr() - window.getstr(n) - window.getstr(y, x) - window.getstr(y, x, n) +.. method:: window.scrollok(flag) - Read a bytes object from the user, with primitive line editing capacity. - The maximum value for *n* is 2047. + Control what happens when the cursor of a window is moved off the edge of the + window or scrolling region, either as a result of a newline action on the bottom + line, or typing the last character of the last line. If *flag* is ``False``, the + cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up + one line. Note that in order to get the physical scrolling effect on the + terminal, it is also necessary to call :meth:`idlok`. - .. versionchanged:: 3.14 - The maximum value for *n* was increased from 1023 to 2047. +.. method:: window.scroll([lines=1]) + Scroll the screen or scrolling region. Scroll upward by *lines* lines if + *lines* is positive, or downward if it is negative. Scrolling has no effect + unless it has been enabled for the window with :meth:`scrollok`. -.. method:: window.getyx() +.. method:: window.setscrreg(top, bottom) - Return a tuple ``(y, x)`` of current cursor position relative to the window's - upper-left corner. + Set the scrolling region from line *top* to line *bottom*. All scrolling actions + will take place in this region. +.. method:: window.getscrreg() -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) + Return a tuple ``(top, bottom)`` of the window's current scrolling region, + as set by :meth:`setscrreg`. - Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. + .. versionadded:: next + +.. method:: window.syncok(flag) + + If *flag* is ``True``, then :meth:`syncup` is called automatically + whenever there is a change in the window. + + +Input options +~~~~~~~~~~~~~ +.. method:: window.keypad(flag) -.. method:: window.idcok(flag) + If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) + will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be + left as is in the input stream. + Keypad mode is disabled by default, but :func:`wrapper` enables it for the + main window. - If *flag* is ``False``, curses no longer considers using the hardware insert/delete - character feature of the terminal; if *flag* is ``True``, use of character insertion - and deletion is enabled. When curses is first initialized, use of character - insert/delete is enabled by default. +.. method:: window.nodelay(flag) + If *flag* is ``True``, :meth:`getch` will be non-blocking. -.. method:: window.idlok(flag) +.. method:: window.notimeout(flag) - If *flag* is ``True``, :mod:`!curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. + If *flag* is ``True``, escape sequences will not be timed out. + If *flag* is ``False``, after a few milliseconds, an escape sequence will not be + interpreted, and will be left in the input stream as is. -.. method:: window.immedok(flag) +.. method:: window.timeout(delay) - If *flag* is ``True``, any change in the window image automatically causes the - window to be refreshed; you no longer have to call :meth:`refresh` yourself. - However, it may degrade performance considerably, due to repeated calls to - wrefresh. This option is disabled by default. + Set blocking or non-blocking read behavior for the window. If *delay* is + negative, blocking read is used (which will wait indefinitely for input). If + *delay* is zero, then non-blocking read is used, and :meth:`getch` will + return ``-1`` if no input is waiting. If *delay* is positive, then + :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is + still no input at the end of that time. -.. method:: window.inch([y, x]) +Overlapping and touch +~~~~~~~~~~~~~~~~~~~~~ - Return the character at the given position in the window. The bottom 8 bits are - the character proper, and upper bits are the attributes. +.. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) + Overlay the window on top of *destwin*. The windows need not be the same size, + only the overlapping region is copied. This copy is non-destructive, which means + that the current background character does not overwrite the old contents of + *destwin*. -.. method:: window.insch(ch[, attr]) - window.insch(y, x, ch[, attr]) + To get fine-grained control over the copied region, the second form of + :meth:`overlay` can be used. *sminrow* and *smincol* are the upper-left + coordinates of the source window, and the other variables mark a rectangle in + the destination window. - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. +.. method:: window.overwrite(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) + Overwrite the window on top of *destwin*. The windows need not be the same size, + in which case only the overlapping region is copied. This copy is destructive, + which means that the current background character overwrites the old contents of + *destwin*. -.. method:: window.insdelln(nlines) + To get fine-grained control over the copied region, the second form of + :meth:`overwrite` can be used. *sminrow* and *smincol* are the upper-left + coordinates of the source window, the other variables mark a rectangle in the + destination window. - Insert *nlines* lines into the specified window above the current line. The - *nlines* bottom lines are lost. For negative *nlines*, delete *nlines* lines - starting with the one under the cursor, and move the remaining lines up. The - bottom *nlines* lines are cleared. The current cursor position remains the - same. +.. method:: window.touchline(start, count[, changed]) + Pretend *count* lines have been changed, starting with line *start*. If + *changed* is supplied, it specifies whether the affected lines are marked as + having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). -.. method:: window.insertln() +.. method:: window.touchwin() - Insert a blank line under the cursor. All following lines are moved down by one - line. + Pretend the whole window has been changed, for purposes of drawing + optimizations. +.. method:: window.untouchwin() -.. method:: window.insnstr(str, n[, attr]) - window.insnstr(y, x, str, n[, attr]) + Mark all lines in the window as unchanged since the last call to + :meth:`refresh`. - Insert a character string (as many characters as will fit on the line) before - the character under the cursor, up to *n* characters. If *n* is zero or - negative, the entire string is inserted. All characters to the right of the - cursor are shifted right, with the rightmost characters on the line being lost. - The cursor position does not change (after moving to *y*, *x*, if specified). +.. method:: window.syncup() + Touch all locations in ancestors of the window that have been changed in the + window. -.. method:: window.insstr(str[, attr]) - window.insstr(y, x, str[, attr]) +.. method:: window.syncdown() - Insert a character string (as many characters as will fit on the line) before - the character under the cursor. All characters to the right of the cursor are - shifted right, with the rightmost characters on the line being lost. The cursor - position does not change (after moving to *y*, *x*, if specified). + Touch each location in the window that has been touched in any of its ancestor + windows. This routine is called by :meth:`refresh`, so it should almost never + be necessary to call it manually. +.. method:: window.cursyncup() -.. method:: window.instr([n]) - window.instr(y, x[, n]) + Update the current cursor position of all the ancestors of the window to + reflect the current cursor position of the window. - Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped - from the characters. If *n* is specified, :meth:`instr` returns a string - at most *n* characters long (exclusive of the trailing NUL). - The maximum value for *n* is 2047. - .. versionchanged:: 3.14 - The maximum value for *n* was increased from 1023 to 2047. +Coordinate conversion +~~~~~~~~~~~~~~~~~~~~~ +.. method:: window.enclose(y, x) -.. method:: window.is_linetouched(line) + Test whether the given pair of screen-relative character-cell coordinates are + enclosed by the given window, returning ``True`` or ``False``. It is useful for + determining what subset of the screen windows enclose the location of a mouse + event. - Return ``True`` if the specified line was modified since the last call to - :meth:`refresh`; otherwise return ``False``. Raise a :exc:`curses.error` - exception if *line* is not valid for the given window. + .. versionchanged:: 3.10 + Previously it returned ``1`` or ``0`` instead of ``True`` or ``False``. +.. method:: window.mouse_trafo(y, x, to_screen) -.. method:: window.is_wintouched() + Convert between window-relative and screen-relative (``stdscr``-relative) character-cell coordinates. + If *to_screen* is true, convert the window-relative coordinates *y*, *x* to screen-relative coordinates; + otherwise convert in the opposite direction. + The two coordinate systems differ when lines are reserved on the screen, for example for soft labels. - Return ``True`` if the specified window was modified since the last call to - :meth:`refresh`; otherwise return ``False``. + Return the converted coordinates as a ``(y, x)`` tuple, or ``None`` if they lie outside the window. + .. versionadded:: next -.. method:: window.keypad(flag) - If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be - left as is in the input stream. +Other +~~~~~ +.. attribute:: window.encoding -.. method:: window.leaveok(flag) + Encoding used to encode the string arguments of the methods and to decode + their results on a build without wide-character support. + The encoding attribute is inherited from the parent window when a subwindow + is created, for example with :meth:`window.subwin`. + By default, current locale encoding is used (see :func:`locale.getencoding`). - If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. + .. versionadded:: 3.3 - If *flag* is ``False``, cursor will always be at "cursor position" after an update. +.. method:: window.putwin(file) + Write all data associated with the window into the provided file object. This + information can be later retrieved using the :func:`getwin` function. -.. method:: window.move(new_y, new_x) +.. method:: window.use(func, /, *args, **kwargs) - Move cursor to ``(new_y, new_x)``. + Call ``func(window, *args, **kwargs)`` with the lock of the window held, + and return its result. + This provides automatic protection for the window + against concurrent access from another thread. + Availability: if the underlying curses library provides ``use_window()``. -.. method:: window.mvderwin(y, x) + .. versionadded:: next - Move the window inside its parent window. The screen-relative parameters of - the window are not changed. This routine is used to display different parts of - the parent window at the same physical position on the screen. +State queries +~~~~~~~~~~~~~ -.. method:: window.mvwin(new_y, new_x) +.. method:: window.is_cleared() - Move the window so its upper-left corner is at ``(new_y, new_x)``. + Return the current value set by :meth:`clearok`. + .. versionadded:: next -.. method:: window.nodelay(flag) +.. method:: window.is_idcok() - If *flag* is ``True``, :meth:`getch` will be non-blocking. + Return the current value set by :meth:`idcok`. + .. versionadded:: next -.. method:: window.notimeout(flag) +.. method:: window.is_idlok() - If *flag* is ``True``, escape sequences will not be timed out. + Return the current value set by :meth:`idlok`. - If *flag* is ``False``, after a few milliseconds, an escape sequence will not be - interpreted, and will be left in the input stream as is. + .. versionadded:: next +.. method:: window.is_immedok() -.. method:: window.noutrefresh() + Return the current value set by :meth:`immedok`. - Mark for refresh but wait. This function updates the data structure - representing the desired state of the window, but does not force an update of - the physical screen. To accomplish that, call :func:`doupdate`. + .. versionadded:: next +.. method:: window.is_keypad() -.. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) + Return the current value set by :meth:`keypad`. - Overlay the window on top of *destwin*. The windows need not be the same size, - only the overlapping region is copied. This copy is non-destructive, which means - that the current background character does not overwrite the old contents of - *destwin*. + .. versionadded:: next - To get fine-grained control over the copied region, the second form of - :meth:`overlay` can be used. *sminrow* and *smincol* are the upper-left - coordinates of the source window, and the other variables mark a rectangle in - the destination window. +.. method:: window.is_leaveok() + Return the current value set by :meth:`leaveok`. -.. method:: window.overwrite(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) + .. versionadded:: next - Overwrite the window on top of *destwin*. The windows need not be the same size, - in which case only the overlapping region is copied. This copy is destructive, - which means that the current background character overwrites the old contents of - *destwin*. +.. method:: window.is_linetouched(line) - To get fine-grained control over the copied region, the second form of - :meth:`overwrite` can be used. *sminrow* and *smincol* are the upper-left - coordinates of the source window, the other variables mark a rectangle in the - destination window. + Return ``True`` if the specified line was modified since the last call to + :meth:`refresh`; otherwise return ``False``. Raise a :exc:`curses.error` + exception if *line* is not valid for the given window. +.. method:: window.is_nodelay() -.. method:: window.putwin(file) + Return the current value set by :meth:`nodelay`. - Write all data associated with the window into the provided file object. This - information can be later retrieved using the :func:`getwin` function. + .. versionadded:: next +.. method:: window.is_notimeout() -.. method:: window.redrawln(beg, num) + Return the current value set by :meth:`notimeout`. - Indicate that the *num* screen lines, starting at line *beg*, are corrupted and - should be completely redrawn on the next :meth:`refresh` call. + .. versionadded:: next +.. method:: window.is_pad() -.. method:: window.redrawwin() + Return ``True`` if the window is a pad created by :func:`newpad`. - Touch the entire window, causing it to be completely redrawn on the next - :meth:`refresh` call. + .. versionadded:: next +.. method:: window.is_scrollok() -.. method:: window.refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) + Return the current value set by :meth:`scrollok`. - Update the display immediately (sync actual screen with previous - drawing/deleting methods). + .. versionadded:: next - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, - *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be - displayed in the pad is calculated from the screen coordinates, since the - rectangles must be the same size. Both rectangles must be entirely contained - within their respective structures. Negative values of *pminrow*, *pmincol*, - *sminrow*, or *smincol* are treated as if they were zero. +.. method:: window.is_subwin() + Return ``True`` if the window is a subwindow created by :meth:`subwin` + or :meth:`derwin`. -.. method:: window.resize(nlines, ncols) + .. versionadded:: next - Reallocate storage for a curses window to adjust its dimensions to the - specified values. If either dimension is larger than the current values, the - window's data is filled with blanks that have the current background - rendition (as set by :meth:`bkgdset`) merged into them. +.. method:: window.is_syncok() + Return the current value set by :meth:`syncok`. -.. method:: window.scroll([lines=1]) + .. versionadded:: next - Scroll the screen or scrolling region upward by *lines* lines. +.. method:: window.is_wintouched() + Return ``True`` if the specified window was modified since the last call to + :meth:`refresh`; otherwise return ``False``. -.. method:: window.scrollok(flag) +.. _curses-screen-objects: - Control what happens when the cursor of a window is moved off the edge of the - window or scrolling region, either as a result of a newline action on the bottom - line, or typing the last character of the last line. If *flag* is ``False``, the - cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up - one line. Note that in order to get the physical scrolling effect on the - terminal, it is also necessary to call :meth:`idlok`. +Screen objects +-------------- +.. class:: screen -.. method:: window.setscrreg(top, bottom) + A *screen* object represents a terminal initialized by :func:`newterm` + (or :func:`new_prescr`), + in addition to the default screen created by :func:`initscr`. + Screen objects are returned by those functions; + they cannot be instantiated directly. - Set the scrolling region from line *top* to line *bottom*. All scrolling actions - will take place in this region. + A screen is freed automatically once it is no longer referenced, + either directly or through one of its windows. + Each window keeps its screen alive, + so a screen remains valid as long as any of its windows does. + .. versionadded:: next -.. method:: window.standend() - Turn off the standout attribute. On some terminals this has the side effect of - turning off all attributes. +.. method:: screen.close() + Detach the screen's standard window, + breaking the reference cycle between them + so the screen can be reclaimed promptly instead of waiting for a + garbage collection. + Afterwards :attr:`~screen.stdscr` is ``None`` + and the window it returned earlier can no longer be used. + The screen's resources are released + once it and all its windows are no longer referenced. -.. method:: window.standout() + .. versionadded:: next - Turn on attribute *A_STANDOUT*. +.. attribute:: screen.stdscr -.. method:: window.subpad(begin_y, begin_x) - window.subpad(nlines, ncols, begin_y, begin_x) + The standard :ref:`window ` of the screen, + covering the whole terminal, + or ``None`` for a screen created by :func:`new_prescr`. - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. +.. method:: screen.use(func, /, *args, **kwargs) -.. method:: window.subwin(begin_y, begin_x) - window.subwin(nlines, ncols, begin_y, begin_x) + Call ``func(screen, *args, **kwargs)`` with the lock of the screen held, + and return its result. + This provides automatic protection for the screen + against concurrent access from another thread. - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Availability: if the underlying curses library provides ``use_screen()``. - By default, the sub-window will extend from the specified position to the lower - right corner of the window. + .. versionadded:: next -.. method:: window.syncdown() +.. _curses-complexchar-objects: - Touch each location in the window that has been touched in any of its ancestor - windows. This routine is called by :meth:`refresh`, so it should almost never - be necessary to call it manually. +Complex character objects +------------------------- +.. class:: complexchar(text, /, attr=0, pair=0) -.. method:: window.syncok(flag) + A *complex character* (or *complexchar*) is an immutable styled + character cell: a spacing character optionally followed by combining + characters, together with a set of attributes and a color pair. - If *flag* is ``True``, then :meth:`syncup` is called automatically - whenever there is a change in the window. + *text* is the cell's text, *attr* a combination of the + :ref:`WA_* attributes ` (equivalent to the matching + ``A_*`` constants), and *pair* a color pair number. Unlike the packed + :class:`chtype ` used by :meth:`~window.inch` and the ``A_*`` methods, + the color pair is stored separately and is not limited to the value that + fits in a :func:`color_pair`. + Complex characters are returned by :meth:`window.in_wch` and + :meth:`window.getbkgrnd`, and are accepted (along with an integer, a byte + or a string) by the character-cell methods such as :meth:`window.addch`, + :meth:`window.insch`, :meth:`window.bkgd`, :meth:`window.border`, + :meth:`window.hline` and :meth:`window.vline`. A complex character already + carries its own rendition, so it cannot be combined with an explicit *attr* + argument. -.. method:: window.syncup() + :func:`str` returns the cell's text; two complex characters are equal when + their text, attributes and color pair all match. - Touch all locations in ancestors of the window that have been changed in the - window. + The same code works on both wide- and narrow-character builds. On a narrow + build a cell holds a single character (no combining marks) that must encode to + one byte in the window's encoding (8-bit locales only), and *pair* is limited + to the value that fits in a :func:`color_pair`. + .. attribute:: attr -.. method:: window.timeout(delay) + The attributes of the character cell (read-only). - Set blocking or non-blocking read behavior for the window. If *delay* is - negative, blocking read is used (which will wait indefinitely for input). If - *delay* is zero, then non-blocking read is used, and :meth:`getch` will - return ``-1`` if no input is waiting. If *delay* is positive, then - :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is - still no input at the end of that time. + .. attribute:: pair + The color pair number of the character cell (read-only). -.. method:: window.touchline(start, count[, changed]) + .. versionadded:: next - Pretend *count* lines have been changed, starting with line *start*. If - *changed* is supplied, it specifies whether the affected lines are marked as - having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). +.. class:: complexstr(cells[, attr[, pair]]) -.. method:: window.touchwin() + A *complex character string* (or *complexstr*) is an immutable sequence of + styled character cells -- the string counterpart of + :class:`complexchar` (as :class:`str` is to a single character). - Pretend the whole window has been changed, for purposes of drawing - optimizations. + If *cells* is a string, it is split into character cells (each a spacing + character optionally followed by combining characters), and *attr* (a + combination of the :ref:`WA_* attributes `) and *pair* + (a color pair number), if given, are applied to every cell. + Otherwise *cells* is an iterable whose items are themselves cells, each a + :class:`complexchar` or a string; each item then carries its own rendition, + and *attr* and *pair* must be omitted. -.. method:: window.untouchwin() + It is returned by :meth:`window.in_wchstr`, and accepted by + :meth:`window.addstr`, :meth:`~window.addnstr`, :meth:`~window.insstr` and + :meth:`~window.insnstr`, so a run read from a window can be written back + unchanged. - Mark all lines in the window as unchanged since the last call to - :meth:`refresh`. + It behaves like an immutable sequence: ``len(s)`` is the number of cells, + ``s[i]`` is the *i*-th cell as a :class:`complexchar`, slicing and + concatenation produce new :class:`!complexstr` instances, and iterating + yields the cells. :func:`str` returns the cells' text joined together, and + two complex character strings are equal when their cells all match. It is + hashable. + To build or edit a run of cells, use an ordinary :class:`list` of + :class:`complexchar` (or strings); a :class:`!complexstr` is the immutable + form returned by a read. -.. method:: window.vline(ch, n[, attr]) - window.vline(y, x, ch, n[, attr]) + Like :class:`complexchar`, this type works on both wide- and narrow-character + builds, with the same per-cell limitations on a narrow build. - Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch* with attributes *attr*. + .. versionadded:: next Constants --------- +General +~~~~~~~ + The :mod:`!curses` module defines the following data members: @@ -1376,85 +2299,102 @@ The :mod:`!curses` module defines the following data members: .. data:: COLS - The width of the screen, i.e., the number of columns. + The width of the screen, that is, the number of columns. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. .. data:: LINES - The height of the screen, i.e., the number of lines. + The height of the screen, that is, the number of lines. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. +Attributes +~~~~~~~~~~ + Some constants are available to specify character cell attributes. The exact constants available are system dependent. -+------------------------+-------------------------------+ -| Attribute | Meaning | -+========================+===============================+ -| .. data:: A_ALTCHARSET | Alternate character set mode | -+------------------------+-------------------------------+ -| .. data:: A_BLINK | Blink mode | -+------------------------+-------------------------------+ -| .. data:: A_BOLD | Bold mode | -+------------------------+-------------------------------+ -| .. data:: A_DIM | Dim mode | -+------------------------+-------------------------------+ -| .. data:: A_INVIS | Invisible or blank mode | -+------------------------+-------------------------------+ -| .. data:: A_ITALIC | Italic mode | -+------------------------+-------------------------------+ -| .. data:: A_NORMAL | Normal attribute | -+------------------------+-------------------------------+ -| .. data:: A_PROTECT | Protected mode | -+------------------------+-------------------------------+ -| .. data:: A_REVERSE | Reverse background and | -| | foreground colors | -+------------------------+-------------------------------+ -| .. data:: A_STANDOUT | Standout mode | -+------------------------+-------------------------------+ -| .. data:: A_UNDERLINE | Underline mode | -+------------------------+-------------------------------+ -| .. data:: A_HORIZONTAL | Horizontal highlight | -+------------------------+-------------------------------+ -| .. data:: A_LEFT | Left highlight | -+------------------------+-------------------------------+ -| .. data:: A_LOW | Low highlight | -+------------------------+-------------------------------+ -| .. data:: A_RIGHT | Right highlight | -+------------------------+-------------------------------+ -| .. data:: A_TOP | Top highlight | -+------------------------+-------------------------------+ -| .. data:: A_VERTICAL | Vertical highlight | -+------------------------+-------------------------------+ +.. _curses-wa-constants: + ++------------------------+-------------------------+------------------------------------------+ +| Attribute | Wide attribute | Meaning | ++========================+=========================+==========================================+ +| .. data:: A_ALTCHARSET | .. data:: WA_ALTCHARSET | Alternate character set mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_BLINK | .. data:: WA_BLINK | Blink mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_BOLD | .. data:: WA_BOLD | Bold mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_DIM | .. data:: WA_DIM | Dim mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_INVIS | .. data:: WA_INVIS | Invisible or blank mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_ITALIC | .. data:: WA_ITALIC | Italic mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_NORMAL | .. data:: WA_NORMAL | Normal attribute | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_PROTECT | .. data:: WA_PROTECT | Protected mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_REVERSE | .. data:: WA_REVERSE | Reverse background and foreground colors | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_STANDOUT | .. data:: WA_STANDOUT | Standout mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_UNDERLINE | .. data:: WA_UNDERLINE | Underline mode | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_HORIZONTAL | .. data:: WA_HORIZONTAL | Horizontal highlight | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_LEFT | .. data:: WA_LEFT | Left highlight | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_LOW | .. data:: WA_LOW | Low highlight | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_RIGHT | .. data:: WA_RIGHT | Right highlight | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_TOP | .. data:: WA_TOP | Top highlight | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: A_VERTICAL | .. data:: WA_VERTICAL | Vertical highlight | ++------------------------+-------------------------+------------------------------------------+ .. versionadded:: 3.7 ``A_ITALIC`` was added. +The :meth:`~window.attr_get`, :meth:`~window.attr_set`, :meth:`~window.attr_on` +and :meth:`~window.attr_off` methods use the parallel set of ``WA_*`` constants +listed above. +Each has the same meaning as the corresponding ``A_*`` attribute +(:const:`WA_BOLD` like :const:`A_BOLD`, and so on), but belongs to the +``attr_t`` type rather than being packed into a character. +In ncurses the two sets share the same values, but other curses implementations +may give them different ones, so use the ``WA_*`` constants with the ``attr_*`` +methods. + +.. versionadded:: next + The ``WA_*`` constants were added. + Several constants are available to extract corresponding attributes returned by some methods. -+-------------------------+-------------------------------+ -| Bit-mask | Meaning | -+=========================+===============================+ -| .. data:: A_ATTRIBUTES | Bit-mask to extract | -| | attributes | -+-------------------------+-------------------------------+ -| .. data:: A_CHARTEXT | Bit-mask to extract a | -| | character | -+-------------------------+-------------------------------+ -| .. data:: A_COLOR | Bit-mask to extract | -| | color-pair field information | -+-------------------------+-------------------------------+ ++-------------------------+--------------------------+--------------------------------------------------+ +| Bit-mask | Wide bit-mask | Meaning | ++=========================+==========================+==================================================+ +| .. data:: A_ATTRIBUTES | .. data:: WA_ATTRIBUTES | Bit-mask to extract attributes | ++-------------------------+--------------------------+--------------------------------------------------+ +| .. data:: A_CHARTEXT | | Bit-mask to extract a character | ++-------------------------+--------------------------+--------------------------------------------------+ +| .. data:: A_COLOR | | Bit-mask to extract color-pair field information | ++-------------------------+--------------------------+--------------------------------------------------+ + +.. _curses-key-constants: + +Keys +~~~~ Keys are referred to by integer constants with names starting with ``KEY_``. The exact keycaps available are system dependent. -.. XXX this table is far too large! should it be alphabetized? - +-------------------------+--------------------------------------------+ | Key constant | Key | +=========================+============================================+ @@ -1672,6 +2612,9 @@ keys); also, the following keypad mappings are standard: | :kbd:`Page Down` | KEY_NPAGE | +------------------+-----------+ +Alternate character set +~~~~~~~~~~~~~~~~~~~~~~~~ + .. _curses-acs-codes: The following table lists characters from the alternate character set. These are @@ -1679,99 +2622,168 @@ inherited from the VT100 terminal, and will generally be available on software emulations such as X terminals. When there is no graphic available, curses falls back on a crude printable ASCII approximation. +Every character has two names. +The ``ACS_*`` code is an integer character, restricted to the 8-bit +alternate character set of the terminal. +The ``WACS_*`` code is the same character as a :class:`complexchar` cell, +which is not restricted to the alternate character set. + .. note:: These are available only after :func:`initscr` has been called. - -+------------------------+------------------------------------------+ -| ACS code | Meaning | -+========================+==========================================+ -| .. data:: ACS_BBSS | alternate name for upper right corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_BLOCK | solid square block | -+------------------------+------------------------------------------+ -| .. data:: ACS_BOARD | board of squares | -+------------------------+------------------------------------------+ -| .. data:: ACS_BSBS | alternate name for horizontal line | -+------------------------+------------------------------------------+ -| .. data:: ACS_BSSB | alternate name for upper left corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_BSSS | alternate name for top tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_BTEE | bottom tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_BULLET | bullet | -+------------------------+------------------------------------------+ -| .. data:: ACS_CKBOARD | checker board (stipple) | -+------------------------+------------------------------------------+ -| .. data:: ACS_DARROW | arrow pointing down | -+------------------------+------------------------------------------+ -| .. data:: ACS_DEGREE | degree symbol | -+------------------------+------------------------------------------+ -| .. data:: ACS_DIAMOND | diamond | -+------------------------+------------------------------------------+ -| .. data:: ACS_GEQUAL | greater-than-or-equal-to | -+------------------------+------------------------------------------+ -| .. data:: ACS_HLINE | horizontal line | -+------------------------+------------------------------------------+ -| .. data:: ACS_LANTERN | lantern symbol | -+------------------------+------------------------------------------+ -| .. data:: ACS_LARROW | left arrow | -+------------------------+------------------------------------------+ -| .. data:: ACS_LEQUAL | less-than-or-equal-to | -+------------------------+------------------------------------------+ -| .. data:: ACS_LLCORNER | lower left-hand corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_LRCORNER | lower right-hand corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_LTEE | left tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_NEQUAL | not-equal sign | -+------------------------+------------------------------------------+ -| .. data:: ACS_PI | letter pi | -+------------------------+------------------------------------------+ -| .. data:: ACS_PLMINUS | plus-or-minus sign | -+------------------------+------------------------------------------+ -| .. data:: ACS_PLUS | big plus sign | -+------------------------+------------------------------------------+ -| .. data:: ACS_RARROW | right arrow | -+------------------------+------------------------------------------+ -| .. data:: ACS_RTEE | right tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_S1 | scan line 1 | -+------------------------+------------------------------------------+ -| .. data:: ACS_S3 | scan line 3 | -+------------------------+------------------------------------------+ -| .. data:: ACS_S7 | scan line 7 | -+------------------------+------------------------------------------+ -| .. data:: ACS_S9 | scan line 9 | -+------------------------+------------------------------------------+ -| .. data:: ACS_SBBS | alternate name for lower right corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_SBSB | alternate name for vertical line | -+------------------------+------------------------------------------+ -| .. data:: ACS_SBSS | alternate name for right tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSBB | alternate name for lower left corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSBS | alternate name for bottom tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSSB | alternate name for left tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSSS | alternate name for crossover or big plus | -+------------------------+------------------------------------------+ -| .. data:: ACS_STERLING | pound sterling | -+------------------------+------------------------------------------+ -| .. data:: ACS_TTEE | top tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_UARROW | up arrow | -+------------------------+------------------------------------------+ -| .. data:: ACS_ULCORNER | upper left corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_URCORNER | upper right corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_VLINE | vertical line | -+------------------------+------------------------------------------+ + The ``WACS_*`` codes are only available if Python is built with + wide character support. + +.. versionadded:: next + The ``WACS_*`` codes. + ++------------------------+-------------------------+------------------------------------------+ +| ACS code | WACS code | Meaning | ++========================+=========================+==========================================+ +| .. data:: ACS_BBSS | .. data:: WACS_BBSS | alternate name for upper-right corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BLOCK | .. data:: WACS_BLOCK | solid square block | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BOARD | .. data:: WACS_BOARD | board of squares | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BSBS | .. data:: WACS_BSBS | alternate name for horizontal line | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BSSB | .. data:: WACS_BSSB | alternate name for upper-left corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BSSS | .. data:: WACS_BSSS | alternate name for top tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BTEE | .. data:: WACS_BTEE | bottom tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_BULLET | .. data:: WACS_BULLET | bullet | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_CKBOARD | .. data:: WACS_CKBOARD | checker board (stipple) | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_DARROW | .. data:: WACS_DARROW | arrow pointing down | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_DEGREE | .. data:: WACS_DEGREE | degree symbol | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_DIAMOND | .. data:: WACS_DIAMOND | diamond | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_GEQUAL | .. data:: WACS_GEQUAL | greater-than-or-equal-to | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_HLINE | .. data:: WACS_HLINE | horizontal line | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_LANTERN | .. data:: WACS_LANTERN | lantern symbol | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_LARROW | .. data:: WACS_LARROW | left arrow | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_LEQUAL | .. data:: WACS_LEQUAL | less-than-or-equal-to | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_LLCORNER | .. data:: WACS_LLCORNER | lower-left corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_LRCORNER | .. data:: WACS_LRCORNER | lower-right corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_LTEE | .. data:: WACS_LTEE | left tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_NEQUAL | .. data:: WACS_NEQUAL | not-equal sign | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_PI | .. data:: WACS_PI | letter pi | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_PLMINUS | .. data:: WACS_PLMINUS | plus-or-minus sign | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_PLUS | .. data:: WACS_PLUS | big plus sign | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_RARROW | .. data:: WACS_RARROW | right arrow | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_RTEE | .. data:: WACS_RTEE | right tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_S1 | .. data:: WACS_S1 | scan line 1 | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_S3 | .. data:: WACS_S3 | scan line 3 | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_S7 | .. data:: WACS_S7 | scan line 7 | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_S9 | .. data:: WACS_S9 | scan line 9 | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SBBS | .. data:: WACS_SBBS | alternate name for lower-right corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SBSB | .. data:: WACS_SBSB | alternate name for vertical line | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SBSS | .. data:: WACS_SBSS | alternate name for right tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SSBB | .. data:: WACS_SSBB | alternate name for lower-left corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SSBS | .. data:: WACS_SSBS | alternate name for bottom tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SSSB | .. data:: WACS_SSSB | alternate name for left tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_SSSS | .. data:: WACS_SSSS | alternate name for crossover or big plus | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_STERLING | .. data:: WACS_STERLING | pound sterling | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_TTEE | .. data:: WACS_TTEE | top tee | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_UARROW | .. data:: WACS_UARROW | up arrow | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_ULCORNER | .. data:: WACS_ULCORNER | upper-left corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_URCORNER | .. data:: WACS_URCORNER | upper-right corner | ++------------------------+-------------------------+------------------------------------------+ +| .. data:: ACS_VLINE | .. data:: WACS_VLINE | vertical line | ++------------------------+-------------------------+------------------------------------------+ + +The following table lists the double-line and thick-line characters. +They have no ``ACS_*`` counterpart, and are not provided by every implementation. +As in the table above, the alternate name spells out the four sides of the +character, clockwise from the top: +``B`` for a blank side, ``S`` for a single line, ``D`` for a double line and +``T`` for a thick line. + ++---------------------------+---------------------+--------------------------------+ +| WACS code | Alternate name | Meaning | ++===========================+=====================+================================+ +| .. data:: WACS_D_BTEE | .. data:: WACS_DDBD | double-line bottom tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_HLINE | .. data:: WACS_BDBD | double-line horizontal line | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_LLCORNER | .. data:: WACS_DDBB | double-line lower-left corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_LRCORNER | .. data:: WACS_DBBD | double-line lower-right corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_LTEE | .. data:: WACS_DDDB | double-line left tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_PLUS | .. data:: WACS_DDDD | double-line big plus sign | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_RTEE | .. data:: WACS_DBDD | double-line right tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_TTEE | .. data:: WACS_BDDD | double-line top tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_ULCORNER | .. data:: WACS_BDDB | double-line upper-left corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_URCORNER | .. data:: WACS_BBDD | double-line upper-right corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_D_VLINE | .. data:: WACS_DBDB | double-line vertical line | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_BTEE | .. data:: WACS_TTBT | thick-line bottom tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_HLINE | .. data:: WACS_BTBT | thick-line horizontal line | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_LLCORNER | .. data:: WACS_TTBB | thick-line lower-left corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_LRCORNER | .. data:: WACS_TBBT | thick-line lower-right corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_LTEE | .. data:: WACS_TTTB | thick-line left tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_PLUS | .. data:: WACS_TTTT | thick-line big plus sign | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_RTEE | .. data:: WACS_TBTT | thick-line right tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_TTEE | .. data:: WACS_BTTT | thick-line top tee | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_ULCORNER | .. data:: WACS_BTTB | thick-line upper-left corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_URCORNER | .. data:: WACS_BBTT | thick-line upper-right corner | ++---------------------------+---------------------+--------------------------------+ +| .. data:: WACS_T_VLINE | .. data:: WACS_TBTB | thick-line vertical line | ++---------------------------+---------------------+--------------------------------+ + +Mouse buttons +~~~~~~~~~~~~~ The following table lists mouse button constants used by :meth:`getmouse`: @@ -1792,13 +2804,16 @@ The following table lists mouse button constants used by :meth:`getmouse`: +----------------------------------+---------------------------------------------+ | .. data:: BUTTON_CTRL | Control was down during button state change | +----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_ALT | Control was down during button state change | +| .. data:: BUTTON_ALT | Alt was down during button state change | +----------------------------------+---------------------------------------------+ .. versionchanged:: 3.10 The ``BUTTON5_*`` constants are now exposed if they are provided by the underlying curses library. +Colors +~~~~~~ + The following table lists the predefined colors: +-------------------------+----------------------------+ @@ -1841,9 +2856,9 @@ The module :mod:`!curses.textpad` defines the following function: Draw a rectangle. The first argument must be a window object; the remaining arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the + arguments are the y and x coordinates of the upper-left corner of the rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using + coordinates of the lower-right corner. The rectangle will be drawn using VT100/IBM PC forms characters on terminals that make this possible (including xterm and most other software terminal emulators). Otherwise it will be drawn with ASCII dashes, vertical bars, and plus signs. @@ -1857,32 +2872,45 @@ Textbox objects You can instantiate a :class:`Textbox` object as follows: -.. class:: Textbox(win) +.. class:: Textbox(win, insert_mode=False) Return a textbox widget object. The *win* argument should be a curses :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. + be contained. If *insert_mode* is true, the textbox inserts typed + characters, shifting existing text to the right, rather than overwriting it. + The edit cursor of the textbox is initially located at the + upper-left corner of the containing window, with coordinates ``(0, 0)``. The instance's :attr:`stripspaces` flag is initially on. + .. versionchanged:: next + Entering and reading back the full Unicode range, including combining + characters, is now supported when curses is built with wide-character + support. + :class:`Textbox` objects have the following methods: - .. method:: edit([validator]) + .. method:: edit(validate=None) This is the entry point you will normally use. It accepts editing keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for + *validate* is supplied, it must be a function. It will be called for each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a + is done on the result. If it returns a false value, the keystroke is + ignored. This method returns the window contents as a string; whether blanks in the window are included is affected by the :attr:`stripspaces` attribute. + .. versionchanged:: next + *validate* is now called with a non-ASCII character as a string; + other keystrokes are still passed as an integer. + .. method:: do_command(ch) - Process a single command keystroke. Here are the supported special - keystrokes: + Process a single command keystroke. Returns ``1`` to continue editing, + or ``0`` if a termination keystroke was processed. Here are the supported + special keystrokes: +------------------+-------------------------------------------+ | Keystroke | Action | @@ -1905,7 +2933,8 @@ You can instantiate a :class:`Textbox` object as follows: | :kbd:`Control-H` | Delete character backward. | +------------------+-------------------------------------------+ | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | + | | otherwise move to the start of the next | + | | line. | +------------------+-------------------------------------------+ | :kbd:`Control-K` | If line is blank, delete it, otherwise | | | clear to end of line. | diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index fd8e0c0bea1cb10..9f4871a55bc8af3 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -100,12 +100,25 @@ Module contents ignored. - *eq*: If true (the default), an :meth:`~object.__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. + generated. - If the class already defines :meth:`!__eq__`, this parameter is - ignored. + This method compares the class by comparing each field in order. Both + instances in the comparison must be of the identical type. + + If the class already defines :meth:`!__eq__`, this parameter is ignored. + + .. versionchanged:: 3.13 + The generated ``__eq__`` method now compares each field individually + (for example, ``self.a == other.a and self.b == other.b``), rather than + comparing tuples of fields as in previous versions. + + This change makes the comparison faster but it may alter results in cases + where attributes compare equal by identity but not by value (such as + ``float('nan')``). + + In Python 3.12 and earlier, the comparison was performed by creating + tuples of the fields and comparing them (for example, + ``(self.a, self.b) == (other.a, other.b)``). - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be @@ -249,8 +262,8 @@ Module contents c = C() c.mylist += [1, 2, 3] - As shown above, the :const:`MISSING` value is a sentinel object used to - detect if some parameters are provided by the user. This sentinel is + As shown above, the :const:`MISSING` value is a :class:`sentinel` object + used to detect if some parameters are provided by the user. This sentinel is used because ``None`` is a valid value for some parameters with a distinct meaning. No code should directly use the :const:`MISSING` value. @@ -371,8 +384,8 @@ Module contents Converts the dataclass *obj* to a dict (by using the factory function *dict_factory*). Each dataclass is converted to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. + frozendicts, lists, and tuples are recursed into. Other objects are copied + with :func:`copy.deepcopy`. Example of using :func:`!asdict` on nested dataclasses:: @@ -402,8 +415,8 @@ Module contents Converts the dataclass *obj* to a tuple (by using the factory function *tuple_factory*). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with + to a tuple of its field values. dataclasses, dicts, frozendicts, lists, + and tuples are recursed into. Other objects are copied with :func:`copy.deepcopy`. Continuing from the previous example:: @@ -418,7 +431,7 @@ Module contents :func:`!astuple` raises :exc:`TypeError` if *obj* is not a dataclass instance. -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, decorator=dataclass) +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False, module=None, qualname=None, decorator=dataclass) Creates a new dataclass with name *cls_name*, fields as defined in *fields*, base classes as given in *bases*, and initialized @@ -434,6 +447,9 @@ Module contents of the dataclass is set to that value. By default, it is set to the module name of the caller. + If *qualname* is defined, the :attr:`~type.__qualname__` attribute of the dataclass + is set to that value. By default, it is set to the value passed to *cls_name*. + The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments as :deco:`dataclass`. By default, the :deco:`dataclass` @@ -464,6 +480,8 @@ Module contents .. versionadded:: 3.14 Added the *decorator* parameter. + .. versionadded:: next + Added the *qualname* parameter. .. function:: replace(obj, /, **changes) @@ -498,7 +516,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -509,11 +528,14 @@ Module contents .. data:: MISSING - A sentinel value signifying a missing default or default_factory. + A :class:`sentinel` object signifying a missing default or *default_factory*. + + .. versionchanged:: 3.15 + :const:`!MISSING` is now a :class:`sentinel` object. .. data:: KW_ONLY - A sentinel value used as a type annotation. Any fields after a + A :class:`sentinel` object used as a type annotation. Any fields after a pseudo-field with the type of :const:`!KW_ONLY` are marked as keyword-only fields. Note that a pseudo-field of type :const:`!KW_ONLY` is otherwise completely ignored. This includes the @@ -538,6 +560,9 @@ Module contents .. versionadded:: 3.10 + .. versionchanged:: 3.15 + :const:`!KW_ONLY` is now a :class:`sentinel` object. + .. exception:: FrozenInstanceError Raised when an implicitly defined :meth:`~object.__setattr__` or diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst index ff51b2779e5fa33..4b4f3b1720accc4 100644 --- a/Doc/library/datatypes.rst +++ b/Doc/library/datatypes.rst @@ -8,9 +8,10 @@ The modules described in this chapter provide a variety of specialized data types such as dates and times, fixed-type arrays, heap queues, double-ended queues, and enumerations. -Python also provides some built-in data types, in particular, -:class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and -:class:`tuple`. The :class:`str` class is used to hold +Python also provides :ref:`some built-in data types `, in particular, +:class:`list`, :class:`tuple`, :class:`dict`, :class:`frozendict`, +:class:`set`, and :class:`frozenset`. +The :class:`str` class is used to hold Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used to hold binary data. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 8993049a720b1c8..f3c4ef9199075c9 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -606,12 +606,11 @@ Other constructors, all class methods: .. note:: - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: @@ -1180,14 +1179,13 @@ Other constructors, all class methods: time tuple. See also :ref:`strftime-strptime-behavior` and :meth:`datetime.fromisoformat`. - .. versionchanged:: 3.13 + .. versionchanged:: 3.15 - If *format* specifies a day of month without a year a - :exc:`DeprecationWarning` is now emitted. This is to avoid a quadrennial + If *format* specifies a day of month (``%d``) without a year, + :exc:`ValueError` is raised. This is to avoid a quadrennial leap year bug in code seeking to parse only a month and day as the default year used in absence of one in the format is not a leap year. - Such *format* values may raise an error as of Python 3.15. The - workaround is to always include a year in your *format*. If parsing + The workaround is to always include a year in your *format*. If parsing *date_string* values that do not have a year, explicitly add a year that is a leap year before parsing: @@ -2572,13 +2570,13 @@ requires, and these work on all supported platforms. | | truncated to an integer as a | | | | | zero-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | -| | zero-padded decimal number. | | | +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9), | +| | zero-padded decimal number. | | \(10) | +-----------+--------------------------------+------------------------+-------+ | ``%D`` | Equivalent to ``%m/%d/%y``. | 11/28/25 | \(9) | | | | | | +-----------+--------------------------------+------------------------+-------+ -| ``%e`` | The day of the month as a | ␣1, ␣2, ..., 31 | | +| ``%e`` | The day of the month as a | ␣1, ␣2, ..., 31 | \(10) | | | space-padded decimal number. | | | +-----------+--------------------------------+------------------------+-------+ | ``%F`` | Equivalent to ``%Y-%m-%d``, | 2025-10-11, | | @@ -2919,11 +2917,12 @@ Notes: >>> dt.datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. datetime.datetime(1984, 2, 29, 0, 0) - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + Using ``%d`` without a year now raises :exc:`ValueError`. + + .. deprecated-removed:: 3.15 3.17 :meth:`~.datetime.strptime` calls using a format string containing - a day of month without a year now emit a - :exc:`DeprecationWarning`. In 3.15 or later we may change this into - an error or change the default year to a leap year. See :gh:`70647`. + ``%e`` without a year now emit a :exc:`DeprecationWarning`. .. rubric:: Footnotes diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 2af5dfce9612b37..ecd973a070b3dff 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1182,6 +1182,14 @@ In addition to the three supplied contexts, new contexts can be created with the Return a duplicate of the context. + :class:`!Context` objects also support :func:`copy.replace`, + which returns a duplicate with the specified fields replaced. + Fields which are not specified keep the values + they have in the original context. + + .. versionchanged:: next + Added support for :func:`copy.replace`. + .. method:: copy_decimal(num, /) Return a copy of the Decimal instance num. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index 5d522556235a02b..0d03eca88b62673 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,4 +1,4 @@ -Tkinter Dialogs +Tkinter dialogs =============== :mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs @@ -15,16 +15,41 @@ The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. -.. function:: askfloat(title, prompt, **kw) - askinteger(title, prompt, **kw) - askstring(title, prompt, **kw) +.. function:: askfloat(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None, use_ttk=True) + askinteger(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None, use_ttk=True) + askstring(title, prompt, *, initialvalue=None, show=None, parent=None, use_ttk=True) - The above three functions provide dialogs that prompt the user to enter a value - of the desired type. + Prompt the user to enter a value of the desired type and return it, or + ``None`` if the dialog is cancelled. -.. class:: Dialog(parent, title=None) + *title* is the dialog title and *prompt* the message shown above the entry. + *initialvalue* is the value initially placed in the entry. + *parent* is the window over which the dialog is shown. + :func:`askinteger` and :func:`askfloat` also accept *minvalue* and + *maxvalue*, which bound the accepted value. + :func:`askstring` also accepts *show*, a character used to mask the entered + text, for example ``'*'`` to hide a password. + They use the themed :mod:`tkinter.ttk` widgets; pass ``use_ttk=False`` for + the classic widgets. + +.. class:: Dialog(parent, title=None, *, use_ttk=False) The base class for custom dialogs. + Instantiating it shows the dialog modally and returns once the user closes + it; the entered value is then available in the :attr:`!result` attribute. + When *use_ttk* is false (the default), the dialog is built from the classic + :mod:`tkinter` widgets, modelled on the classic ``tk_dialog``; when true, + from the themed :mod:`tkinter.ttk` widgets, modelled on the Tk message box. + The default is classic for compatibility, since the themed widgets set a + themed background that classic widgets added in :meth:`body` would not match. + + .. versionchanged:: next + Added the *use_ttk* parameter. + + .. attribute:: result + + The value produced by :meth:`apply`, or ``None`` if the dialog was + cancelled. .. method:: body(master) @@ -36,6 +61,61 @@ functions for creating simple modal dialogs to get a value from the user. Default behaviour adds OK and Cancel buttons. Override for custom button layouts. + .. method:: validate() + + Validate the data entered by the user. + Return true if it is valid, in which case the dialog proceeds to + :meth:`apply`; return false to keep the dialog open. + The default implementation always returns true; override it to check the + input. + + .. method:: apply() + + Process the data entered by the user, for example by storing it in the + :attr:`!result` attribute. + Called after :meth:`validate` succeeds and just before the dialog is + destroyed. + The default implementation does nothing; override it to act on or store + the result. + + .. method:: destroy() + + Destroy the dialog window, clearing the reference to the widget that had + the initial focus. + + +.. class:: SimpleDialog(master, text='', buttons=[], default=None, cancel=None, title=None, class_=None, *, bitmap=None, detail='', use_ttk=True) + + A simple modal dialog that displays the message *text* above a row of push + buttons given by *buttons*, and returns the index of the button the user + presses. + Each entry of *buttons* is either a button label, or a mapping of button + options such as ``{'text': 'OK', 'underline': 0}``; an ``underline`` option + makes :kbd:`Alt` plus the underlined character invoke the button. + *default* is the index of the default button, activated by the Return key + when no button has the focus, *cancel* the index returned when the window is + closed through the window manager, *title* the window title, and *class_* + the Tk class name of the window. + *bitmap* is the name of a bitmap displayed beside the message + (for example ``'warning'`` or ``'question'``); the standard names + ``'error'``, ``'info'``, ``'question'`` and ``'warning'`` are shown as + themed icons when *use_ttk* is true. + *detail* is a secondary message displayed below *text*. + When *use_ttk* is true (the default), the dialog is built from the themed + :mod:`tkinter.ttk` widgets, modelled on the Tk message box; when false, from + the classic :mod:`tkinter` widgets, modelled on ``tk_dialog``. + + .. versionchanged:: next + The dialog is now built from the themed :mod:`tkinter.ttk` widgets by + default, instead of the classic :mod:`tkinter` widgets. + Added the *bitmap*, *detail* and *use_ttk* parameters. + Entries of *buttons* may be mappings of button options. + + .. method:: go() + + Display the dialog, wait until the user presses a button or closes the + window, and return the index of the chosen button. + :mod:`!tkinter.filedialog` --- File selection dialogs @@ -51,7 +131,7 @@ functions for creating simple modal dialogs to get a value from the user. The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. -Native Load/Save Dialogs +Native load/save dialogs ------------------------ The following classes and functions provide file dialog windows that combine a @@ -77,56 +157,84 @@ listed below: **Static factory functions** The below functions when called create a modal, native look-and-feel dialog, -wait for the user's selection, then return the selected value(s) or ``None`` to the -caller. +wait for the user's selection, and return it. +The exact return value depends on the function (see below); when the dialog is +cancelled it is the empty value documented for that function -- an empty +string, an empty tuple, an empty list or ``None``. .. function:: askopenfile(mode="r", **options) - askopenfiles(mode="r", **options) - The above two functions create an :class:`Open` dialog and return the opened - file object(s) in read-only mode. + Create an :class:`Open` dialog and return the opened file object, + or ``None`` if the dialog is cancelled. + The file is opened in mode *mode* (read-only ``'r'`` by default). + +.. function:: askopenfiles(mode="r", **options) + + Create an :class:`Open` dialog and return a list of the opened file objects, + or an empty list if cancelled. + The files are opened in mode *mode* (read-only ``'r'`` by default). + + .. deprecated-removed:: next 3.19 + Opening several files at once is error-prone, + and the returned list cannot be used in a :keyword:`with` statement. + Iterate over the names returned by :func:`askopenfilenames` + and open them one by one instead. .. function:: asksaveasfile(mode="w", **options) - Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. + Create a :class:`SaveAs` dialog and return the opened file object, or + ``None`` if the dialog is cancelled. + The file is opened in mode *mode* (``'w'`` by default). .. function:: askopenfilename(**options) askopenfilenames(**options) - The above two functions create an :class:`Open` dialog and return the - selected filename(s) that correspond to existing file(s). + Create an :class:`Open` dialog. + :func:`askopenfilename` returns the selected filename as a string, or an + empty string if the dialog is cancelled. + :func:`askopenfilenames` returns a tuple of the selected filenames, or an + empty tuple if cancelled. .. function:: asksaveasfilename(**options) - Create a :class:`SaveAs` dialog and return the selected filename. + Create a :class:`SaveAs` dialog and return the selected filename as a + string, or an empty string if the dialog is cancelled. .. function:: askdirectory(**options) - | Prompt user to select a directory. - | Additional keyword option: - | *mustexist* - determines if selection must be an existing directory. + Prompt the user to select a directory, and return its path as a string, or + an empty string if the dialog is cancelled. + Additional keyword option: *mustexist* - if true, the user may only select + an existing directory (false by default). .. class:: Open(master=None, **options) SaveAs(master=None, **options) + Directory(master=None, **options) - The above two classes provide native dialog windows for saving and loading - files. + The above three classes provide native dialog windows for loading and saving + files and for selecting a directory. **Convenience classes** The below classes are used for creating file/directory windows from scratch. These do not emulate the native look-and-feel of the platform. -.. class:: Directory(master=None, **options) - - Create a dialog prompting the user to select a directory. - .. note:: The *FileDialog* class should be subclassed for custom event handling and behaviour. -.. class:: FileDialog(master, title=None) +.. class:: FileDialog(master, title=None, *, use_ttk=True) Create a basic file selection dialog. + Its layout -- a filter entry, side-by-side directory and file lists, and a + selection entry -- follows the classic Motif file selection dialog. + When *use_ttk* is true (the default), the dialog is built from the themed + :mod:`tkinter.ttk` widgets; when false, from the classic :mod:`tkinter` + widgets. + + .. versionchanged:: next + The dialog is now built from the themed :mod:`tkinter.ttk` widgets by + default, instead of the classic :mod:`tkinter` widgets. + Added the *use_ttk* parameter. .. method:: cancel_command(event=None) @@ -168,6 +276,13 @@ These do not emulate the native look-and-feel of the platform. Exit dialog returning current selection. + .. method:: ok_command() + + Called when the user confirms the current selection. + The base implementation accepts the selection and closes the dialog; + :class:`LoadFileDialog` and :class:`SaveFileDialog` override it to check + the selection first. + .. method:: quit(how=None) Exit dialog returning filename, if any. @@ -181,21 +296,27 @@ These do not emulate the native look-and-feel of the platform. Update the current file selection to *file*. -.. class:: LoadFileDialog(master, title=None) +.. class:: LoadFileDialog(master, title=None, *, use_ttk=True) A subclass of FileDialog that creates a dialog window for selecting an existing file. + .. versionchanged:: next + Added the *use_ttk* parameter. + .. method:: ok_command() Test that a file is provided and that the selection indicates an already existing file. -.. class:: SaveFileDialog(master, title=None) +.. class:: SaveFileDialog(master, title=None, *, use_ttk=True) A subclass of FileDialog that creates a dialog window for selecting a destination file. + .. versionchanged:: next + Added the *use_ttk* parameter. + .. method:: ok_command() Test whether or not the selection points to a valid file that is not a @@ -222,6 +343,42 @@ is the base class for dialogs defined in other supporting modules. Render the Dialog window. +:mod:`!tkinter.dialog` --- Classic Tk dialog boxes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.dialog + :synopsis: A simple dialog box built on the classic Tk widgets. + +**Source code:** :source:`Lib/tkinter/dialog.py` + +-------------- + +The :mod:`!tkinter.dialog` module provides a simple modal dialog box built on +the classic (non-themed) Tk widgets. + +.. data:: DIALOG_ICON + + The name of a bitmap (``'questhead'``) suitable for use as the *bitmap* + of a :class:`Dialog`. + +.. class:: Dialog(master=None, cnf={}, **kw) + + Display a modal dialog box built from the classic (non-themed) Tk widgets + and wait for the user to press one of its buttons. + The options, given through *cnf* or as keyword arguments, are all required: + *title* (the window title), *text* (the message), *bitmap* (the name of a + bitmap icon, such as :data:`DIALOG_ICON`), *default* (the index of the + default button) and *strings* (the sequence of button labels). + After construction, the :attr:`!num` attribute holds the index of the button + the user pressed. + + .. method:: destroy() + + Do nothing. + The dialog window is destroyed automatically before the constructor + returns, so there is nothing left for this method to do. + + .. seealso:: Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 8b812c173b59536..5339186b72f6bc7 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -13,45 +13,87 @@ -------------- -This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce information -about file differences in various formats, including HTML and context and unified -diffs. For comparing directories and files, see also, the :mod:`filecmp` module. - - -.. class:: SequenceMatcher - :noindex: - - This is a flexible class for comparing pairs of sequences of any type, so long - as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a - little fancier than, an algorithm published in the late 1980's by Ratcliff and - Obershelp under the hyperbolic name "gestalt pattern matching." The idea is to - find the longest contiguous matching subsequence that contains no "junk" - elements; these "junk" elements are ones that are uninteresting in some - sense, such as blank lines or whitespace. (Handling junk is an - extension to the Ratcliff and Obershelp algorithm.) The same - idea is then applied recursively to the pieces of the sequences to the left and - to the right of the matching subsequence. This does not yield minimal edit - sequences, but does tend to yield matches that "look right" to people. - - **Timing:** The basic Ratcliff-Obershelp algorithm is cubic time in the worst - case and quadratic time in the expected case. :class:`SequenceMatcher` is - quadratic time for the worst case and has expected-case behavior dependent in a - complicated way on how many elements the sequences have in common; best case - time is linear. - - **Automatic junk heuristic:** :class:`SequenceMatcher` supports a heuristic that - automatically treats certain sequence items as junk. The heuristic counts how many - times each individual item appears in the sequence. If an item's duplicates (after - the first one) account for more than 1% of the sequence and the sequence is at least - 200 items long, this item is marked as "popular" and is treated as junk for - the purpose of sequence matching. This heuristic can be turned off by setting - the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. +This module provides classes and functions for comparing sequences. +Most of them compare sequences of text lines (for example lists of strings, +or :term:`file objects `) and +produce :dfn:`diffs` -- reports on the differences. +Diffs can be produced in various formats, including HTML and context +and unified diffs -- formats produced by tools like +:manpage:`diff ` and :manpage:`git diff `. - .. versionchanged:: 3.2 - Added the *autojunk* parameter. +Comparisons are done using a matching algorithm implemented in +:class:`SequenceMatcher` -- a flexible class for comparing pairs of sequences +of any type, not just text, so long as the sequence elements are +:term:`hashable`. + + +.. _difflib-junk: + +Junk heuristic +-------------- + +:mod:`!difflib` uses a :dfn:`junk` heuristic: some items are deemed to be +:dfn:`junk`, and ignored when searching for similarities. +Ideally, these are uninteresting or common items, such as blank lines +or whitespace. + +This heuristic can speed the algorithm up (because it reduces the number of +possible combinations) and it can produce results that are more understandable +for humans (typically breaking on whitespace). +But it can also cause pathological cases: + +- Inappropriately chosen junk items can cause an unexpectedly **large** (but + still correct) result. +- The default heuristic is **asymmetric**: only the second sequence is + inspected when determining what is considered junk, so comparing A to B can + give different results than comparing B to A and reversing the result. + +By default, if the second input sequence is at least 200 items long, items +that account for more than 1% it are considered *junk*. + +Depending on your data, you should consider turning this heuristic off +(setting :class:`~difflib.SequenceMatcher`'s *autojunk* argument to ``False``) +or tuning it (using the *isjunk* argument, perhaps to one of the +:ref:`predefined functions `). + + +The :mod:`!difflib` algorithm +----------------------------- + +The algorithm used in :class:`SequenceMatcher` predates, and is a little +fancier than, an algorithm published in the late 1980s by Ratcliff and +Obershelp under the hyperbolic name "gestalt pattern matching." +The idea is to find the longest contiguous subsequence common to both inputs, +then recursively handle the pieces of the sequences to the left and to the +right of the matching subsequence. + +.. seealso:: + + `Pattern Matching: The Gestalt Approach `_ + Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This + was published in Dr. Dobb's Journal in July, 1988. + +As an extension to the Ratcliff and Obershelp algorithm, :mod:`!difflib` +searches for the longest *junk-free* contiguous subsequence. +See the :ref:`difflib-junk` section for details. + +.. impl-detail:: Timing + + The basic Ratcliff-Obershelp algorithm is cubic time in the worst + case and quadratic time in the expected case. + :mod:`difflib`'s algorithm is quadratic time for the worst case and has + expected-case behavior dependent in a complicated way on how many elements + the sequences have in common; + best case time is linear. +.. _difflib-diff-generation: + +Diff generation +--------------- + +.. _differ-objects: + .. class:: Differ This is a class for comparing sequences of lines of text, and producing @@ -77,6 +119,47 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. and were not present in either input sequence. These lines can be confusing if the sequences contain whitespace characters, such as spaces, tabs or line breaks. + Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** + diffs. To the contrary, minimal diffs are often counter-intuitive for humans, + because they synch up anywhere possible, sometimes at accidental matches + 100 pages apart. + Restricting synch points to contiguous matches preserves some notion of + locality, at the occasional cost of producing a longer diff. + + The :class:`Differ` class has this constructor: + + .. method:: __init__(linejunk=None, charjunk=None) + + Optional keyword parameters *linejunk* and *charjunk* are for filter functions + (or ``None``): + + *linejunk*: A function that accepts a single string argument, and returns true + if the string is junk. The default is ``None``, meaning that no line is + considered junk. + + *charjunk*: A function that accepts a single character argument (a string of + length 1), and returns true if the character is junk. The default is ``None``, + meaning that no character is considered junk. + + These junk-filtering functions speed up matching to find + differences and do not cause any differing lines or characters to + be ignored. Read the description of the + :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* + parameter for an explanation. + + :class:`Differ` objects are used (deltas generated) via a single method: + + + .. method:: Differ.compare(a, b) + + Compare two sequences of lines, and generate the delta (a sequence of lines). + + Each sequence must contain individual single-line strings ending with + newlines. Such sequences can be obtained from the + :meth:`~io.IOBase.readlines` method of file-like objects. The generated + delta also consists of newline-terminated strings, ready to be + printed as-is via the :meth:`~io.IOBase.writelines` method of a + file-like object. .. class:: HtmlDiff @@ -85,6 +168,11 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. with inter-line and intra-line change highlights. The table can be generated in either full or contextual difference mode. + .. warning:: + + The trailing newlines get stripped before the diff, so the result can be + incomplete. See :gh:`71896` for details. + The constructor for this class is: @@ -339,6 +427,12 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. versionadded:: 3.5 + +.. _difflib-isjunk-functions: + +Junk definition functions +------------------------- + .. function:: IS_LINE_JUNK(line) Return ``True`` for ignorable lines. The line *line* is ignorable if *line* is @@ -353,21 +447,11 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. parameter *charjunk* in :func:`ndiff`. -.. seealso:: - - `Pattern Matching: The Gestalt Approach `_ - Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in Dr. Dobb's Journal in July, 1988. - - .. _sequence-matcher: SequenceMatcher objects ----------------------- -The :class:`SequenceMatcher` class has this constructor: - - .. class:: SequenceMatcher(isjunk=None, a='', b='', autojunk=True) Optional argument *isjunk* must be ``None`` (the default) or a one-argument @@ -553,16 +637,6 @@ The :class:`SequenceMatcher` class has this constructor: to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an upper bound. - .. note:: - - Caution: The result of a :meth:`ratio` call may depend on the order of - the arguments. For instance:: - - >>> SequenceMatcher(None, 'tide', 'diet').ratio() - 0.25 - >>> SequenceMatcher(None, 'diet', 'tide').ratio() - 0.5 - .. method:: quick_ratio() @@ -588,10 +662,13 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: 1.0 +Examples +-------- + .. _sequencematcher-examples: SequenceMatcher examples ------------------------- +........................ This example compares two strings, considering blanks to be "junk": @@ -639,59 +716,10 @@ If you want to know how to change the first sequence into the second, use built with :class:`SequenceMatcher`. -.. _differ-objects: - -Differ objects --------------- - -Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** -diffs. To the contrary, minimal diffs are often counter-intuitive, because they -synch up anywhere possible, sometimes accidental matches 100 pages apart. -Restricting synch points to contiguous matches preserves some notion of -locality, at the occasional cost of producing a longer diff. - -The :class:`Differ` class has this constructor: - - -.. class:: Differ(linejunk=None, charjunk=None) - :noindex: - - Optional keyword parameters *linejunk* and *charjunk* are for filter functions - (or ``None``): - - *linejunk*: A function that accepts a single string argument, and returns true - if the string is junk. The default is ``None``, meaning that no line is - considered junk. - - *charjunk*: A function that accepts a single character argument (a string of - length 1), and returns true if the character is junk. The default is ``None``, - meaning that no character is considered junk. - - These junk-filtering functions speed up matching to find - differences and do not cause any differing lines or characters to - be ignored. Read the description of the - :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* - parameter for an explanation. - - :class:`Differ` objects are used (deltas generated) via a single method: - - - .. method:: Differ.compare(a, b) - - Compare two sequences of lines, and generate the delta (a sequence of lines). - - Each sequence must contain individual single-line strings ending with - newlines. Such sequences can be obtained from the - :meth:`~io.IOBase.readlines` method of file-like objects. The delta - generated also consists of newline-terminated strings, ready to be - printed as-is via the :meth:`~io.IOBase.writelines` method of a - file-like object. - - .. _differ-examples: Differ example --------------- +.............. This example compares two texts. First we set up the texts, sequences of individual single-line strings ending with newlines (such sequences can also be @@ -724,7 +752,7 @@ Finally, we compare the two: >>> result = list(d.compare(text1, text2)) -``result`` is a list of strings, so let's pretty-print it: +``result`` is a list of strings, so let's pretty-print it:: >>> from pprint import pprint >>> pprint(result) @@ -739,7 +767,7 @@ Finally, we compare the two: '? ++++ ^ ^\n', '+ 5. Flat is better than nested.\n'] -As a single multi-line string it looks like this: +As a single multi-line string it looks like this:: >>> import sys >>> sys.stdout.writelines(result) @@ -758,14 +786,14 @@ As a single multi-line string it looks like this: .. _difflib-interface: A command-line interface to difflib ------------------------------------ +................................... This example shows how to use difflib to create a ``diff``-like utility. .. literalinclude:: ../includes/diff.py ndiff example -------------- +............. This example shows how to use :func:`difflib.ndiff`. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 1f7014e9cd426f0..af654f7f82323af 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -139,7 +139,7 @@ code. .. class:: Bytecode(x, *, first_line=None, current_offset=None,\ show_caches=False, adaptive=False, show_offsets=False,\ - show_positions=False) + show_positions=False, show_jit=False) Analyse the bytecode corresponding to a function, generator, asynchronous generator, coroutine, method, string of source code, or a code object (as @@ -170,6 +170,9 @@ code. If *show_positions* is ``True``, :meth:`.dis` will include instruction source code positions in the output. + If *show_jit* is ``True``, :meth:`.dis` will show ``ENTER_EXECUTOR`` + instructions, which mark JIT entry points and are hidden by default. + .. classmethod:: from_traceback(tb, *, show_caches=False) Construct a :class:`Bytecode` instance from the given traceback, setting @@ -205,6 +208,9 @@ code. .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + Example: .. doctest:: @@ -259,7 +265,8 @@ operation is being performed, so the intermediate analysis object isn't useful: .. function:: dis(x=None, *, file=None, depth=None, show_caches=False,\ - adaptive=False, show_offsets=False, show_positions=False) + adaptive=False, show_offsets=False, show_positions=False,\ + show_jit=False) Disassemble the *x* object. *x* can denote either a module, a class, a method, a function, a generator, an asynchronous generator, a coroutine, @@ -286,6 +293,9 @@ operation is being performed, so the intermediate analysis object isn't useful: If *adaptive* is ``True``, this function will display specialized bytecode that may be different from the original bytecode. + If *show_jit* is ``True``, this function will show ``ENTER_EXECUTOR`` + instructions, which mark JIT entry points and are hidden by default. + .. versionchanged:: 3.4 Added *file* parameter. @@ -304,8 +314,11 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False,\ - show_offset=False, show_positions=False) + show_offsets=False, show_positions=False, show_jit=False) Disassemble the top-of-stack function of a traceback, using the last traceback if none was passed. The instruction causing the exception is @@ -326,10 +339,14 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False,\ - adaptive=False, show_offsets=False, show_positions=False) + adaptive=False, show_offsets=False, show_positions=False,\ + show_jit=False) disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False,\ - show_offsets=False, show_positions=False) + show_offsets=False, show_positions=False, show_jit=False) Disassemble a code object, indicating the last instruction if *lasti* was provided. The output is divided in the following columns: @@ -362,7 +379,10 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.14 Added the *show_positions* parameter. -.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) + .. versionchanged:: next + Added the *show_jit* parameter. + +.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False, show_jit=False) Return an iterator over the instructions in the supplied function, method, source code string or code object. @@ -377,6 +397,8 @@ operation is being performed, so the intermediate analysis object isn't useful: The *adaptive* parameter works as it does in :func:`dis`. + The *show_jit* parameter works as it does in :func:`dis`. + .. versionadded:: 3.4 .. versionchanged:: 3.11 @@ -388,6 +410,9 @@ operation is being performed, so the intermediate analysis object isn't useful: field populated (regardless of the value of *show_caches*) and it no longer generates separate items for the cache entries. + .. versionchanged:: next + Added the *show_jit* parameter. + .. function:: findlinestarts(code) This generator function uses the :meth:`~codeobject.co_lines` method @@ -400,7 +425,7 @@ operation is being performed, so the intermediate analysis object isn't useful: .. versionchanged:: 3.10 The :pep:`626` :meth:`~codeobject.co_lines` method is used instead of the - :attr:`~codeobject.co_firstlineno` and :attr:`~codeobject.co_lnotab` + :attr:`~codeobject.co_firstlineno` and :attr:`!codeobject.co_lnotab` attributes of the :ref:`code object `. .. versionchanged:: 3.13 @@ -1031,12 +1056,6 @@ iterations of the loop. The compiler tries to use :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. -.. opcode:: DELETE_NAME (namei) - - Implements ``del name``, where *namei* is the index into :attr:`~codeobject.co_names` - attribute of the :ref:`code object `. - - .. opcode:: UNPACK_SEQUENCE (count) Unpacks ``STACK[-1]`` into *count* individual values, which are put onto the stack @@ -1076,27 +1095,11 @@ iterations of the loop. where *namei* is the index of name in :attr:`~codeobject.co_names` of the :ref:`code object `. -.. opcode:: DELETE_ATTR (namei) - - Implements:: - - obj = STACK.pop() - del obj.name - - where *namei* is the index of name into :attr:`~codeobject.co_names` of the - :ref:`code object `. - - .. opcode:: STORE_GLOBAL (namei) Works as :opcode:`STORE_NAME`, but stores the name as a global. -.. opcode:: DELETE_GLOBAL (namei) - - Works as :opcode:`DELETE_NAME`, but deletes a global name. - - .. opcode:: LOAD_CONST (consti) Pushes ``co_consts[consti]`` onto the stack. @@ -1896,25 +1899,29 @@ iterations of the loop. The operand determines which intrinsic function is called: - +----------------------------------------+-----------------------------------+ - | Operand | Description | - +========================================+===================================+ - | ``INTRINSIC_2_INVALID`` | Not valid | - +----------------------------------------+-----------------------------------+ - | ``INTRINSIC_PREP_RERAISE_STAR`` | Calculates the | - | | :exc:`ExceptionGroup` to raise | - | | from a ``try-except*``. | - +----------------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEVAR_WITH_BOUND`` | Creates a :class:`typing.TypeVar` | - | | with a bound. | - +----------------------------------------+-----------------------------------+ - | ``INTRINSIC_TYPEVAR_WITH_CONSTRAINTS`` | Creates a | - | | :class:`typing.TypeVar` with | - | | constraints. | - +----------------------------------------+-----------------------------------+ - | ``INTRINSIC_SET_FUNCTION_TYPE_PARAMS`` | Sets the ``__type_params__`` | - | | attribute of a function. | - +----------------------------------------+-----------------------------------+ + +------------------------------------------+-----------------------------------+ + | Operand | Description | + +==========================================+===================================+ + | ``INTRINSIC_2_INVALID`` | Not valid | + +------------------------------------------+-----------------------------------+ + | ``INTRINSIC_PREP_RERAISE_STAR`` | Calculates the | + | | :exc:`ExceptionGroup` to raise | + | | from a ``try-except*``. | + +------------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR_WITH_BOUND`` | Creates a :class:`typing.TypeVar` | + | | with a bound. | + +------------------------------------------+-----------------------------------+ + | ``INTRINSIC_TYPEVAR_WITH_CONSTRAINTS`` | Creates a | + | | :class:`typing.TypeVar` with | + | | constraints. | + +------------------------------------------+-----------------------------------+ + | ``INTRINSIC_SET_FUNCTION_TYPE_PARAMS`` | Sets the ``__type_params__`` | + | | attribute of a function. | + +------------------------------------------+-----------------------------------+ + | ``INTRINSIC_ADD_CONDITIONAL_ANNOTATION`` | Adds an annotation index to the | + | | ``__conditional_annotations__`` | + | | set. | + +------------------------------------------+-----------------------------------+ .. versionadded:: 3.12 diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 3298697af8511b6..4b61c07919dd025 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -1162,6 +1162,12 @@ from text files and modules with doctests: .. versionchanged:: 3.15 Run each example as a :ref:`subtest `. + .. versionchanged:: next + Report every example, as in verbose mode, if the test runner reports + more than the test names, i.e. its + :attr:`~unittest.TestResult.verbosity` is 3 or higher (for example + with ``python -m unittest -vv``). + Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 5754c2b65b239f9..11b4d908fcd7d60 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -96,7 +96,7 @@ Here are the methods of the :class:`Message` class: text = fp.getvalue() If the message object contains binary data that is not encoded according - to RFC standards, the non-compliant data will be replaced by unicode + to RFC standards, the non-compliant data will be replaced by Unicode "unknown character" code points. (See also :meth:`.as_bytes` and :class:`~email.generator.BytesGenerator`.) diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index 04a41667f7dc2cd..b28160082f6858c 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -96,9 +96,9 @@ Currently the email package provides only one concrete content manager, This content manager provides only a minimum interface beyond that provided by :class:`~email.message.Message` itself: it deals only with text, raw - byte strings, and :class:`~email.message.Message` objects. Nevertheless, it + bytes, and :class:`~email.message.Message` objects. Nevertheless, it provides significant advantages compared to the base API: ``get_content`` on - a text part will return a unicode string without the application needing to + a text part will return a string without the application needing to manually decode it, ``set_content`` provides a rich set of options for controlling the headers added to a part and controlling the content transfer encoding, and it enables the use of the various ``add_`` methods, thereby @@ -111,7 +111,7 @@ Currently the email package provides only one concrete content manager, parts), or a ``bytes`` object (for all other non-multipart types). Raise a :exc:`KeyError` if called on a ``multipart``. If the part is a ``text`` part and *errors* is specified, use it as the error handler when - decoding the payload to unicode. The default error handler is + decoding the payload to a string. The default error handler is ``replace``. .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8', \ diff --git a/Doc/library/email.examples.rst b/Doc/library/email.examples.rst index 492a8354d8bf852..7586cbd25f9f3be 100644 --- a/Doc/library/email.examples.rst +++ b/Doc/library/email.examples.rst @@ -7,7 +7,7 @@ Here are a few examples of how to use the :mod:`email` package to read, write, and send simple email messages, as well as more complex MIME messages. First, let's see how to create and send a simple text message (both the -text content and the addresses may contain unicode characters): +text content and the addresses may contain Unicode characters): .. literalinclude:: ../includes/email-simple.py diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index e7e21d036e07deb..65e2d0367fd7684 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -49,7 +49,7 @@ For example:: Notice here how we wanted the :mailheader:`Subject` field to contain a non-ASCII character? We did this by creating a :class:`Header` instance and passing in -the character set that the byte string was encoded in. When the subsequent +the character set to use when encoding it. When the subsequent :class:`~email.message.Message` instance was flattened, the :mailheader:`Subject` field was properly :rfc:`2047` encoded. MIME-aware mail readers would show this header using the embedded ISO-8859-1 character. @@ -150,7 +150,7 @@ Here is the :class:`Header` class description: .. method:: __str__() Returns an approximation of the :class:`Header` as a string, using an - unlimited line length. All pieces are converted to unicode using the + unlimited line length. All pieces are decoded using the specified encoding and joined together appropriately. Any pieces with a charset of ``'unknown-8bit'`` are decoded as ASCII using the ``'replace'`` error handler. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index c6924a0ac29c972..1cdc0b80c19dc5c 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -40,7 +40,7 @@ headers. *name* and *value* are passed to ``BaseHeader`` from the :attr:`~email.policy.EmailPolicy.header_factory` call. The string value of - any header object is the *value* fully decoded to unicode. + any header object is the *value* fully decoded to a string. This base class defines the following read-only properties: @@ -93,11 +93,12 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to a string). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid - unicode characters as well so that it can parse un-encoded header values. + Unicode characters as well so that it can parse un-encoded header values. ``BaseHeader``'s ``__new__`` then creates the header instance, and calls its ``init`` method. The specialized class only needs to provide an ``init`` @@ -125,7 +126,7 @@ headers. mechanism for encoding non-ASCII text as ASCII characters within a header value. When a *value* containing encoded words is passed to the constructor, the ``UnstructuredHeader`` parser converts such encoded words - into unicode, following the :rfc:`2047` rules for unstructured text. The + into a string, following the :rfc:`2047` rules for unstructured text. The parser uses heuristics to attempt to decode certain non-compliant encoded words. Defects are registered in such cases, as well as defects for issues such as invalid characters within the encoded words or the non-encoded text. @@ -202,8 +203,8 @@ headers. the list of addresses is "flattened" into a one dimensional list). The ``decoded`` value of the header will have all encoded words decoded to - unicode. :class:`~encodings.idna` encoded domain names are also decoded to - unicode. The ``decoded`` value is set by :ref:`joining ` the + a string. :class:`~encodings.idna` encoded domain names are also decoded to + a string. The ``decoded`` value is set by :ref:`joining ` the :class:`str` value of the elements of the ``groups`` attribute with ``', '``. @@ -391,7 +392,7 @@ construct structured values to assign to specific headers. *domain*, in which case *username* and *domain* will be parsed from the *addr_spec*. An *addr_spec* must be a properly RFC quoted string; if it is not ``Address`` will raise an error. Unicode characters are allowed and - will be property encoded when serialized. However, per the RFCs, unicode is + will be property encoded when serialized. However, per the RFCs, Unicode is *not* allowed in the username portion of the address. .. attribute:: display_name diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8f6e4218c97b38c..82a2b305f8daedb 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -403,11 +403,26 @@ added matters. To illustrate:: .. attribute:: utf8 If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages + headers by encoding them as :rfc:`2047` "encoded words". If ``True``, + follow :rfc:`6532` and use ``utf-8`` encoding for headers. Messages formatted in this way may be passed to SMTP servers that support the ``SMTPUTF8`` extension (:rfc:`6531`). + When ``False``, the generator will raise + :exc:`~email.errors.HeaderWriteError` if any header includes non-ASCII + characters in a context where :rfc:`2047` does not permit encoded words. + This particularly applies to mailboxes ("addr-spec") with non-ASCII + characters, which can be created via + :class:`~email.headerregistry.Address`. To use a mailbox with a non-ASCII + domain name with ``utf8=False``, first encode the domain using the + third-party :pypi:`idna` or :pypi:`uts46` module or with + :mod:`encodings.idna`. It is not possible to use a non-ASCII username + ("local-part") in a mailbox when ``utf8=False``. + + .. versionchanged:: 3.15 + Can trigger the raising of :exc:`~email.errors.HeaderWriteError`. + (Earlier versions incorrectly applied :rfc:`2047` in certain contexts, + mostly notably in addr-specs.) .. attribute:: refold_source @@ -490,7 +505,7 @@ added matters. To illustrate:: Otherwise the *name*, and the *value* with any CR or LF characters removed, are passed to the ``header_factory``, and the resulting header object is returned. Any surrogateescaped bytes get turned into - the unicode unknown-character glyph. + the Unicode unknown-character glyph. .. method:: fold(name, value) @@ -585,10 +600,10 @@ the email package is changed from the Python 3.2 API in the following ways: From the application view, this means that any header obtained through the :class:`~email.message.EmailMessage` is a header object with extra -attributes, whose string value is the fully decoded unicode value of the +attributes, whose string value is the fully decoded value of the header. Likewise, a header may be assigned a new value, or a new header -created, using a unicode string, and the policy will take care of converting -the unicode string into the correct RFC encoded form. +created, using a string, and the policy will take care of converting +the string into the correct RFC encoded form. The header objects and their attributes are described in :mod:`~email.headerregistry`. diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 98b47ffd74096c0..d1a3d9a63eca2ee 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -54,7 +54,7 @@ server. The email package does its best to hide the details of the various governing RFCs from the application. Conceptually the application should be able to -treat the email message as a structured tree of unicode text and binary +treat the email message as a structured tree of Unicode text and binary attachments, without having to worry about how these are represented when serialized. In practice, however, it is often necessary to be aware of at least some of the rules governing MIME messages and their structure, @@ -84,7 +84,7 @@ to advanced applications. Following those is a set of examples of using the fundamental parts of the APIs covered in the preceding sections. -The foregoing represent the modern (unicode friendly) API of the email package. +The foregoing represent the modern (Unicode friendly) API of the email package. The remaining sections, starting with the :class:`~email.message.Message` class, cover the legacy :data:`~email.policy.compat32` API that deals much more directly with the details of how email messages are represented. The diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index e0d2c19a3b0737a..acf7093ae10a807 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -70,7 +70,7 @@ of the new API. Add *strict* optional parameter and reject malformed inputs by default. -.. function:: formataddr(pair, charset='utf-8') +.. function:: formataddr(pair, charset='utf-8', *, strict=True) The inverse of :meth:`parseaddr`, this takes a 2-tuple of the form ``(realname, email_address)`` and returns the string value suitable for a :mailheader:`To` or @@ -82,9 +82,16 @@ of the new API. characters. Can be an instance of :class:`str` or a :class:`~email.charset.Charset`. Defaults to ``utf-8``. + If *strict* is true (the default), raise :exc:`ValueError` for inputs that + contain CR or LF, which are not allowed in an email address. Set *strict* + to ``False`` to allow non-strict inputs. + .. versionchanged:: 3.3 Added the *charset* option. + .. versionchanged:: next + Added the *strict* parameter. + .. function:: getaddresses(fieldvalues, *, strict=True) @@ -205,7 +212,7 @@ of the new API. When a header parameter is encoded in :rfc:`2231` format, :meth:`Message.get_param ` may return a 3-tuple containing the character set, - language, and value. :func:`collapse_rfc2231_value` turns this into a unicode + language, and value. :func:`collapse_rfc2231_value` turns this into a string. Optional *errors* is passed to the *errors* argument of :class:`str`'s :func:`~str.encode` method; it defaults to ``'replace'``. Optional *fallback_charset* specifies the character set to use if the one in the diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 0a1d2a0bac33abc..559a76bb3963e09 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -115,30 +115,30 @@ Module contents :class:`StrEnum` defaults to the lower-cased version of the member name, while other Enums default to 1 and increase from there. - :func:`~enum.property` + :deco:`~enum.property` Allows :class:`Enum` members to have attributes without conflicting with member names. The ``value`` and ``name`` attributes are implemented this way. - :func:`unique` + :deco:`unique` Enum class decorator that ensures only one name is bound to any one value. - :func:`verify` + :deco:`verify` Enum class decorator that checks user-selectable constraints on an enumeration. - :func:`member` + :deco:`member` Make ``obj`` a member. Can be used as a decorator. - :func:`nonmember` + :deco:`nonmember` Do not make ``obj`` a member. Can be used as a decorator. - :func:`global_enum` + :deco:`global_enum` Modify the :class:`str() ` and :func:`repr` of an enum to show its members as belonging to the module instead of its class, @@ -222,7 +222,7 @@ Data types names of the members in *cls*:: >>> dir(Color) - ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__'] + ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__', '_generate_next_value_', '_missing_'] .. method:: EnumType.__getitem__(cls, name) @@ -240,7 +240,7 @@ Data types .. method:: EnumType.__len__(cls) - Returns the number of member in *cls*:: + Returns the number of members in *cls*:: >>> len(Color) 3 @@ -355,7 +355,7 @@ Data types ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) - ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] + ['__class__', '__doc__', '__eq__', '__hash__', '__module__', '_add_alias_', '_add_value_alias_', '_generate_next_value_', '_missing_', 'name', 'today', 'value'] .. method:: Enum._generate_next_value_(name, start, count, last_values) @@ -502,7 +502,7 @@ Data types Using :class:`auto` with :class:`Enum` results in integers of increasing value, starting with ``1``. - .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support`. .. method:: Enum._add_alias_ @@ -629,7 +629,7 @@ Data types >>> white in purple False - .. method:: __iter__(self): + .. method:: __iter__(self) Returns all contained non-alias members:: @@ -640,7 +640,7 @@ Data types .. versionadded:: 3.11 - .. method:: __len__(self): + .. method:: __len__(self) Returns number of members in flag:: @@ -651,7 +651,7 @@ Data types .. versionadded:: 3.11 - .. method:: __bool__(self): + .. method:: __bool__(self) Returns *True* if any members in flag, *False* otherwise:: @@ -688,7 +688,7 @@ Data types >>> purple ^ Color.GREEN - .. method:: __invert__(self): + .. method:: __invert__(self) Returns all the flags in *type(self)* that are not in *self*:: @@ -977,9 +977,9 @@ Utilities and decorators *auto* can be used in place of a value. If used, the *Enum* machinery will call an :class:`Enum`'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For :class:`Enum` and :class:`IntEnum` that appropriate value will be the last value plus - one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater - than the highest value; for :class:`StrEnum` it will be the lower-cased version of + For :class:`Enum` and :class:`IntEnum` that appropriate value will be the highest value seen + plus one; for :class:`Flag` and :class:`IntFlag` it will be the first power-of-two greater + than the highest value seen; for :class:`StrEnum` it will be the lower-cased version of the member's name. Care must be taken if mixing *auto()* with manually specified values. @@ -989,8 +989,8 @@ Utilities and decorators * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``[, -3]`` is used to - create the ``THREE`` enum member) + * ``THIRD = [auto(), -3]`` will *not* work (``[, -3]`` is used to + create the ``THIRD`` enum member) .. versionchanged:: 3.11.1 @@ -1000,17 +1000,17 @@ Utilities and decorators ``_generate_next_value_`` can be overridden to customize the values used by *auto*. - .. note:: in 3.13 the default ``_generate_next_value_`` will always return + .. note:: In version 3.13 the default ``_generate_next_value_`` will always return the highest member value incremented by 1, and will fail if any member is an incompatible type. .. decorator:: property - A decorator similar to the built-in *property*, but specifically for + A decorator similar to the built-in :deco:`property`, but specifically for enumerations. It allows member attributes to have the same names as members themselves. - .. note:: the *property* and the member must be defined in separate classes; + .. note:: The *property* and the member must be defined in separate classes; for example, the *value* and *name* attributes are defined in the *Enum* class, and *Enum* subclasses can define members with the names ``value`` and ``name``. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 7fc6055aa9a8812..ecf62fb6391b1b1 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -215,6 +215,8 @@ The following exceptions are the exceptions that are usually raised. The object that was accessed for the named attribute. + When possible, :attr:`name` and :attr:`obj` are set automatically. + .. versionchanged:: 3.10 Added the :attr:`name` and :attr:`obj` attributes. @@ -984,6 +986,9 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + .. impl-detail:: The ``excs`` parameter may be any sequence, but lists and tuples are diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 677966a8b2eaabb..905e1acd433617d 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -31,7 +31,8 @@ tracebacks: * Each string is limited to 500 characters. * Only the filename, the function name and the line number are displayed. (no source code) -* It is limited to 100 frames and 100 threads. +* It is limited to 100 frames per thread, and 100 threads + (configurable via *max_threads*). * The order is reversed: the most recent call is shown first. By default, the Python traceback is written to :data:`sys.stderr`. To see @@ -55,16 +56,20 @@ at Python startup. Dumping the traceback --------------------- -.. function:: dump_traceback(file=sys.stderr, all_threads=True) +.. function:: dump_traceback(file=sys.stderr, all_threads=True, *, max_threads=100) Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. + ``False``, dump only the current thread. *max_threads* caps the number + of threads dumped. .. seealso:: :func:`traceback.print_tb`, which can be used to print a traceback object. .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + Dumping the C stack ------------------- @@ -100,7 +105,7 @@ instead of the stack, even if the operating system supports dumping stacks. Fault handler state ------------------- -.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True) +.. function:: enable(file=sys.stderr, all_threads=True, c_stack=True, *, max_threads=100) Enable the fault handler: install handlers for the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` @@ -116,6 +121,8 @@ Fault handler state traceback, unless the system does not support it. See :func:`dump_c_stack` for more information on compatibility. + *max_threads* caps the number of threads dumped when a fatal signal fires. + .. versionchanged:: 3.5 Added support for passing file descriptor to this function. @@ -133,6 +140,9 @@ Fault handler state .. versionchanged:: 3.14 The dump now displays the C stack trace if *c_stack* is true. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: disable() Disable the fault handler: uninstall the signal handlers installed by @@ -146,7 +156,7 @@ Fault handler state Dumping the tracebacks after a timeout -------------------------------------- -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) +.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False, *, max_threads=100) Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call @@ -154,7 +164,7 @@ Dumping the tracebacks after a timeout :c:func:`!_exit` exits the process immediately, which means it doesn't do any cleanup like flushing file buffers.) If the function is called twice, the new call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. + sub-second resolution. *max_threads* caps the number of threads dumped. The *file* must be kept open until the traceback is dumped or :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file @@ -168,6 +178,9 @@ Dumping the tracebacks after a timeout .. versionchanged:: 3.7 This function is now always available. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: cancel_dump_traceback_later() Cancel the last call to :func:`dump_traceback_later`. @@ -176,11 +189,12 @@ Dumping the tracebacks after a timeout Dumping the traceback on a user signal -------------------------------------- -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) +.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False, *, max_threads=100) Register a user signal: install a handler for the *signum* signal to dump the traceback of all threads, or of the current thread if *all_threads* is ``False``, into *file*. Call the previous handler if chain is ``True``. + *max_threads* caps the number of threads dumped. The *file* must be kept open until the signal is unregistered by :func:`unregister`: see :ref:`issue with file descriptors `. @@ -190,6 +204,9 @@ Dumping the traceback on a user signal .. versionchanged:: 3.5 Added support for passing file descriptor to this function. + .. versionchanged:: 3.15 + Added the *max_threads* keyword argument. + .. function:: unregister(signum) Unregister a user signal: uninstall the handler of the *signum* signal diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index f8365b44c5a502d..4488322032c681a 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -34,9 +34,9 @@ The :mod:`!filecmp` module defines the following functions: file changes. The entire cache may be cleared using :func:`clear_cache`. -.. function:: cmpfiles(dir1, dir2, common, shallow=True) +.. function:: cmpfiles(a, b, common, shallow=True) - Compare the files in the two directories *dir1* and *dir2* whose names are + Compare the files in the two directories *a* and *b* whose names are given by *common*. Returns three lists of file names: *match*, *mismatch*, diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ee654b7a83e203b..fc99b7fd1e7523a 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -51,7 +51,7 @@ Unless stated otherwise, "filename string" and "pattern string" either refer to functions documented below do not allow to mix a :class:`!bytes` pattern with a :class:`!str` filename, and vice-versa. -Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 +Finally, note that :deco:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. @@ -103,7 +103,8 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter .. function:: translate(pat) Return the shell-style pattern *pat* converted to a regular expression for - using with :func:`re.match`. The pattern is expected to be a :class:`str`. + using with :func:`re.prefixmatch`. The pattern is expected to be a + :class:`str`. Example: @@ -113,7 +114,7 @@ functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filter >>> regex '(?s:.*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foobar.txt') + >>> reobj.prefixmatch('foobar.txt') diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e8c4605d0578e28..013150535cb089c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -19,24 +19,25 @@ are always available. They are listed here in alphabetical order. | | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | | | | | :func:`float` | | :func:`max` | | |func-set|_ | | | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | -| | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | +| | :func:`bin` | | |func-frozendict|_ | | :func:`min` | | :func:`sentinel` | +| | :func:`bool` | | |func-frozenset|_ | | | | :func:`slice` | +| | :func:`breakpoint` | | | | **N** | | :func:`sorted` | +| | |func-bytearray|_ | | **G** | | :func:`next` | | :func:`staticmethod` | +| | |func-bytes|_ | | :func:`getattr` | | | | |func-str|_ | +| | | | :func:`globals` | | **O** | | :func:`sum` | +| | **C** | | | | :func:`object` | | :func:`super` | +| | :func:`callable` | | **H** | | :func:`oct` | | | +| | :func:`chr` | | :func:`hasattr` | | :func:`open` | | **T** | +| | :func:`classmethod` | | :func:`hash` | | :func:`ord` | | |func-tuple|_ | +| | :func:`compile` | | :func:`help` | | | | :func:`type` | +| | :func:`complex` | | :func:`hex` | | **P** | | | +| | | | | | :func:`pow` | | **V** | +| | **D** | | **I** | | :func:`print` | | :func:`vars` | +| | :func:`delattr` | | :func:`id` | | :func:`property` | | | +| | |func-dict|_ | | :func:`input` | | | | **Z** | +| | :func:`dir` | | :func:`int` | | | | :func:`zip` | +| | :func:`divmod` | | :func:`isinstance` | | | | | +| | | | :func:`issubclass` | | | | **_** | | | | | :func:`iter` | | | | :func:`__import__` | +-------------------------+-----------------------+-----------------------+-------------------------+ @@ -44,6 +45,7 @@ are always available. They are listed here in alphabetical order. used, with replacement texts to make the output in the table consistent .. |func-dict| replace:: ``dict()`` +.. |func-frozendict| replace:: ``frozendict()`` .. |func-frozenset| replace:: ``frozenset()`` .. |func-memoryview| replace:: ``memoryview()`` .. |func-set| replace:: ``set()`` @@ -63,14 +65,54 @@ are always available. They are listed here in alphabetical order. .. function:: aiter(async_iterable, /) + aiter(callable, /, stop_value, *, stop_exception=StopAsyncIteration) + aiter(callable, /, *, stop_exception) + + Return an :term:`asynchronous iterator` object. + The first argument is interpreted very differently + depending on the presence of the other arguments. + Without other arguments, + the single argument must be an :term:`asynchronous iterable`, + and the result is equivalent to calling ``x.__aiter__()``. + + If *stop_value* or *stop_exception* is given, + then the first argument must be a callable object. + The asynchronous iterator created in this case + calls *callable* with no arguments and awaits the result + for each call to its :meth:`~object.__anext__` method; + if the awaited value is equal to *stop_value*, + or if the call raises an exception matching *stop_exception*, + :exc:`StopAsyncIteration` will be raised, + otherwise the value will be returned. + The callable is only called when the result of :meth:`~object.__anext__` + is awaited. + + *stop_exception* is an exception class or a tuple of exception classes. + If *stop_value* is not specified, + the iteration stops only when the callable raises an exception. + If the callable raises :exc:`StopAsyncIteration` + which does not match *stop_exception*, + it is replaced with a :exc:`RuntimeError`, + as for asynchronous generators (see :pep:`525`). + + For example, reading fixed-size chunks from an asynchronous stream + until the end of file is reached:: - Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. - Equivalent to calling ``x.__aiter__()``. + from functools import partial + async for chunk in aiter(partial(reader.read, 1024), b''): + process_chunk(chunk) + + Or consuming an :class:`asyncio.Queue` until it is shut down:: - Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. + from asyncio import QueueShutDown + async for item in aiter(queue.get, stop_exception=QueueShutDown): + process_item(item) .. versionadded:: 3.10 + .. versionchanged:: next + Added the *stop_value* and *stop_exception* parameters. + .. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable @@ -483,10 +525,10 @@ are always available. They are listed here in alphabetical order. :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. + See also :ref:`typesmapping` for documentation about this class. - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. + For other containers see the built-in :class:`frozendict`, :class:`list`, + :class:`set`, and :class:`tuple` classes, as well as the :mod:`collections` module. .. function:: dir() @@ -609,9 +651,10 @@ are always available. They are listed here in alphabetical order. untrusted user-supplied input will lead to security vulnerabilities. The *source* argument is parsed and evaluated as a Python expression - (technically speaking, a condition list) using the *globals* and *locals* - mappings as global and local namespace. If the *globals* dictionary is - present and does not contain a value for the key ``__builtins__``, a + (technically speaking, an :ref:`expression list `) + using the *globals* and *locals* mappings as global and local namespace. + If the *globals* dictionary is present and does not contain a value for the + key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is inserted under that key before *source* is parsed. Overriding ``__builtins__`` can be used to restrict or change the available @@ -631,6 +674,9 @@ are always available. They are listed here in alphabetical order. >>> eval('x+1') 2 + >>> eval("1, 2") + (1, 2) + This function can also be used to execute arbitrary code objects (such as those created by :func:`compile`). In this case, pass a code object instead of a string. If the code object has been compiled with ``'exec'`` as the @@ -644,7 +690,7 @@ are always available. They are listed here in alphabetical order. If the given source is a string, then leading and trailing spaces and tabs are stripped. - See :func:`ast.literal_eval` for a function that can safely evaluate strings + See :func:`ast.literal_eval` for a function to evaluate strings with expressions containing only literals. .. audit-event:: exec code_object eval @@ -864,12 +910,27 @@ are always available. They are listed here in alphabetical order. if *format_spec* is not an empty string. +.. _func-frozendict: +.. class:: frozendict(**kwargs) + frozendict(mapping, /, **kwargs) + frozendict(iterable, /, **kwargs) + :noindex: + + Create a new frozen dictionary. The :class:`frozendict` object is a built-in class. + See also :ref:`typesmapping` for documentation about this class. + + For other containers see the built-in :class:`dict`, :class:`list`, :class:`set`, + and :class:`tuple` classes, as well as the :mod:`collections` module. + + .. versionadded:: 3.15 + + .. _func-frozenset: .. class:: frozenset(iterable=(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and + *iterable*. :class:`frozenset` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`set`, :class:`list`, @@ -1107,13 +1168,13 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: issubclass(class, classinfo, /) +.. function:: issubclass(cls, classinfo, /) - Return ``True`` if *class* is a subclass (direct, indirect, or :term:`virtual + Return ``True`` if *cls* is a subclass (direct, indirect, or :term:`virtual `) of *classinfo*. A class is considered a subclass of itself. *classinfo* may be a tuple of class objects (or recursively, other such tuples) - or a :ref:`types-union`, in which case return ``True`` if *class* is a + or a :ref:`types-union`, in which case return ``True`` if *cls* is a subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` exception is raised. @@ -1122,22 +1183,34 @@ are always available. They are listed here in alphabetical order. .. function:: iter(iterable, /) - iter(callable, sentinel, /) + iter(callable, /, stop_value, *, stop_exception=StopIteration) + iter(callable, /, *, stop_exception) Return an :term:`iterator` object. The first argument is interpreted very - differently depending on the presence of the second argument. Without a - second argument, the single argument must be a collection object which supports the + differently depending on the presence of the other arguments. Without other + arguments, the single argument must be a collection object which supports the :term:`iterable` protocol (the :meth:`~object.__iter__` method), or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, - :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, + :exc:`TypeError` is raised. + + If *stop_value* or *stop_exception* is given, then the first argument must be a callable object. The iterator created in this case will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to - *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will + *stop_value*, or if the call raises an exception matching *stop_exception*, + :exc:`StopIteration` will be raised, otherwise the value will be returned. + *stop_exception* is an exception class or a tuple of exception classes. + If *stop_value* is not specified, + the iteration stops only when the callable raises an exception. + If the callable raises :exc:`StopIteration` + which does not match *stop_exception*, + it is replaced with a :exc:`RuntimeError`, + as for generators (see :pep:`479`). + See also :ref:`typeiter`. One useful application of the second form of :func:`iter` is to build a @@ -1149,6 +1222,19 @@ are always available. They are listed here in alphabetical order. for block in iter(partial(f.read, 64), b''): process_block(block) + *stop_exception* is useful for callables + which report exhaustion by raising an exception + instead of returning a special value. + For example, draining a queue:: + + import queue + for item in iter(input_queue.get_nowait, stop_exception=queue.Empty): + process_item(item) + + .. versionchanged:: next + Added the *stop_exception* parameter + and allowed passing *stop_value* by keyword. + .. function:: len(object, /) @@ -1663,7 +1749,7 @@ are always available. They are listed here in alphabetical order. If given, *doc* will be the docstring of the property attribute. Otherwise, the property will copy *fget*'s docstring (if it exists). This makes it possible to - create read-only properties easily using :func:`property` as a :term:`decorator`:: + create read-only properties easily using :deco:`property` as a :term:`decorator`:: class Parrot: def __init__(self): @@ -1797,7 +1883,7 @@ are always available. They are listed here in alphabetical order. :noindex: Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and + *iterable*. :class:`set` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`frozenset`, :class:`list`, @@ -1827,6 +1913,69 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. +.. class:: sentinel(name, /, *, repr=None) + + Return a new unique sentinel object. *name* must be a :class:`str`, and is + used by default as the returned object's representation:: + + >>> MISSING = sentinel("MISSING") + >>> MISSING + MISSING + + The optional *repr* argument can be used to specify a different representation:: + + >>> MISSING = sentinel("MISSING", repr="") + >>> MISSING + + + Sentinel objects are truthy and compare equal only to themselves. They are + intended to be compared with the :keyword:`is` operator. + + ``sentinel`` does not support subclassing. + + Shallow and deep copies of a sentinel object return the object itself. + + Sentinels are conventionally assigned to a variable with a matching name. + Sentinels defined in this way can be used in :term:`type hints `:: + + MISSING = sentinel("MISSING") + + def next_value(default: int | MISSING = MISSING): + ... + + Sentinel objects support the :ref:`| ` operator for use in type expressions. + + :mod:`Pickling ` is supported for sentinel objects that are + placed in the global scope of a module under a name matching the sentinel's + name, and for sentinels placed in class scopes with a name matching the + :term:`qualified name` of the sentinel. Other sentinels, such as those + defined in a function scope, are not picklable. The identity of the sentinel is preserved + after pickling:: + + import pickle + + PICKLABLE = sentinel("PICKLABLE") + + assert pickle.loads(pickle.dumps(PICKLABLE)) is PICKLABLE + + class Cls: + PICKLABLE = sentinel("Cls.PICKLABLE") + + assert pickle.loads(pickle.dumps(Cls.PICKLABLE)) is Cls.PICKLABLE + + Sentinel objects have the following attributes: + + .. attribute:: __name__ + + The sentinel's name. + + .. attribute:: __module__ + + The name of the module where the sentinel was created. This attribute is writable. + + .. versionadded:: 3.15 + + .. class:: slice(stop, /) slice(start, stop, step=None, /) @@ -1904,7 +2053,7 @@ are always available. They are listed here in alphabetical order. be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see - :func:`classmethod` for a variant that is useful for creating alternate class + :deco:`classmethod` for a variant that is useful for creating alternate class constructors. Like all decorators, it is also possible to call ``staticmethod`` as @@ -2095,6 +2244,11 @@ are always available. They are listed here in alphabetical order. in the same way that keywords in a class definition (besides *metaclass*) would. + Unlike a :keyword:`class` statement, the three argument form does not + call the metaclass ``__prepare__`` method (see :ref:`prepare`). Use + :func:`types.new_class` to dynamically create a class using the + appropriate metaclass. + See also :ref:`class-customization`. .. versionchanged:: 3.6 diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 265610db3caabd5..debb7a7c3c2cc0e 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -27,7 +27,7 @@ The :mod:`!functools` module defines the following functions: Returns the same as ``lru_cache(maxsize=None)``, creating a thin wrapper around a dictionary lookup for the function arguments. Because it never needs to evict old values, this is smaller and faster than - :func:`lru_cache` with a size limit. + :deco:`lru_cache` with a size limit. For example:: @@ -61,7 +61,7 @@ The :mod:`!functools` module defines the following functions: Transform a method of a class into a property whose value is computed once and then cached as a normal attribute for the life of the instance. Similar - to :func:`property`, with the addition of caching. Useful for expensive + to :deco:`property`, with the addition of caching. Useful for expensive computed properties of instances that are otherwise effectively immutable. Example:: @@ -75,8 +75,8 @@ The :mod:`!functools` module defines the following functions: def stdev(self): return statistics.stdev(self._data) - The mechanics of :func:`cached_property` are somewhat different from - :func:`property`. A regular property blocks attribute writes unless a + The mechanics of :deco:`cached_property` are somewhat different from + :deco:`property`. A regular property blocks attribute writes unless a setter is defined. In contrast, a *cached_property* allows writes. The *cached_property* decorator only runs on lookups and only when an @@ -108,14 +108,14 @@ The :mod:`!functools` module defines the following functions: (as such classes don't provide a ``__dict__`` attribute at all). If a mutable mapping is not available or if space-efficient key sharing is - desired, an effect similar to :func:`cached_property` can also be achieved by - stacking :func:`property` on top of :func:`lru_cache`. See - :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. + desired, an effect similar to :deco:`cached_property` can also be achieved by + stacking :deco:`property` on top of :deco:`lru_cache`. See + :ref:`faq-cache-method-calls` for more details on how this differs from :deco:`cached_property`. .. versionadded:: 3.8 .. versionchanged:: 3.12 - Prior to Python 3.12, ``cached_property`` included an undocumented lock to + Prior to Python 3.12, :deco:`!cached_property` included an undocumented lock to ensure that in multi-threaded usage the getter function was guaranteed to run only once per instance. However, the lock was per-property, not per-instance, which could result in unacceptably high lock contention. In @@ -386,9 +386,9 @@ The :mod:`!functools` module defines the following functions: only one positional argument is provided, but there are two placeholders that must be filled in. - If :func:`!partial` is applied to an existing :func:`!partial` object, - :data:`!Placeholder` sentinels of the input object are filled in with - new positional arguments. + If :func:`!partial` is applied to an existing + :ref:`partial object `, :data:`!Placeholder` sentinels of the + input object are filled in with new positional arguments. A placeholder can be retained by inserting a new :data:`!Placeholder` sentinel to the place held by a previous :data:`!Placeholder`: @@ -468,7 +468,7 @@ The :mod:`!functools` module defines the following functions: Roughly equivalent to:: - initial_missing = object() + initial_missing = sentinel('initial_missing') def reduce(function, iterable, /, initial=initial_missing): it = iter(iterable) @@ -732,7 +732,7 @@ The :mod:`!functools` module defines the following functions: function's :attr:`~function.__dict__`, i.e. the instance dictionary). To allow access to the original function for introspection and other purposes - (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function + (e.g. bypassing a caching decorator such as :deco:`lru_cache`), this function automatically adds a ``__wrapped__`` attribute to the wrapper that refers to the function being wrapped. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 652475886fc30fe..65533e7e57adc33 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -37,18 +37,11 @@ The :mod:`!gc` module provides the following functions: .. function:: collect(generation=2) - Perform a collection. The optional argument *generation* + With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. - Calling ``gc.collect(0)`` will perform a GC collection on the young generation. - - Calling ``gc.collect(1)`` will perform a GC collection on the young generation - and an increment of the old generation. - - Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection - The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -60,6 +53,9 @@ The :mod:`!gc` module provides the following functions: .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. + .. versionchanged:: 3.14.5 + ``generation=1`` performs collection of the middle generation. + .. function:: set_debug(flags) @@ -75,13 +71,9 @@ The :mod:`!gc` module provides the following functions: .. function:: get_objects(generation=None) - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not ``None``, return only the objects as follows: - - * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.14) - * 2: All objects in the old generation + returned. If *generation* is not ``None``, return only the objects tracked by + the collector that are in that generation. .. versionchanged:: 3.8 New *generation* parameter. @@ -89,6 +81,9 @@ The :mod:`!gc` module provides the following functions: .. versionchanged:: 3.14 Generation 1 is removed + .. versionchanged:: 3.14.5 + Generation 1 is reintroduced to maintain GC behavior from 3.13. + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -124,33 +119,28 @@ The :mod:`!gc` module provides the following functions: Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into two generations depending on whether they have - survived a collection. New objects are placed in the young generation. If an - object survives a collection it is moved into the old generation. - - In order to decide when to run, the collector keeps track of the number of object + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. For each collection, all the objects in the young generation and some - fraction of the old generation is collected. - - In the free-threaded build, the increase in process memory usage is also - checked before running the collector. If the memory usage has not increased - by 10% since the last collection and the net number of object allocations - has not exceeded 40 times *threshold0*, the collection is not run. + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. - The fraction of the old generation that is collected is **inversely** proportional - to *threshold1*. The larger *threshold1* is, the slower objects in the old generation - are collected. - For the default value of 10, 1% of the old generation is scanned during each collection. - - *threshold2* is ignored. - - See `Garbage collector design `_ for more information. + See `Garbage collector design `_ for more information. .. versionchanged:: 3.14 *threshold2* is ignored + .. versionchanged:: 3.14.5 + *threshold2* is restored to match Python 3.13 behavior. + .. function:: get_count() diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 5c63009e22d58c4..9995f114d063e82 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -62,6 +62,8 @@ exception: option ``--fo`` will match as ``--foo``, but ``--f`` will not match uniquely, so :exc:`GetoptError` will be raised. + If *longopts* is a string it gets treated as a list of a single element. + The return value consists of two elements: the first is a list of ``(option, value)`` pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of *args*). Each diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index fd96f3bbf6a574f..f60cc63414f764d 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -18,7 +18,7 @@ The :mod:`!getpass` module provides two functions: the string *prompt*, which defaults to ``'Password: '``. On Unix, the prompt is written to the file-like object *stream* using the replace error handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this + (:file:`/dev/tty`) or if that is unavailable to :data:`sys.stderr` (this argument is ignored on Windows). The *echo_char* argument controls how user input is displayed while typing. @@ -27,12 +27,12 @@ The :mod:`!getpass` module provides two functions: typed character is replaced by it. For example, ``echo_char='*'`` will display asterisks instead of the actual input. - If echo free input is unavailable getpass() falls back to printing - a warning message to *stream* and reading from ``sys.stdin`` and + If echo-free input is unavailable, :func:`getpass` falls back to printing + a warning message to *stream* and reading from :data:`sys.stdin` and issuing a :exc:`GetPassWarning`. .. note:: - If you call getpass from within IDLE, the input may be done in the + If you call :func:`getpass` from within IDLE, the input may be done in the terminal you launched IDLE from rather than the idle window itself. .. note:: diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index 2de16fe40362b3e..2ab7ba7df19cf14 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -51,19 +51,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -78,15 +78,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -223,20 +223,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -244,7 +244,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -322,7 +322,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -331,7 +331,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -352,7 +352,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -363,7 +363,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 52c449281533372..bbeb808159d446b 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -29,7 +29,7 @@ not by actually invoking a subshell. The pathnames are returned in no particular order. If you need a specific order, sort the results. -Files beginning with a dot (``.``) can only be matched by +By default, files beginning with a dot (``.``) can only be matched by patterns that also start with a dot, unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. For tilde and shell variable expansion, use :func:`os.path.expanduser` and @@ -70,7 +70,8 @@ The :mod:`!glob` module defines the following functions: pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not match. - If *include_hidden* is true, "``**``" pattern will match hidden directories. + If *include_hidden* is true, wildcards can match path segments that + begin with a dot (``.``). .. audit-event:: glob.glob pathname,recursive glob.glob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.glob @@ -83,6 +84,11 @@ The :mod:`!glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -106,6 +112,11 @@ The :mod:`!glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -121,7 +132,7 @@ The :mod:`!glob` module defines the following functions: Escape all special characters (``'?'``, ``'*'`` and ``'['``). This is useful if you want to match an arbitrary literal string that may have special characters in it. Special characters in drive/UNC - sharepoints are not escaped, e.g. on Windows + sharepoints are not escaped, for example on Windows ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. .. versionadded:: 3.4 @@ -130,7 +141,8 @@ The :mod:`!glob` module defines the following functions: .. function:: translate(pathname, *, recursive=False, include_hidden=False, seps=None) Convert the given path specification to a regular expression for use with - :func:`re.match`. The path specification can contain shell-style wildcards. + :func:`re.prefixmatch`. The path specification can contain shell-style + wildcards. For example: @@ -140,7 +152,7 @@ The :mod:`!glob` module defines the following functions: >>> regex '(?s:(?:.+/)?[^/]*\\.txt)\\z' >>> reobj = re.compile(regex) - >>> reobj.match('foo/bar/baz.txt') + >>> reobj.prefixmatch('foo/bar/baz.txt') Path separators and segments are meaningful to this function, unlike diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ed9fdaf1d727b08..9211e5f18c6b6e6 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -28,7 +28,7 @@ Note that additional file formats which can be decompressed by the The module defines the following items: -.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode='rb', compresslevel=6, encoding=None, errors=None, newline=None, *, mtime=None) Open a gzip-compressed file in binary or text mode, returning a :term:`file object`. @@ -43,9 +43,12 @@ The module defines the following items: The *compresslevel* argument is an integer from 0 to 9, as for the :class:`GzipFile` constructor. + The keyword-only argument *mtime* represents a Unix timestamp. + For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. + constructor: ``GzipFile(filename, mode, compresslevel, mtime=mtime)``. + In this case, the *encoding*, *errors* and *newline* arguments must not + be provided. For text mode, a :class:`GzipFile` object is created, and wrapped in an :class:`io.TextIOWrapper` instance with the specified encoding, error @@ -66,6 +69,10 @@ The module defines the following items: It is the default level used by most compression tools and a better tradeoff between speed and performance. + .. versionchanged:: next + Added keyword-only argument *mtime* which is passed to the class + constructor of :class:`~gzip.GzipFile`. + .. exception:: BadGzipFile An exception raised for invalid gzip files. It inherits from :exc:`OSError`. @@ -108,9 +115,13 @@ The module defines the following items: is no compression. The default is ``9``. The optional *mtime* argument is the timestamp requested by gzip. The time - is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. - If *mtime* is omitted or ``None``, the current time is used. Use *mtime* = 0 - to generate a compressed stream that does not depend on creation time. + is in Unix format, i.e., seconds since 00:00:00 UTC, January 1, 1970. Set + *mtime* to ``0`` to generate a compressed stream that does not depend on + creation time. If *mtime* is omitted or ``None``, the current time is used; + however, if the current time is outside the range 00:00:00 UTC, January 1, + 1970 through 06:28:15 UTC, February 7, 2106, or explicitly passed *mtime* + argument is outside the range ``0`` to ``2**32-1``, then the value ``0`` + is used instead. See below for the :attr:`mtime` attribute that is set when decompressing. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index ed0b0b2735b5c3d..5b666c5adac6456 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -50,12 +50,15 @@ hash supplied more than 2047 bytes of data at once in its constructor or .. index:: single: OpenSSL; (use in module hashlib) Constructors for hash algorithms that are always present in this module are -:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`, -:func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, -:func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`. -:func:`md5` is normally available as well, though it may be missing or blocked -if you are using a rare "FIPS compliant" build of Python. -These correspond to :data:`algorithms_guaranteed`. +:func:`md5`, :func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, +:func:`sha512`, :func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, +:func:`sha3_512`, :func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and +:func:`blake2s`. These correspond to :data:`algorithms_guaranteed`. + +Any of these may nonetheless be missing or blocked in unusual environments, +such as a rare "FIPS compliant" build of Python or when OpenSSL's "FIPS mode" +is configured to exclude some algorithms from its default provider. Calling +the constructor of an algorithm that is unavailable raises :exc:`ValueError`. Additional algorithms may also be available if your Python distribution's :mod:`!hashlib` was linked against a build of OpenSSL that provides others. diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 341a8337ba2ceb8..11f851d4f6c4b74 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -141,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -317,6 +317,18 @@ without further parsing: Data : alert("hello! ☺"); End tag : script +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: + + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + Parsing comments: .. doctest:: diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index ddf3d40d221fcd7..98ea09d8f72d8b6 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -433,6 +433,7 @@ HTTPConnection Objects The maximum number of allowed response headers to help prevent denial-of-service attacks. By default, the maximum number of allowed headers is set to 100. + The same limit applies to the trailer section of a chunked response. .. versionadded:: 3.15 diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 5ee783b7fae950f..ce65bc961a98a68 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -26,7 +26,7 @@ introduced with RFC 2965. .. note:: The various named parameters found in :mailheader:`Set-Cookie` and - :mailheader:`Set-Cookie2` headers (eg. ``domain`` and ``expires``) are + :mailheader:`Set-Cookie2` headers (for example, ``domain`` and ``expires``) are conventionally referred to as :dfn:`attributes`. To distinguish them from Python attributes, the documentation for this module uses the term :dfn:`cookie-attribute` instead. @@ -85,7 +85,7 @@ The following classes are provided: Constructor arguments should be passed as keyword arguments only. *blocked_domains* is a sequence of domain names that we never accept cookies - from, nor return cookies to. *allowed_domains* if not :const:`None`, this is a + from, nor return cookies to. *allowed_domains* if not ``None``, this is a sequence of the only domains for which we accept and return cookies. *secure_protocols* is a sequence of protocols for which secure cookies can be added to. By default *https* and *wss* (secure websocket) are considered @@ -93,12 +93,12 @@ The following classes are provided: :class:`CookiePolicy` and :class:`DefaultCookiePolicy` objects. :class:`DefaultCookiePolicy` implements the standard accept / reject rules for - Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (ie. cookies + Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (that is, cookies received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of 1) are treated according to the RFC 2965 rules. However, if RFC 2965 handling is turned off or :attr:`rfc2109_as_netscape` is ``True``, RFC 2109 cookies are 'downgraded' by the :class:`CookieJar` instance to Netscape cookies, by - setting the :attr:`version` attribute of the :class:`Cookie` instance to 0. + setting the :attr:`~Cookie.version` attribute of the :class:`Cookie` instance to 0. :class:`DefaultCookiePolicy` also provides some parameters to allow some fine-tuning of policy. @@ -107,7 +107,7 @@ The following classes are provided: This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not expected that users of :mod:`!http.cookiejar` construct their own :class:`Cookie` - instances. Instead, if necessary, call :meth:`make_cookies` on a + instances. Instead, if necessary, call :meth:`~CookieJar.make_cookies` on a :class:`CookieJar` instance. @@ -141,7 +141,7 @@ The following classes are provided: .. _cookie-jar-objects: -CookieJar and FileCookieJar Objects +CookieJar and FileCookieJar objects ----------------------------------- :class:`CookieJar` objects support the :term:`iterator` protocol for iterating over @@ -154,20 +154,27 @@ contained :class:`Cookie` objects. Add correct :mailheader:`Cookie` header to *request*. - If policy allows (ie. the :attr:`rfc2965` and :attr:`hide_cookie2` attributes of + If policy allows (that is, the :attr:`~CookiePolicy.rfc2965` and + :attr:`~CookiePolicy.hide_cookie2` attributes of the :class:`CookieJar`'s :class:`CookiePolicy` instance are true and false respectively), the :mailheader:`Cookie2` header is also added when appropriate. The *request* object (usually a :class:`urllib.request.Request` instance) - must support the methods :meth:`get_full_url`, :meth:`has_header`, - :meth:`get_header`, :meth:`header_items`, :meth:`add_unredirected_header` - and the attributes :attr:`host`, :attr:`!type`, :attr:`unverifiable` - and :attr:`origin_req_host` as documented by :mod:`urllib.request`. + must support the methods :meth:`~urllib.request.Request.get_full_url`, + :meth:`~urllib.request.Request.has_header`, + :meth:`~urllib.request.Request.get_header`, + :meth:`~urllib.request.Request.header_items`, + :meth:`~urllib.request.Request.add_unredirected_header` + and the attributes :attr:`~urllib.request.Request.host`, + :attr:`~urllib.request.Request.type`, :attr:`~urllib.request.Request.unverifiable` + and :attr:`~urllib.request.Request.origin_req_host` as documented by + :mod:`urllib.request`. .. versionchanged:: 3.3 - *request* object needs :attr:`origin_req_host` attribute. Dependency on a - deprecated method :meth:`get_origin_req_host` has been removed. + *request* object needs :attr:`~urllib.request.Request.origin_req_host` + attribute. Dependency on a deprecated method + :meth:`!get_origin_req_host` has been removed. .. method:: CookieJar.extract_cookies(response, request) @@ -180,20 +187,24 @@ contained :class:`Cookie` objects. as appropriate (subject to the :meth:`CookiePolicy.set_ok` method's approval). The *response* object (usually the result of a call to - :meth:`urllib.request.urlopen`, or similar) should support an :meth:`info` - method, which returns an :class:`email.message.Message` instance. + :meth:`urllib.request.urlopen`, or similar) should support an + :meth:`~http.client.HTTPResponse.info` method, which returns an + :class:`email.message.Message` instance. The *request* object (usually a :class:`urllib.request.Request` instance) - must support the method :meth:`get_full_url` and the attributes - :attr:`host`, :attr:`unverifiable` and :attr:`origin_req_host`, + must support the method :meth:`~urllib.request.Request.get_full_url` and + the attributes :attr:`~urllib.request.Request.host`, + :attr:`~urllib.request.Request.unverifiable` + and :attr:`~urllib.request.Request.origin_req_host`, as documented by :mod:`urllib.request`. The request is used to set default values for cookie-attributes as well as for checking that the cookie is allowed to be set. .. versionchanged:: 3.3 - *request* object needs :attr:`origin_req_host` attribute. Dependency on a - deprecated method :meth:`get_origin_req_host` has been removed. + *request* object needs :attr:`~urllib.request.Request.origin_req_host` + attribute. Dependency on a deprecated method + :meth:`!get_origin_req_host` has been removed. .. method:: CookieJar.set_policy(policy) @@ -236,13 +247,13 @@ contained :class:`Cookie` objects. Discard all session cookies. - Discards all contained cookies that have a true :attr:`discard` attribute + Discards all contained cookies that have a true :attr:`~Cookie.discard` attribute (usually because they had either no ``max-age`` or ``expires`` cookie-attribute, or an explicit ``discard`` cookie-attribute). For interactive browsers, the end of a session usually corresponds to closing the browser window. - Note that the :meth:`save` method won't save session cookies anyway, unless you - ask otherwise by passing a true *ignore_discard* argument. + Note that the :meth:`~FileCookieJar.save` method won't save session cookies + anyway, unless you ask otherwise by passing a true *ignore_discard* argument. :class:`FileCookieJar` implements the following additional methods: @@ -255,8 +266,9 @@ contained :class:`Cookie` objects. method unimplemented. *filename* is the name of file in which to save cookies. If *filename* is not - specified, :attr:`self.filename` is used (whose default is the value passed to - the constructor, if any); if :attr:`self.filename` is :const:`None`, + specified, :attr:`self.filename ` is used (whose + default is the value passed to the constructor, if any); if + :attr:`self.filename ` is ``None``, :exc:`ValueError` is raised. *ignore_discard*: save even cookies set to be discarded. *ignore_expires*: save @@ -349,7 +361,7 @@ writing. .. _cookie-policy-objects: -CookiePolicy Objects +CookiePolicy objects -------------------- Objects implementing the :class:`CookiePolicy` interface have the following @@ -433,7 +445,7 @@ setting and receiving any and all cookies (this is unlikely to be useful). .. _default-cookie-policy-objects: -DefaultCookiePolicy Objects +DefaultCookiePolicy objects --------------------------- Implements the standard rules for accepting and returning cookies. @@ -463,10 +475,10 @@ blocking some benign cookies). A domain blocklist and allowlist is provided (both off by default). Only domains not in the blocklist and present in the allowlist (if the allowlist is active) participate in cookie setting and returning. Use the *blocked_domains* -constructor argument, and :meth:`blocked_domains` and -:meth:`set_blocked_domains` methods (and the corresponding argument and methods -for *allowed_domains*). If you set an allowlist, you can turn it off again by -setting it to :const:`None`. +constructor argument, and :meth:`~DefaultCookiePolicy.blocked_domains` and +:meth:`~DefaultCookiePolicy.set_blocked_domains` methods (and the corresponding +argument and methods for *allowed_domains*). If you set an allowlist, you can +turn it off again by setting it to ``None``. Domains in block or allow lists that do not start with a dot must equal the cookie domain to be matched. For example, ``"example.com"`` matches a blocklist @@ -498,12 +510,12 @@ and ``".168.1.2"``, 192.168.1.2 is blocked, but 193.168.1.2 is not. .. method:: DefaultCookiePolicy.allowed_domains() - Return :const:`None`, or the sequence of allowed domains (as a tuple). + Return ``None``, or the sequence of allowed domains (as a tuple). .. method:: DefaultCookiePolicy.set_allowed_domains(allowed_domains) - Set the sequence of allowed domains, or :const:`None`. + Set the sequence of allowed domains, or ``None``. .. method:: DefaultCookiePolicy.is_not_allowed(domain) @@ -519,9 +531,9 @@ all be assigned to. .. attribute:: DefaultCookiePolicy.rfc2109_as_netscape If true, request that the :class:`CookieJar` instance downgrade :rfc:`2109` cookies - (ie. cookies received in a :mailheader:`Set-Cookie` header with a version + (that is, cookies received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of 1) to Netscape cookies by setting the version attribute of - the :class:`Cookie` instance to 0. The default value is :const:`None`, in which + the :class:`Cookie` instance to 0. The default value is ``None``, in which case RFC 2109 cookies are downgraded if and only if :rfc:`2965` handling is turned off. Therefore, RFC 2109 cookies are downgraded by default. @@ -574,7 +586,7 @@ both flags are set). .. attribute:: DefaultCookiePolicy.DomainStrictNoDots - When setting cookies, the 'host prefix' must not contain a dot (eg. + When setting cookies, the 'host prefix' must not contain a dot (for example, ``www.foo.bar.com`` can't set a cookie for ``.bar.com``, because ``www.foo`` contains a dot). @@ -582,7 +594,7 @@ both flags are set). .. attribute:: DefaultCookiePolicy.DomainStrictNonDomain Cookies that did not explicitly specify a ``domain`` cookie-attribute can only - be returned to a domain equal to the domain that set the cookie (eg. + be returned to a domain equal to the domain that set the cookie (for example, ``spam.example.com`` won't be returned cookies from ``example.com`` that had no ``domain`` cookie-attribute). @@ -597,7 +609,7 @@ combinations of the above flags: .. attribute:: DefaultCookiePolicy.DomainLiberal - Equivalent to 0 (ie. all of the above Netscape domain strictness flags switched + Equivalent to 0 (that is, all of the above Netscape domain strictness flags switched off). @@ -606,7 +618,7 @@ combinations of the above flags: Equivalent to ``DomainStrictNoDots|DomainStrictNonDomain``. -Cookie Objects +Cookie objects -------------- :class:`Cookie` instances have Python attributes roughly corresponding to the @@ -624,7 +636,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.version - Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and + Integer or ``None``. Netscape cookies have :attr:`version` 0. :rfc:`2965` and :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -637,13 +649,13 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.value - Cookie value (a string), or :const:`None`. + Cookie value (a string), or ``None``. .. attribute:: Cookie.port - String representing a port or a set of ports (eg. '80', or '80,8080'), or - :const:`None`. + String representing a port or a set of ports (for example, '80', or '80,8080'), or + ``None``. .. attribute:: Cookie.domain @@ -653,7 +665,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.path - Cookie path (a string, eg. ``'/acme/rocket_launchers'``). + Cookie path (a string, for example, ``'/acme/rocket_launchers'``). .. attribute:: Cookie.secure @@ -663,7 +675,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.expires - Integer expiry date in seconds since epoch, or :const:`None`. See also the + Integer expiry date in seconds since epoch, or ``None``. See also the :meth:`is_expired` method. @@ -675,18 +687,18 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.comment String comment from the server explaining the function of this cookie, or - :const:`None`. + ``None``. .. attribute:: Cookie.comment_url URL linking to a comment from the server explaining the function of this cookie, - or :const:`None`. + or ``None``. .. attribute:: Cookie.rfc2109 - ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie + ``True`` if this cookie was received as an :rfc:`2109` cookie (that is, the cookie arrived in a :mailheader:`Set-Cookie` header, and the value of the Version cookie-attribute in that header was 1). This attribute is provided because :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index b3fcd21c7e22448..4965c5fc3ba1d86 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -25,10 +25,8 @@ The character set, :data:`string.ascii_letters`, :data:`string.digits` and in a cookie name (as :attr:`~Morsel.key`). .. versionchanged:: 3.3 - Allowed '``:``' as a valid cookie name character. + Allowed ':' as a valid cookie name character. -.. versionchanged:: 3.15 - Allowed '``"``' as a valid cookie value character. .. note:: @@ -109,6 +107,12 @@ Cookie Objects The meaning for *attrs* is the same as in :meth:`output`. + .. deprecated-removed:: 3.15 3.19 + This method generates a JavaScript snippet to set cookies in the browser, + which is no longer considered a standard or recommended approach. + Use :meth:`~http.cookies.BaseCookie.output` instead to generate HTTP + headers. + .. method:: BaseCookie.load(rawdata) @@ -225,6 +229,12 @@ Morsel Objects The meaning for *attrs* is the same as in :meth:`output`. + .. deprecated-removed:: 3.15 3.19 + This method generates a JavaScript snippet to set cookies in the browser, + which is no longer considered a standard or recommended approach. + Use :meth:`~http.cookies.Morsel.output` instead to generate HTTP + headers. + .. method:: Morsel.OutputString(attrs=None) @@ -313,10 +323,3 @@ The following example demonstrates how to use the :mod:`!http.cookies` module. >>> print(C) Set-Cookie: number=7 Set-Cookie: string=seven - >>> import json - >>> C = cookies.SimpleCookie() - >>> C.load(f'cookies=7; mixins="{json.dumps({"chips": "dark chocolate"})}"; state=gooey') - >>> print(C) - Set-Cookie: cookies=7 - Set-Cookie: mixins="{"chips": "dark chocolate"}" - Set-Cookie: state=gooey diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index cb8b5f0df88d6cb..0d427db19c0fcdb 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -39,7 +39,17 @@ handler. Code to create and run the server looks like this:: This class builds on the :class:`~socketserver.TCPServer` class by storing the server address as instance variables named :attr:`server_name` and :attr:`server_port`. The server is accessible by the handler, typically - through the handler's :attr:`server` instance variable. + through the handler's :attr:`~socketserver.BaseRequestHandler.server` + instance variable. + + .. attribute:: server_name + + The HTTP server's fully qualified domain name. + + .. attribute:: server_port + + The HTTP server's port number obtained from *server_address*. + .. class:: ThreadingHTTPServer(server_address, RequestHandlerClass) @@ -60,7 +70,7 @@ handler. Code to create and run the server looks like this:: object fails with a :exc:`RuntimeError`. The *certfile* argument is the path to the SSL certificate chain file, - and the *keyfile* is the path to file containing the private key. + and the *keyfile* is the path to the file containing the private key. A *password* can be specified for files protected and wrapped with PKCS#8, but beware that this could possibly expose hardcoded passwords in clear. @@ -99,7 +109,7 @@ instantiation, of which this module provides three different variants: This class is used to handle the HTTP requests that arrive at the server. By itself, it cannot respond to any actual HTTP requests; it must be subclassed - to handle each request method (e.g. GET or POST). + to handle each request method (for example, ``'GET'`` or ``'POST'``). :class:`BaseHTTPRequestHandler` provides a number of class and instance variables, and methods for use by subclasses. @@ -140,7 +150,7 @@ instantiation, of which this module provides three different variants: .. attribute:: path - Contains the request path. If query component of the URL is present, + Contains the request path. If the query component of the URL is present, then ``path`` includes the query. Using the terminology of :rfc:`3986`, ``path`` here includes ``hier-part`` and the ``query``. @@ -190,7 +200,7 @@ instantiation, of which this module provides three different variants: Specifies a format string that should be used by :meth:`send_error` method for building an error response to the client. The string is filled by default with variables from :attr:`responses` based on the status code - that passed to :meth:`send_error`. + passed to :meth:`send_error`. .. attribute:: error_content_type @@ -238,10 +248,10 @@ instantiation, of which this module provides three different variants: .. method:: handle_expect_100() When an HTTP/1.1 conformant server receives an ``Expect: 100-continue`` - request header it responds back with a ``100 Continue`` followed by ``200 - OK`` headers. + request header it responds with a ``100 Continue`` followed by ``200 OK`` + headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can choose to send ``417 + want the client to continue. For example, the server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 @@ -295,8 +305,8 @@ instantiation, of which this module provides three different variants: .. method:: send_response_only(code, message=None) Sends the response header only, used for the purposes when ``100 - Continue`` response is sent by the server to the client. The headers not - buffered and sent directly the output stream.If the *message* is not + Continue`` response is sent by the server to the client. The headers are + not buffered and sent directly the output stream. If the *message* is not specified, the HTTP message corresponding the response *code* is sent. This method does not reject *message* containing CRLF sequences. @@ -338,7 +348,7 @@ instantiation, of which this module provides three different variants: to create custom error logging mechanisms. The *format* argument is a standard printf-style format string, where the additional arguments to :meth:`log_message` are applied as inputs to the formatting. The client - ip address and current date and time are prefixed to every message logged. + IP address and current date and time are prefixed to every message logged. .. method:: version_string() @@ -366,7 +376,8 @@ instantiation, of which this module provides three different variants: delays, it now always returns the IP address. -.. class:: SimpleHTTPRequestHandler(request, client_address, server, directory=None) +.. class:: SimpleHTTPRequestHandler(request, client_address, server, \ + *, directory=None, extra_response_headers=None) This class serves files from the directory *directory* and below, or the current directory if *directory* is not provided, directly @@ -378,6 +389,9 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.9 The *directory* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.15 + Added *extra_response_headers* parameter. + A lot of the work, such as parsing the request, is done by the base class :class:`BaseHTTPRequestHandler`. This class implements the :func:`do_GET` and :func:`do_HEAD` functions. @@ -390,6 +404,22 @@ instantiation, of which this module provides three different variants: This will be ``"SimpleHTTP/" + __version__``, where ``__version__`` is defined at the module level. + .. attribute:: default_content_type + + Specifies the Content-Type header value sent when the MIME type + cannot be guessed from the file extension of the requested URL. + By default, it is set to ``'application/octet-stream'``. + + .. versionadded:: 3.15 + + .. attribute:: index_pages + + Specifies the filenames that are treated as directory index pages. + + Defaults to ``("index.html", "index.htm")``. + + .. versionadded:: 3.12 + .. attribute:: extensions_map A dictionary mapping suffixes into MIME types, contains custom overrides @@ -400,6 +430,15 @@ instantiation, of which this module provides three different variants: This dictionary is no longer filled with the default system mappings, but only contains overrides. + .. attribute:: extra_response_headers + + A sequence of ``(name, value)`` pairs containing user-defined extra HTTP + response headers to add to each successful HTTP status 200 response. These + headers are not included in other status code responses. + + Headers that the server sends automatically such as ``Content-Type`` + will not be overwritten by :attr:`!extra_response_headers`. + The :class:`SimpleHTTPRequestHandler` class defines the following methods: .. method:: do_HEAD() @@ -413,8 +452,8 @@ instantiation, of which this module provides three different variants: The request is mapped to a local file by interpreting the request as a path relative to the current working directory. - If the request was mapped to a directory, the directory is checked for a - file named ``index.html`` or ``index.htm`` (in that order). If found, the + If the request was mapped to a directory, the directory is checked for + an index page as specified by :attr:`index_pages`. If found, the file's contents are returned; otherwise a directory listing is generated by calling the :meth:`list_directory` method. This method uses :func:`os.listdir` to scan the directory, and returns a ``404`` error @@ -432,6 +471,9 @@ instantiation, of which this module provides three different variants: followed by a ``'Content-Length:'`` header with the file's size and a ``'Last-Modified:'`` header with the file's modification time. + The instance attribute :attr:`extra_response_headers` is a sequence of + ``(name, value)`` pairs containing user-defined extra response headers. + Then follows a blank line signifying the end of the headers, and then the contents of the file are output. @@ -441,9 +483,32 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.7 Support of the ``'If-Modified-Since'`` header. -The :class:`SimpleHTTPRequestHandler` class can be used in the following -manner in order to create a very basic webserver serving files relative to -the current directory:: + .. method:: list_directory(path) + + Helper to list the contents of *path* when no index page is present. + + This returns either a :term:`file-like object` (which must be closed + by the caller) or ``None`` to indicate an error, in which case the + caller has nothing further to do. In either case, the headers are sent. + + .. method:: guess_type(path) + + Guess the type of the file at the given *path*. + + This returns a string of the form ``type/subtype``, usable for + a MIME Content-type header. + + The default implementation looks the file's extension up in + :attr:`extensions_map`, falling back to + :func:`mimetypes.guess_file_type` and then to + :attr:`default_content_type`. + + .. versionchanged:: 3.13 + Add :func:`mimetypes.guess_file_type` as a fallback. + + +The :class:`SimpleHTTPRequestHandler` class can be used to create a very basic +webserver serving files relative to the current directory as follows:: import http.server import socketserver @@ -459,7 +524,7 @@ the current directory:: :class:`SimpleHTTPRequestHandler` can also be subclassed to enhance behavior, such as using different index file names by overriding the class attribute -:attr:`index_pages`. +:attr:`~SimpleHTTPRequestHandler.index_pages`. .. _http-server-cli: @@ -469,7 +534,9 @@ Command-line interface :mod:`!http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve -files relative to the current directory:: +files relative to the current directory: + +.. code-block:: bash python -m http.server [OPTIONS] [port] @@ -480,7 +547,9 @@ The following options are accepted: .. option:: port The server listens to port 8000 by default. The default can be overridden - by passing the desired port number as an argument:: + by passing the desired port number as an argument: + + .. code-block:: bash python -m http.server 9000 @@ -489,7 +558,9 @@ The following options are accepted: Specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. By default, the server binds itself to all interfaces. For example, the following command causes the server to bind - to localhost only:: + to localhost only: + + .. code-block:: bash python -m http.server --bind 127.0.0.1 @@ -502,7 +573,9 @@ The following options are accepted: Specifies a directory to which it should serve the files. By default, the server uses the current directory. For example, the following command - uses a specific directory:: + uses a specific directory: + + .. code-block:: bash python -m http.server --directory /tmp/ @@ -512,15 +585,31 @@ The following options are accepted: Specifies the HTTP version to which the server is conformant. By default, the server is conformant to HTTP/1.0. For example, the following command - runs an HTTP/1.1 conformant server:: + runs an HTTP/1.1 conformant server: + + .. code-block:: bash python -m http.server --protocol HTTP/1.1 .. versionadded:: 3.11 +.. option:: --content-type + + Specifies the default Content-Type HTTP header used when the MIME type + cannot be guessed from the URL's file extension. By default, the server + uses ``'application/octet-stream'``: + + .. code-block:: bash + + python -m http.server --content-type text/html + + .. versionadded:: 3.15 + .. option:: --tls-cert - Specifies a TLS certificate chain for HTTPS connections:: + Specifies a TLS certificate chain for HTTPS connections: + + .. code-block:: bash python -m http.server --tls-cert fullchain.pem @@ -536,17 +625,28 @@ The following options are accepted: .. option:: --tls-password-file - Specifies the password file for password-protected private keys:: + Specifies the password file for password-protected private keys: + + .. code-block:: bash python -m http.server \ --tls-cert cert.pem \ --tls-key key.pem \ --tls-password-file password.txt - This option requires `--tls-cert`` to be specified. + This option requires ``--tls-cert`` to be specified. .. versionadded:: 3.14 +.. option:: -H, --header
+ + Specify an additional HTTP Response Header to send on successful HTTP 200 + responses. Can be used multiple times to send additional custom response + headers. Headers that are sent automatically by the server (for instance + Content-Type) will not be overwritten by the server. + + .. versionadded:: 3.15 + .. _http.server-security: @@ -556,13 +656,13 @@ Security considerations .. index:: pair: http.server; security :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling -requests, this makes it possible for files outside of the specified directory +requests which makes it possible for files outside of the specified directory to be served. Methods :meth:`BaseHTTPRequestHandler.send_header` and :meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input -and does not perform input validation such as checking for the presence of CRLF -sequences. Untrusted input may result in HTTP Header injection attacks. +and do not perform input validation such as checking for the presence of CRLF +sequences. Untrusted input may result in HTTP header injection attacks. Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index b29b02d3cf5fe8b..745683673ccf507 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -16,7 +16,7 @@ This module defines three classes, :class:`IMAP4`, :class:`IMAP4_SSL` and :class:`IMAP4_stream`, which encapsulate a connection to an IMAP4 server and implement a large subset of the IMAP4rev1 client protocol as defined in -:rfc:`2060`. It is backward compatible with IMAP4 (:rfc:`1730`) servers, but +:rfc:`3501`. It is backward compatible with IMAP4 (:rfc:`1730`) servers, but note that the ``STATUS`` command is not supported in IMAP4. .. include:: ../includes/wasm-notavail.rst @@ -89,6 +89,13 @@ There's also a subclass for secure connections: (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. + .. note:: + + With the default *ssl_context*, the connection is encrypted but the + server certificate and hostname are not verified. + To verify them, pass a context created by + :func:`ssl.create_default_context`. + The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. @@ -120,11 +127,11 @@ The second subclass allows for connections created by a child process: The following utility functions are defined: -.. function:: Internaldate2tuple(datestr) +.. function:: Internaldate2tuple(resp) - Parse an IMAP4 ``INTERNALDATE`` string and return corresponding local - time. The return value is a :class:`time.struct_time` tuple or - ``None`` if the string has wrong format. + Parse a :term:`bytes-like object` containing an IMAP4 ``INTERNALDATE`` + response and return the corresponding local time. The return value is a + :class:`time.struct_time` tuple or ``None`` if the input has wrong format. .. function:: Int2AP(num) @@ -132,9 +139,11 @@ The following utility functions are defined: [``A`` .. ``P``]. -.. function:: ParseFlags(flagstr) +.. function:: ParseFlags(resp) - Converts an IMAP4 ``FLAGS`` response to a tuple of individual flags. + Converts a :term:`bytes-like object` containing an IMAP4 ``FLAGS`` response + to a tuple of individual flags as :class:`bytes`. The return value is an + empty tuple if the input has wrong format. .. function:: Time2Internaldate(date_time) @@ -179,6 +188,23 @@ enclosed with either parentheses or double quotes) each string is quoted. However, the *password* argument to the ``LOGIN`` command is always quoted. If you want to avoid having an argument string quoted (eg: the *flags* argument to ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``). +Or you can quote the string yourself; +an argument that is already enclosed in double quotes is left unchanged. +In general, however, it is better to pass arguments unquoted +and let the module quote them as needed. + +Mailbox names are encoded as modified UTF-7 (:rfc:`3501`, section 5.1.3), +so a mailbox name containing non-ASCII characters can be passed as an +ordinary :class:`str`. +A :class:`str` that is already valid modified UTF-7 is left unchanged, +so that a name obtained from :meth:`~IMAP4.list` (raw ``bytes`` decoded to +text) round-trips; pass :class:`bytes` to send the exact bytes with no +encoding. +When ``UTF8=ACCEPT`` is enabled (see :meth:`~IMAP4.enable`), mailbox names +are sent as UTF-8 instead. + +.. versionchanged:: next + Non-ASCII mailbox names are automatically encoded as modified UTF-7. Most commands return a tuple: ``(type, [data, ...])`` where *type* is usually ``'OK'`` or ``'NO'``, and *data* is either the text from the command response, @@ -186,19 +212,94 @@ or mandated results from the command. Each *data* is either a ``bytes``, or a tuple. If a tuple, then the first part is the header of the response, and the second part contains the data (ie: 'literal' value). -The *message_set* options to commands below is a string specifying one or more -messages to be acted upon. It may be a simple message number (``'1'``), a range -of message numbers (``'2:4'``), or a group of non-contiguous ranges separated by -commas (``'1:3,6:9'``). A range can contain an asterisk to indicate an infinite -upper bound (``'3:*'``). +The *message_set* options to the commands below can be a string specifying +one or more messages to be acted upon. +It may be a simple message number (``'1'``), +a range of message numbers (``'2:4'``), +or a group of non-contiguous ranges separated by commas (``'1:3,6:9'``). +A range can contain an asterisk +to indicate an infinite upper bound (``'3:*'``). + +Alternatively it can be specified using integers and :class:`range` objects. +It may be a single message number or a sequence. +The sequence items may be integers, ``(start, stop)`` tuples +(where ``None`` or ``'*'`` stands for the last message), +or :class:`range` objects. +For example, ``[1, (3, 5), 8]`` and ``[range(1, 6), 8]`` +are both equivalent to ``'1,3:5,8'``. + +.. versionchanged:: next + Added support for the structured *message_set*. + +Command arguments that are parenthesized lists of atoms --- +such as the *flag_list* argument of :meth:`~IMAP4.store` and the *flags* +argument of :meth:`~IMAP4.append`, +the *names* argument of :meth:`~IMAP4.status`, +the *sort_criteria* argument of :meth:`~IMAP4.sort`, +or the *message_parts* argument of :meth:`~IMAP4.fetch` --- +can be passed as a sequence of strings instead of a single preformatted string. +For example, ``[r'\Seen', r'\Answered']`` +is equivalent to ``(\Seen \Answered)``. + +.. versionchanged:: next + Added support for passing these arguments as a sequence. + +.. _imap4-params: + +The value-bearing arguments of the search and fetch commands +can be quoted by hand, but this is error prone. +Instead, they may contain ``?`` placeholders that are substituted, and quoted +as required, from a *params* keyword argument, +in the manner of :mod:`sqlite3` parameter substitution:: + + # SEARCH FROM me@example.com SUBJECT "trip report" + M.search(None, 'FROM ? SUBJECT ?', params=['me@example.com', 'trip report']) + + # FETCH 1:5 (FLAGS BODY[HEADER.FIELDS (DATE FROM)]) + M.fetch('1:5', 'FLAGS BODY[HEADER.FIELDS ?]', params=[['DATE', 'FROM']]) + +The placeholders are: + +* ``?`` --- an ``astring``: a string (which will be quoted if necessary), + an integer, or a list of integers and/or strings + (which will be sent as a parenthesized list); +* ``?f`` --- a flag or a list of flags, sent verbatim without quoting; +* ``?s`` --- a *message_set* in the structured form described above. + +``??`` stands for a literal ``?``. + +Substitution is only performed when *params* is given; +if no *params* are given, an argument containing a literal ``?`` is unchanged. +The *params* keyword is accepted by :meth:`~IMAP4.search`, +:meth:`~IMAP4.fetch`, :meth:`~IMAP4.sort`, :meth:`~IMAP4.thread` and +:meth:`~IMAP4.uid`. + +.. versionadded:: next + The *params* keyword argument. An :class:`IMAP4` instance has the following methods: -.. method:: IMAP4.append(mailbox, flags, date_time, message) +.. method:: IMAP4.append(mailbox, flags, date_time, message, *, translate_line_endings=True) Append *message* to named mailbox. + *flags* may be ``None`` or a string of IMAP flag tokens. Multiple + flags are separated by spaces, for example ``r'\Seen \Answered'``. + If *flags* is not already enclosed in parentheses, parentheses are + added automatically. + + If *translate_line_endings* is true (the default), + line endings in *message* are translated to CRLF. + Pass ``False`` to send the message literal exactly as given, + which is required to preserve messages that contain bare CR or LF. + In that case *message* must already use CRLF line endings as required + by :rfc:`3501`; for example, serialize :mod:`email` messages using + :class:`email.policy.SMTP`. + + .. versionchanged:: next + Added the *translate_line_endings* parameter. + .. method:: IMAP4.authenticate(mechanism, authobject) @@ -232,10 +333,16 @@ An :class:`IMAP4` instance has the following methods: mailbox. This is the recommended command before ``LOGOUT``. -.. method:: IMAP4.copy(message_set, new_mailbox) +.. method:: IMAP4.copy(message_set, new_mailbox, *, uid=False) Copy *message_set* messages onto end of *new_mailbox*. + If *uid* is true, *message_set* is a set of UIDs and the ``UID COPY`` + command is used instead of ``COPY``. + + .. versionchanged:: next + Added the *uid* parameter. + .. method:: IMAP4.create(mailbox) @@ -262,19 +369,36 @@ An :class:`IMAP4` instance has the following methods: The :meth:`enable` method itself, and :RFC:`6855` support. -.. method:: IMAP4.expunge() +.. method:: IMAP4.expunge(message_set=None, *, uid=False) Permanently remove deleted items from selected mailbox. Generates an ``EXPUNGE`` response for each deleted message. Returned data contains a list of ``EXPUNGE`` message numbers in order received. + If *uid* is true, the ``UID EXPUNGE`` command (:rfc:`4315`) is used to remove + only the messages that both are marked as deleted and have a UID in + *message_set*. *message_set* is required in this case, and must be omitted + otherwise. -.. method:: IMAP4.fetch(message_set, message_parts) + .. versionchanged:: next + Added the *message_set* and *uid* parameters. + + +.. method:: IMAP4.fetch(message_set, message_parts, *, uid=False, params=None) Fetch (parts of) messages. *message_parts* should be a string of message part names enclosed within parentheses, eg: ``"(UID BODY[TEXT])"``. Returned data are tuples of message part envelope and data. + If *uid* is true, *message_set* is a set of UIDs and the message numbers in + the response are UIDs (``UID FETCH``). + + If *params* is given, ``?`` placeholders in *message_parts* are substituted + with the quoted parameters (see :ref:`the placeholders `). + + .. versionchanged:: next + Added the *params* and *uid* parameters. + .. method:: IMAP4.getacl(mailbox) @@ -300,6 +424,19 @@ An :class:`IMAP4` instance has the following methods: of the IMAP4 QUOTA extension defined in rfc2087. +.. method:: IMAP4.id(fields=None) + + Send client identification information to the server + and return the identification information sent back by the server + (the ``ID`` command, defined in :rfc:`2971`). + *fields* is a mapping of field names to values + (for example, ``{'name': 'myclient', 'version': '1.0'}``); + a value can be ``None``. + The server must support the ``ID`` capability. + + .. versionadded:: next + + .. method:: IMAP4.idle(duration=None) Return an :class:`!Idler`: an iterable context manager implementing the @@ -387,7 +524,7 @@ An :class:`IMAP4` instance has the following methods: .. versionadded:: 3.14 -.. method:: IMAP4.list([directory[, pattern]]) +.. method:: IMAP4.list(directory='', pattern='*') List mailbox names in *directory* matching *pattern*. *directory* defaults to the top-level mail folder, and *pattern* defaults to match anything. Returned @@ -402,13 +539,30 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.login_cram_md5(user, password) Force use of ``CRAM-MD5`` authentication when identifying the client to protect - the password. Will only work if the server ``CAPABILITY`` response includes the - phrase ``AUTH=CRAM-MD5``. + the password. It will only work if the server ``CAPABILITY`` response includes + the phrase ``AUTH=CRAM-MD5``. .. versionchanged:: 3.15 An :exc:`IMAP4.error` is raised if MD5 support is not available. +.. method:: IMAP4.login_plain(user, password) + + Authenticate using the ``PLAIN`` SASL mechanism (:rfc:`4616`). + + This is a plaintext authentication mechanism that can be used instead + of :meth:`login` when UTF-8 support is required (see :rfc:`6855`). + Since the credentials are only base64-encoded, not encrypted, this + method should only be used over a TLS-protected connection, such as + :class:`IMAP4_SSL` or after :meth:`starttls`. + + It will only work if the server supports the ``PLAIN`` mechanism, + which it need not advertise as ``AUTH=PLAIN`` in its ``CAPABILITY`` + response. + + .. versionadded:: next + + .. method:: IMAP4.logout() Shutdown connection to server. Returns server ``BYE`` response. @@ -417,13 +571,25 @@ An :class:`IMAP4` instance has the following methods: The method no longer ignores silently arbitrary exceptions. -.. method:: IMAP4.lsub(directory='""', pattern='*') +.. method:: IMAP4.lsub(directory='', pattern='*') List subscribed mailbox names in directory matching pattern. *directory* defaults to the top level directory and *pattern* defaults to match any mailbox. Returned data are tuples of message part envelope and data. +.. method:: IMAP4.move(message_set, new_mailbox, *, uid=False) + + Move *message_set* messages onto end of *new_mailbox*. + + The server must support the ``MOVE`` capability (:rfc:`6851`). + + If *uid* is true, *message_set* is a set of UIDs and the ``UID MOVE`` + command is used instead of ``MOVE``. + + .. versionadded:: next + + .. method:: IMAP4.myrights(mailbox) Show my ACLs for a mailbox (i.e. the rights that I have on mailbox). @@ -495,7 +661,7 @@ An :class:`IMAP4` instance has the following methods: code, instead of the usual type. -.. method:: IMAP4.search(charset, criterion[, ...]) +.. method:: IMAP4.search(charset, criterion[, ...], *, uid=False, params=None) Search mailbox for matching messages. *charset* may be ``None``, in which case no ``CHARSET`` will be specified in the request to the server. The IMAP @@ -504,13 +670,33 @@ An :class:`IMAP4` instance has the following methods: the ``UTF8=ACCEPT`` capability was enabled using the :meth:`enable` command. + If *uid* is true, the message numbers in the response are UIDs + (``UID SEARCH``). + + A criterion passed as :class:`str` is encoded to *charset* + (which must name a codec known to Python); + pass :class:`bytes` to send a criterion that is already encoded, + for example when *charset* is one that Python does not support. + When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), + the criterion is sent using the connection's encoding instead. + + If *params* is given, ``?`` placeholders in the criteria are substituted + with the quoted parameters (see :ref:`the placeholders `). + Example:: # M is a connected IMAP4 instance... - typ, msgnums = M.search(None, 'FROM', '"LDJ"') + typ, msgnums = M.search(None, 'FROM', '"John Smith"') # or: - typ, msgnums = M.search(None, '(FROM "LDJ")') + typ, msgnums = M.search(None, '(FROM "John Smith")') + + # or, letting the module quote the value (this is recommended): + typ, msgnums = M.search(None, 'FROM ?', params=['John Smith']) + + .. versionchanged:: next + Added the *params* and *uid* parameters. + ``str`` search criteria are encoded to *charset*. .. method:: IMAP4.select(mailbox='INBOX', readonly=False) @@ -556,7 +742,7 @@ An :class:`IMAP4` instance has the following methods: Returns socket instance used to connect to server. -.. method:: IMAP4.sort(sort_criteria, charset, search_criterion[, ...]) +.. method:: IMAP4.sort(sort_criteria, charset, search_criterion[, ...], *, uid=False, params=None) The ``sort`` command is a variant of ``search`` with sorting semantics for the results. Returned data contains a space separated list of matching message @@ -571,8 +757,21 @@ An :class:`IMAP4` instance has the following methods: the interpretation of strings in the searching criteria. It then returns the numbers of matching messages. + If *uid* is true, the message numbers in the response are UIDs (``UID SORT``). + + As with :meth:`search`, + a *search_criterion* passed as :class:`str` is encoded to *charset*; + pass :class:`bytes` to send one already encoded. + + If *params* is given, ``?`` placeholders in the search criteria are + substituted with the quoted parameters (see :ref:`the placeholders `). + This is an ``IMAP4rev1`` extension command. + .. versionchanged:: next + Added the *params* and *uid* parameters. + ``str`` search criteria are encoded to *charset*. + .. method:: IMAP4.starttls(ssl_context=None) @@ -581,6 +780,13 @@ An :class:`IMAP4` instance has the following methods: encryption on the IMAP connection. Please read :ref:`ssl-security` for best practices. + .. note:: + + With the default *ssl_context*, the connection is encrypted but the + server certificate and hostname are not verified. + To verify them, pass a context created by + :func:`ssl.create_default_context`. + .. versionadded:: 3.2 .. versionchanged:: 3.4 @@ -594,37 +800,43 @@ An :class:`IMAP4` instance has the following methods: Request named status conditions for *mailbox*. -.. method:: IMAP4.store(message_set, command, flag_list) +.. method:: IMAP4.store(message_set, command, flag_list, *, uid=False) Alters flag dispositions for messages in mailbox. *command* is specified by - section 6.4.6 of :rfc:`2060` as being one of "FLAGS", "+FLAGS", or "-FLAGS", + section 6.4.6 of :rfc:`3501` as being one of "FLAGS", "+FLAGS", or "-FLAGS", optionally with a suffix of ".SILENT". + If *uid* is true, *message_set* is a set of UIDs and the ``UID STORE`` + command is used instead of ``STORE``. + For example, to set the delete flag on all messages:: typ, data = M.search(None, 'ALL') for num in data[0].split(): - M.store(num, '+FLAGS', '\\Deleted') + M.store(num, '+FLAGS', r'\Deleted') M.expunge() .. note:: Creating flags containing ']' (for example: "[test]") violates :rfc:`3501` (the IMAP protocol). However, imaplib has historically - allowed creation of such tags, and popular IMAP servers, such as Gmail, + allowed creation of such flags, and popular IMAP servers, such as Gmail, accept and produce such flags. There are non-Python programs which also - create such tags. Although it is an RFC violation and IMAP clients and + create such flags. Although it is an RFC violation and IMAP clients and servers are supposed to be strict, imaplib still continues to allow - such tags to be created for backward compatibility reasons, and as of + such flags to be created for backward compatibility reasons, and as of Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. + .. versionchanged:: next + Added the *uid* parameter. + .. method:: IMAP4.subscribe(mailbox) Subscribe to new mailbox. -.. method:: IMAP4.thread(threading_algorithm, charset, search_criterion[, ...]) +.. method:: IMAP4.thread(threading_algorithm, charset, search_criterion[, ...], *, uid=False, params=None) The ``thread`` command is a variant of ``search`` with threading semantics for the results. Returned data contains a space separated list of thread members. @@ -642,16 +854,37 @@ An :class:`IMAP4` instance has the following methods: returns the matching messages threaded according to the specified threading algorithm. + If *uid* is true, the message numbers in the response are UIDs + (``UID THREAD``). + + As with :meth:`search`, + a *search_criterion* passed as :class:`str` is encoded to *charset*; + pass :class:`bytes` to send one already encoded. + + If *params* is given, ``?`` placeholders in the search criteria are + substituted with the quoted parameters (see :ref:`the placeholders `). + This is an ``IMAP4rev1`` extension command. + .. versionchanged:: next + Added the *params* and *uid* parameters. + ``str`` search criteria are encoded to *charset*. + -.. method:: IMAP4.uid(command, arg[, ...]) +.. method:: IMAP4.uid(command, arg[, ...], *, params=None) Execute command args with messages identified by UID, rather than message number. Returns response appropriate to command. At least one argument must be supplied; if none are provided, the server will return an error and an exception will be raised. + If *params* is given, ``?`` placeholders in the ``SEARCH``, ``SORT`` and + ``THREAD`` criteria or in the ``FETCH`` parts are substituted with the quoted + parameters (see :ref:`the placeholders `). + + .. versionchanged:: next + Added the *params* parameter. + .. method:: IMAP4.unsubscribe(mailbox) @@ -674,6 +907,20 @@ An :class:`IMAP4` instance has the following methods: The following attributes are defined on instances of :class:`IMAP4`: +.. attribute:: IMAP4.capabilities + + A tuple of the capabilities advertised by the server, in upper case. + + It is set when the connection is established, + and refreshed after a successful :meth:`~IMAP4.login`, + :meth:`~IMAP4.authenticate` or :meth:`~IMAP4.starttls`, + because the server can advertise different capabilities + in different connection states. + + .. versionchanged:: 3.14.7 + Refreshed after :meth:`~IMAP4.login` and :meth:`~IMAP4.authenticate`. + + .. attribute:: IMAP4.PROTOCOL_VERSION The most recent supported protocol in the ``CAPABILITY`` response from the @@ -695,6 +942,16 @@ The following attributes are defined on instances of :class:`IMAP4`: .. versionadded:: 3.5 +.. property:: IMAP4.file + + Internal :class:`~io.BufferedReader` associated with the underlying socket. + This property is documented for legacy purposes but not part of the public + interface. The caller is responsible to ensure that the current file is + closed before changing it. + + .. deprecated-removed:: 3.15 3.19 + + .. _imap4-example: IMAP4 Example @@ -715,3 +972,10 @@ retrieves and prints all messages:: M.close() M.logout() +.. note:: + + A ``FETCH`` response may contain additional or unsolicited data + (see :rfc:`3501`, section 7.4.2), + so production code should inspect the whole response + rather than rely on ``data[0][1]``. + diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index cc426326b29932f..e11db37b9fad501 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -18,11 +18,9 @@ the metadata of an installed `Distribution Package `_\s, modules, if any). Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources`, -this package can eliminate the need to use the older and less efficient -``pkg_resources`` package. +provides the entry point and metadata APIs that were previously +exposed by the now-removed ``pkg_resources`` package. Along with +:mod:`importlib.resources`, it supersedes ``pkg_resources``. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as @@ -107,6 +105,13 @@ You can also get a :ref:`distribution's version number `, list its current Python environment. +.. exception:: MetadataNotFound + + Subclass of :class:`FileNotFoundError` raised when attempting to load metadata + from a distribution folder that is empty or otherwise does not contain a + metadata file. + + Functional API ============== @@ -226,6 +231,9 @@ Distribution metadata Raises :exc:`PackageNotFoundError` if the named distribution package is not installed in the current Python environment. + Raises :exc:`MetadataNotFound` if a distribution package is + present but no METADATA file is present. + .. class:: PackageMetadata A concrete implementation of the @@ -254,6 +262,12 @@ all the metadata in a JSON-compatible form per :PEP:`566`:: The full set of available metadata is not described here. See the PyPA `Core metadata specification `_ for additional details. +.. versionchanged:: 3.15 + Previously and incidentally, if a METADATA file was missing from a distribution, an + empty ``PackageMetadata`` would be returned, indistinguishable from + an empty METADATA file. Now, a missing METADATA file triggers a + ``MetadataNotFound`` exception. + .. versionchanged:: 3.10 The ``Description`` is now included in the metadata when presented through the payload. Line continuation characters have been removed. @@ -467,6 +481,9 @@ The same applies for :func:`entry_points` and :func:`files`. .. attribute:: metadata :type: PackageMetadata + Raises :exc:`MetadataNotFound` if the METADATA file is not present in + the distribution. + There are all kinds of additional metadata available on :class:`!Distribution` instances as a :class:`PackageMetadata` instance:: @@ -717,7 +734,3 @@ packages served by the ``DatabaseImporter``, assuming that the The ``DatabaseDistribution`` may also provide other metadata files, like ``RECORD`` (required for :attr:`!Distribution.files`) or override the implementation of :attr:`!Distribution.files`. See the source for more inspiration. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index 20297f9fe307b54..72db66f9f06f890 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -31,15 +31,13 @@ not** have to exist as physical files and directories on the file system: for example, a package and its resources can be imported from a zip file using :py:mod:`zipimport`. -.. note:: +.. warning:: + + :mod:`importlib.resources` follows the same security model as the built-in + :func:`open` function. Passing untrusted inputs to the functions + in this module is unsafe. - This module provides functionality similar to `pkg_resources - `_ `Basic - Resource Access - `_ - without the performance overhead of that package. This makes reading - resources included in packages easier, with more stable and consistent - semantics. +.. note:: The standalone backport of this module provides more information on `using importlib.resources @@ -242,7 +240,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 785f6c614b43917..0b76020eacc1da2 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -286,7 +286,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other @@ -340,7 +340,7 @@ ABC hierarchy:: This method can potentially yield a very large number of objects, and it may carry out IO operations when computing these values. - Because of this, it will generaly be desirable to compute the result + Because of this, it will generally be desirable to compute the result values on-the-fly, as they are needed. As such, the returned object is only guaranteed to be an :class:`iterable `, instead of a :class:`list` or other diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 8fc77be520d4268..f28c03e2fae092f 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -44,6 +44,7 @@ the `Python Package Index `_. stdtypes.rst exceptions.rst threadsafety.rst + time-complexity.rst text.rst binary.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index ff893a4513926a0..3ca63d1d026135c 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -195,10 +195,6 @@ attributes (see :ref:`import-mod-attrs` for module attributes): | | | read more :ref:`here | | | | `| +-----------------+-------------------+---------------------------+ -| | co_lnotab | encoded mapping of line | -| | | numbers to bytecode | -| | | indices | -+-----------------+-------------------+---------------------------+ | | co_freevars | tuple of names of free | | | | variables (referenced via | | | | a function's closure) | @@ -420,17 +416,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -438,11 +464,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. @@ -597,8 +635,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: ismethoddescriptor(object) Return ``True`` if the object is a method descriptor, but not if - :func:`ismethod`, :func:`isclass`, :func:`isfunction` or :func:`isbuiltin` - are true. + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. This, for example, is true of ``int.__add__``. An object passing this test has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` @@ -606,10 +643,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): attributes varies. A :attr:`~definition.__name__` attribute is usually sensible, and :attr:`~definition.__doc__` often is. - Methods implemented via descriptors that also pass one of the other tests - return ``False`` from the :func:`ismethoddescriptor` test, simply because the - other tests promise more -- you can, e.g., count on having the - :attr:`~method.__func__` attribute (etc) when an object passes + Method descriptors that also pass any of the other tests (:func:`!isclass`, + :func:`!ismethod` or :func:`!isfunction`) make this function return ``False``, + simply because those other tests promise more -- you can, for example, count + on having the :attr:`~method.__func__` attribute when an object passes :func:`ismethod`. .. versionchanged:: 3.13 @@ -620,16 +657,28 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isdatadescriptor(object) - Return ``True`` if the object is a data descriptor. + Return ``True`` if the object is a data descriptor, but not if + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. - Data descriptors have a :attr:`~object.__set__` or a :attr:`~object.__delete__` method. - Examples are properties (defined in Python), getsets, and members. The - latter two are defined in C and there are more specific tests available for - those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes - (properties, getsets, and members have both of these attributes), but this is - not guaranteed. + Data descriptors always have a :meth:`~object.__set__` method and/or + a :meth:`~object.__delete__` method. Optionally, they may also have a + :meth:`~object.__get__` method. + Examples of data descriptors are :func:`properties `, getsets and + member descriptors. Note that for the latter two (defined only in C extension + modules), more specific tests are available: :func:`isgetsetdescriptor` and + :func:`ismemberdescriptor`, respectively. + + While data descriptors may also have :attr:`~definition.__name__` and + :attr:`!__doc__` attributes (as properties, getsets and member descriptors + do), this is not necessarily the case in general. + + .. versionchanged:: 3.8 + This function now reports objects with only a :meth:`~object.__set__` method + as being data descriptors (the presence of :meth:`~object.__get__` is no + longer required for that). Moreover, objects with :meth:`~object.__delete__`, + but not :meth:`~object.__set__`, are now properly recognized as data + descriptors as well, which was not the case previously. .. function:: isgetsetdescriptor(object) @@ -658,9 +707,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Retrieving source code ---------------------- -.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True) +.. function:: getdoc(object, *, inherit_class_doc=True, fallback_to_class_doc=True, dedent=True) - Get the documentation string for an object, cleaned up with :func:`cleandoc`. + Get the documentation string for an object, cleaned up with :func:`cleandoc` + (with the same meaning of *dedent*). If the documentation string for an object is not provided: * if the object is a class and *inherit_class_doc* is true (by default), @@ -681,6 +731,9 @@ Retrieving source code Documentation strings on :class:`~functools.cached_property` objects are now inherited if not overridden. + .. versionchanged:: next + Added the *dedent* parameter. + .. function:: getcomments(object) @@ -694,6 +747,7 @@ Retrieving source code .. function:: getfile(object) Return the name of the (text or binary) file in which an object was defined. + An :exc:`OSError` is raised if the source code cannot be retrieved. This will fail with a :exc:`TypeError` if the object is a built-in module, class, or function. @@ -707,9 +761,10 @@ Retrieving source code .. function:: getsourcefile(object) Return the name of the Python source file in which an object was defined - or ``None`` if no way can be identified to get the source. This - will fail with a :exc:`TypeError` if the object is a built-in module, class, or - function. + or ``None`` if no way can be identified to get the source. An :exc:`OSError` is + raised if the source code cannot be retrieved. + This will fail with a :exc:`TypeError` if the object is a built-in module, + class, or function. .. function:: getsourcelines(object) @@ -742,15 +797,23 @@ Retrieving source code former. -.. function:: cleandoc(doc) +.. function:: cleandoc(doc, *, dedent=True) Clean up indentation from docstrings that are indented to line up with blocks of code. All leading whitespace is removed from the first line. Any leading whitespace - that can be uniformly removed from the second line onwards is removed. Empty - lines at the beginning and end are subsequently removed. Also, all tabs are - expanded to spaces. + that can be uniformly removed from the second line onwards is removed, unless + *dedent* is false. Empty lines at the beginning and end are subsequently + removed. Also, all tabs are expanded to spaces. + + Since Python 3.13 the compiler removes the indentation of docstrings, so + *dedent* only affects documentation strings which are not written as + docstrings in the source code, like those generated by Argument Clinic, + where the indentation is meaningful. + + .. versionchanged:: next + Added the *dedent* parameter. .. _inspect-signature-object: @@ -1193,7 +1256,7 @@ Classes and functions times. -.. function:: getfullargspec(func) +.. function:: getfullargspec(func, *, annotation_format=Format.VALUE) Get the names and default values of a Python function's parameters. A :term:`named tuple` is returned: @@ -1223,6 +1286,14 @@ Classes and functions APIs. This function is retained primarily for use in code that needs to maintain compatibility with the Python 2 ``inspect`` module API. + A member of the + :class:`annotationlib.Format` enum can be passed to the + *annotation_format* parameter to control the format of the returned + annotations. For example, use + ``annotation_format=annotationlib.Format.STRING`` to return annotations in string + format. Note that with the default ``VALUE`` format, creation of some argspecs + may raise an exception. + .. versionchanged:: 3.4 This function is now based on :func:`signature`, but still ignores ``__wrapped__`` attributes and includes the already bound first @@ -1240,6 +1311,9 @@ Classes and functions order of keyword-only parameters as of version 3.7, although in practice this order had always been preserved in Python 3. + .. versionchanged:: 3.15 + The *annotation_format* parameter was added. + .. function:: getargvalues(frame) @@ -1565,10 +1639,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` @@ -1837,8 +1912,15 @@ from the command line. By default, accepts the name of a module and prints the source of that module. A class or function within the module can be printed instead by -appended a colon and the qualified name of the target object. +appending a colon and the qualified name of the target object. .. option:: --details Print information about the specified object rather than the source code + +.. versionchanged:: 3.15 + + The ``--details`` option now supports basic introspection for modules + without available source code and indicates when modules are frozen. + It also indicates when the given target reference is not the canonical + name of the referenced object. diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 494e57fe1c04743..ecaa053b4e18b9e 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -38,6 +38,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -65,6 +66,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -103,6 +105,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -478,8 +487,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -493,13 +505,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -513,6 +531,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -641,7 +666,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -661,10 +690,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -676,6 +701,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -731,14 +763,28 @@ than raw I/O does. .. note:: As long as the view exists, the :class:`BytesIO` object cannot be - resized or closed. + resized. Closing it does not invalidate the view. .. versionadded:: 3.2 + .. versionchanged:: next + The :class:`BytesIO` object can now be closed while the view exists. + .. method:: getvalue() Return :class:`bytes` containing the entire contents of the buffer. + .. method:: peek(size=0, /) + + Return a copy of the buffer from the current position onwards without + advancing the position. + + If *size* is less than one or omitted, at most + :data:`DEFAULT_BUFFER_SIZE` bytes are returned. + Otherwise, at most *size* bytes are returned. + Return an empty :class:`bytes` object at EOF. + + .. versionadded:: next .. method:: read1(size=-1, /) @@ -824,9 +870,9 @@ than raw I/O does. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index ba20310d63b2775..387d8034bb8c84a 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -367,9 +367,9 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: ipv4_mapped - For addresses that appear to be IPv4 mapped addresses (starting with - ``::FFFF/96``), this property will report the embedded IPv4 address. - For any other address, this property will be ``None``. + For addresses that appear to be IPv4 mapped addresses in the range + ``::FFFF:0:0/96`` as defined by :RFC:`4291`, this property reports the + embedded IPv4 address. For any other address, this property will be ``None``. .. attribute:: scope_id @@ -718,6 +718,35 @@ dictionaries. .. deprecated:: 3.7 It uses the same ordering and comparison algorithm as "<", "==", and ">" + .. method:: next_network(next_prefix=None) + + Finds the next closest network of prefix size *next_prefix*. If + *next_prefix=None*, then the current network prefix will be used. + Returns a single network object. + + Raises :exc:`ValueError` if *next_prefix* is out of range or no further + network of the requested size exists. + + >>> IPv4Network('192.0.2.0/24').next_network() + IPv4Network('192.0.3.0/24') + >>> IPv4Network('192.0.2.0/24').next_network(next_prefix=25) + IPv4Network('192.0.3.0/25') + >>> IPv4Network('192.0.2.0/24').next_network(next_prefix=23) + IPv4Network('192.0.4.0/23') + >>> IPv4Network('192.0.80.0/22').next_network(next_prefix=18) + IPv4Network('192.0.128.0/18') + >>> IPv4Network('192.0.80.0/22').next_network(next_prefix=50) + Traceback (most recent call last): + File "", line 1, in + raise ValueError( + ValueError: next prefix must be between 1 and 32 + >>> IPv4Network('255.255.255.0/24').next_network() + Traceback (most recent call last): + File "", line 1, in + raise ValueError( + ValueError: out of address space, cannot make another /24 network + + .. versionadded:: 3.16 .. class:: IPv6Network(address, strict=True) @@ -790,6 +819,7 @@ dictionaries. .. method:: supernet(prefixlen_diff=1, new_prefix=None) .. method:: subnet_of(other) .. method:: supernet_of(other) + .. method:: next_network(next_prefix=None) .. method:: compare_networks(other) Refer to the corresponding attribute documentation in diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 5a0ac60ab7d2ec9..e1730608887b3dd 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -197,6 +197,10 @@ loops that truncate the stream. for iterable in iterables: yield from iterable + Note that :ref:`unpacking in comprehensions ` + provides similar functionality so that ``list(chain(p, q))`` could be + written as ``[*s for s in (p, q)]``. + .. classmethod:: chain.from_iterable(iterable) @@ -208,6 +212,10 @@ loops that truncate the stream. for iterable in iterables: yield from iterable + Note that :ref:`unpacking in comprehensions ` + provides similar functionality so that ``list(chain.from_iterable(iterables))`` + could be written as ``[*s for s in iterables]``. + .. function:: combinations(iterable, r) @@ -833,6 +841,7 @@ and :term:`generators ` which incur interpreter overhead. from collections import Counter, deque from contextlib import suppress from functools import reduce + from heapq import heappush, heappushpop, heappush_max, heappushpop_max from math import comb, isqrt, prod, sumprod from operator import getitem, is_not, itemgetter, mul, neg, truediv @@ -848,11 +857,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def running_mean(iterable): - "Yield the average of all values seen so far." - # running_mean([8.5, 9.5, 7.5, 6.5]) → 8.5 9.0 8.5 8.0 - return map(truediv, accumulate(iterable), count(1)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -1150,6 +1154,49 @@ and :term:`generators ` which incur interpreter overhead. return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1226,10 +1273,6 @@ and :term:`generators ` which incur interpreter overhead. [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) - [8.5, 9.0, 8.5, 8.0] - - >>> for _ in loops(5): ... print('hi') ... @@ -1789,6 +1832,28 @@ and :term:`generators ` which incur interpreter overhead. True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b354e7ba534835f..5e8c452a5ab9a1f 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -211,7 +211,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: @@ -486,7 +486,7 @@ Encoders and Decoders +----------------------------------------+---------------+ | Python | JSON | +========================================+===============+ - | dict | object | + | dict, frozendict | object | +----------------------------------------+---------------+ | list, tuple | array | +----------------------------------------+---------------+ @@ -504,6 +504,9 @@ Encoders and Decoders .. versionchanged:: 3.4 Added support for int- and float-derived Enum classes. + .. versionchanged:: 3.15 + Added support for :class:`frozendict`. + To extend this to recognize other objects, subclass and implement a :meth:`~JSONEncoder.default` method with another method that returns a serializable object for ``o`` if possible, otherwise it should call the superclass implementation diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index e02cbe7d669f8b4..933ad039b6505f9 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -342,6 +342,10 @@ The :mod:`!locale` module defines the following exception and functions: .. versionchanged:: 3.14 The function now temporarily sets the ``LC_CTYPE`` locale in some cases. + .. versionchanged:: next + On glibc, the ``LC_TIME`` items (except ``ERA``) are now decoded + independently of the ``LC_CTYPE`` encoding. + .. function:: getdefaultlocale([envvars]) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 30bf7860a751192..3842c1e9d477d29 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -124,12 +124,12 @@ in :mod:`logging` itself) and defining handlers which are declared either in application (e.g. based on command-line parameters or other aspects of the runtime environment) before being passed to ``fileConfig``. - .. versionchanged:: 3.10 - Added the *encoding* parameter. + .. versionchanged:: 3.10 + Added the *encoding* parameter. - .. versionchanged:: 3.12 - An exception will be thrown if the provided file - doesn't exist or is invalid or empty. + .. versionchanged:: 3.12 + An exception will be thrown if the provided file + doesn't exist or is invalid or empty. .. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 714db5fa12af0aa..7230224b4fc5323 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -402,7 +402,8 @@ timed intervals. rollover interval. When computing the next rollover time for the first time (when the handler - is created), the last modification time of an existing log file, or else + is created), the creation time (if supported by the OS and file system) + or the last modification of an existing log file, or else the current time, is used to compute when the next rotation will occur. If the *utc* argument is true, times in UTC will be used; otherwise @@ -449,6 +450,10 @@ timed intervals. .. versionchanged:: 3.9 The *errors* parameter was added. + .. versionchanged:: next + Use the creation time instead of the last modification time, if supported by the OS and file system. + + .. method:: doRollover() Does a rollover, as described above. @@ -626,7 +631,8 @@ supports sending logging messages to a remote or local Unix syslog. the form of a ``(host, port)`` tuple. If *address* is not specified, ``('localhost', 514)`` is used. The address is used to open a socket. An alternative to providing a ``(host, port)`` tuple is providing an address as a - string, for example '/dev/log'. In this case, a Unix domain socket is used to + string or a :class:`bytes` object, for example '/dev/log'. + In this case, a Unix domain socket is used to send the message to the syslog. If *facility* is not specified, :const:`LOG_USER` is used. The type of socket opened depends on the *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus @@ -659,6 +665,9 @@ supports sending logging messages to a remote or local Unix syslog. .. versionchanged:: 3.14 *timeout* was added. + .. versionchanged:: next + *address* can now be a :class:`bytes` object. + .. method:: close() Closes the socket to the remote host. diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index aba530844d7177a..a3d117c10702414 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -690,7 +690,7 @@ Formatter Objects :param defaults: A dictionary with default values to use in custom fields. For example, ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - :type defaults: dict[str, Any] + :type defaults: dict[str, typing.Any] .. versionchanged:: 3.2 Added the *style* parameter. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 6cede00b2186789..580829cfbb46e07 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -152,35 +152,14 @@ Compressing and decompressing data in memory :func:`compress`. The *format* argument specifies what container format should be used. - Possible values are: - - * :const:`FORMAT_XZ`: The ``.xz`` container format. - This is the default format. - - * :const:`FORMAT_ALONE`: The legacy ``.lzma`` container format. - This format is more limited than ``.xz`` -- it does not support integrity - checks or multiple filters. - - * :const:`FORMAT_RAW`: A raw data stream, not using any container format. - This format specifier does not support integrity checks, and requires that - you always specify a custom filter chain (for both compression and - decompression). Additionally, data compressed in this manner cannot be - decompressed using :const:`FORMAT_AUTO` (see :class:`LZMADecompressor`). + Possible values are :const:`FORMAT_XZ` (the default), + :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. The *check* argument specifies the type of integrity check to include in the compressed data. This check is used when decompressing, to ensure that the - data has not been corrupted. Possible values are: - - * :const:`CHECK_NONE`: No integrity check. - This is the default (and the only acceptable value) for - :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. - - * :const:`CHECK_CRC32`: 32-bit Cyclic Redundancy Check. - - * :const:`CHECK_CRC64`: 64-bit Cyclic Redundancy Check. - This is the default for :const:`FORMAT_XZ`. - - * :const:`CHECK_SHA256`: 256-bit Secure Hash Algorithm. + data has not been corrupted. Possible values are :const:`CHECK_NONE`, + :const:`CHECK_CRC32`, :const:`CHECK_CRC64` (the default for + :const:`FORMAT_XZ`) and :const:`CHECK_SHA256`. If the specified check is not supported, an :class:`LZMAError` is raised. @@ -356,12 +335,26 @@ options. Valid filter IDs are as follows: * Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` + * :const:`!FILTER_X86` + * :const:`!FILTER_IA64` + * :const:`!FILTER_ARM` + * :const:`!FILTER_ARMTHUMB` + * :const:`!FILTER_POWERPC` + * :const:`!FILTER_SPARC` + + The above work on all lzma runtime library versions. + + * :const:`!FILTER_ARM64` + + Only works if the lzma version is 5.4.0 or later. + + .. versionadded:: next + + * :const:`!FILTER_RISCV` + + Only works if the lzma version is 5.6.0 or later. + + .. versionadded:: next A filter chain can consist of up to 4 filters, and cannot be empty. The last filter in the chain must be a compression filter, and any other filters must be @@ -398,6 +391,106 @@ These filters support one option, ``start_offset``. This specifies the address that should be mapped to the beginning of the input data. The default is 0. +Constants +--------- + +The following module-level constants are provided for use as the *format*, +*check*, *preset* and *filters* arguments of the classes and functions above. + +Container formats: + +.. data:: FORMAT_XZ + + The ``.xz`` container format. + +.. data:: FORMAT_ALONE + + The legacy ``.lzma`` container format. This format is more limited than + ``.xz`` -- it does not support integrity checks or multiple filters. + +.. data:: FORMAT_RAW + + A raw data stream, not using any container format. This format specifier + does not support integrity checks, and requires that you always specify a + custom filter chain (for both compression and decompression). Additionally, + data compressed in this manner cannot be decompressed using + :const:`FORMAT_AUTO`. + +.. data:: FORMAT_AUTO + + Used for decompression only. The container format is detected + automatically, so that both ``.xz`` and ``.lzma`` files can be decompressed. + +Integrity checks: + +.. data:: CHECK_NONE + + No integrity check. This is the default (and the only acceptable value) for + :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. + +.. data:: CHECK_CRC32 + + A 32-bit Cyclic Redundancy Check. + +.. data:: CHECK_CRC64 + + A 64-bit Cyclic Redundancy Check. This is the default for + :const:`FORMAT_XZ`. + +.. data:: CHECK_SHA256 + + A 256-bit Secure Hash Algorithm. + +.. data:: CHECK_UNKNOWN + + The integrity check used by a stream could not yet be determined. This may + be the value of the :attr:`LZMADecompressor.check` attribute until enough of + the input has been decoded. + +.. data:: CHECK_ID_MAX + + The largest supported integrity-check ID. + +Compression presets: + +.. data:: PRESET_DEFAULT + + The default compression preset, equivalent to preset level ``6``. + +.. data:: PRESET_EXTREME + + A flag that may be bitwise OR-ed with a preset level (``0`` to ``9``) to + select a slower but more thorough variant of that preset. + +Filter IDs and options: + +.. data:: FILTER_LZMA1 + FILTER_LZMA2 + + The LZMA1 and LZMA2 compression filters. :const:`FILTER_LZMA1` is for use + with :const:`FORMAT_ALONE`, while :const:`FILTER_LZMA2` is for use with + :const:`FORMAT_XZ` and :const:`FORMAT_RAW`. + +.. data:: FILTER_DELTA + + The delta filter. + +.. data:: MODE_FAST + MODE_NORMAL + + Compression modes that may be used as the ``mode`` option of a filter + specifier (see :ref:`filter-chain-specs`). + +.. data:: MF_HC3 + MF_HC4 + MF_BT2 + MF_BT3 + MF_BT4 + + Match finders that may be used as the ``mf`` option of a filter specifier + (see :ref:`filter-chain-specs`). + + Examples -------- diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 4a11aec15dfb737..efe411e5a43f27d 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -83,13 +83,20 @@ noted otherwise, all return values are floats. **Trigonometric functions** -------------------------------------------------------------------------------------------------- -:func:`acos(x) ` Arc cosine of *x* -:func:`asin(x) ` Arc sine of *x* -:func:`atan(x) ` Arc tangent of *x* -:func:`atan2(y, x) ` ``atan(y / x)`` -:func:`cos(x) ` Cosine of *x* -:func:`sin(x) ` Sine of *x* -:func:`tan(x) ` Tangent of *x* +:func:`acos(x) ` Arc cosine of *x*, in radians +:func:`acospi(x) ` Arc cosine of *x*, in half-turns +:func:`asin(x) ` Arc sine of *x*, in radians +:func:`asinpi(x) ` Arc sine of *x*, in half-turns +:func:`atan(x) ` Arc tangent of *x*, in radians +:func:`atanpi(x) ` Arc tangent of *x*, in half-turns +:func:`atan2(y, x) ` ``atan(y / x)``, in radians +:func:`atan2pi(y, x) ` ``atan(y / x)``, in half-turns +:func:`cos(x) ` Cosine of *x* radians +:func:`cospi(x) ` Cosine of *x⋅π* radians +:func:`sin(x) ` Sine of *x* radians +:func:`sinpi(x) ` Sine of *x⋅π* radians +:func:`tan(x) ` Tangent of *x* radians +:func:`tanpi(x) ` Tangent of *x⋅π* radians **Hyperbolic functions** -------------------------------------------------------------------------------------------------- @@ -255,10 +262,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of @@ -597,18 +606,42 @@ Trigonometric functions ``pi``. +.. function:: acospi(x) + + Return the arc cosine of *x*, in half-turns. The result is between ``0`` and + ``1``. + + .. versionadded:: next + + .. function:: asin(x) Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: asinpi(x) + + Return the arc sine of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan(x) Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and ``pi/2``. +.. function:: atanpi(x) + + Return the arc tangent of *x*, in half-turns. The result is between ``-0.5`` and + ``0.5``. + + .. versionadded:: next + + .. function:: atan2(y, x) Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. @@ -619,21 +652,54 @@ Trigonometric functions -1)`` is ``-3*pi/4``. +.. function:: atan2pi(y, x) + + Return ``atanpi(y / x)``, in half-turns. The result is between ``-1`` and ``1``. + The vector in the plane from the origin to point ``(x, y)`` makes this angle + with the positive X axis. The point of :func:`atan2pi` is that the signs of both + inputs are known to it, so it can compute the correct quadrant for the angle. + For example, ``atanpi(1)`` and ``atan2pi(1, 1)`` are both ``0.25``, but + ``atan2pi(-1, -1)`` is ``-0.75``. + + .. versionadded:: next + + .. function:: cos(x) Return the cosine of *x* radians. +.. function:: cospi(x) + + Return the cosine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: sin(x) Return the sine of *x* radians. +.. function:: sinpi(x) + + Return the sine of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + .. function:: tan(x) Return the tangent of *x* radians. +.. function:: tanpi(x) + + Return the tangent of *x* half-turns (*x⋅π* radians). + + .. versionadded:: next + + Hyperbolic functions -------------------- @@ -781,9 +847,8 @@ the following functions from the :mod:`math.integer` module: Floats with integral values (like ``5.0``) are no longer accepted in the :func:`factorial` function. -.. deprecated:: 3.15 - These aliases are :term:`soft deprecated` in favor of the - :mod:`math.integer` functions. +.. soft-deprecated:: 3.15 + Use the :mod:`math.integer` functions instead of these aliases. Constants diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 1e599bde8bcddd2..bd2407949db90dc 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -39,8 +39,8 @@ the information :func:`init` sets up. (e.g. :program:`compress` or :program:`gzip`). The encoding is suitable for use as a :mailheader:`Content-Encoding` header, **not** as a :mailheader:`Content-Transfer-Encoding` header. The mappings are table driven. - Encoding suffixes are case sensitive; type suffixes are first tried case - sensitively, then case insensitively. + Encoding suffixes are case-sensitive. Suffix mappings and type suffixes are + first tried case-sensitively, then case-insensitively. The optional *strict* argument is a flag specifying whether the list of known MIME types is limited to only the official types `registered with IANA @@ -54,8 +54,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. @@ -116,12 +116,13 @@ behavior of the module. Previously, Windows registry settings were ignored. -.. function:: read_mime_types(filename) +.. function:: read_mime_types(file) - Load the type map given in the file *filename*, if it exists. The type map is - returned as a dictionary mapping filename extensions, including the leading dot - (``'.'``), to strings of the form ``'type/subtype'``. If the file *filename* - does not exist or cannot be read, ``None`` is returned. + Load the type map given in the file named by *file*, if it exists. *file* + must be a string specifying the name of the file to read. The type map is + returned as a dictionary mapping file extensions, including the leading dot + (``'.'``), to strings of the form ``'type/subtype'``. If the file does not + exist or cannot be read, ``None`` is returned. .. function:: add_type(type, ext, strict=True) @@ -129,10 +130,20 @@ behavior of the module. Add a mapping from the MIME type *type* to the extension *ext*. When the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Valid extensions are empty or start with a ``'.'``. + + Registered lower-case extensions are matched case-insensitively. When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. + + .. data:: inited @@ -200,8 +211,8 @@ than one MIME-type database; it provides an interface similar to the one of the .. class:: MimeTypes(filenames=(), strict=True) This class represents a MIME-types database. By default, it provides access to - the same database as the rest of this module. The initial database is a copy of - that provided by the module, and may be extended by loading additional + the same database as the rest of this module. The initial database is created from + Python's built-in MIME type tables. It may be extended by loading additional :file:`mime.types`\ -style files into the database using the :meth:`read` or :meth:`readfp` methods. The mapping dictionaries may also be cleared before loading additional data if the default data is not desired. @@ -215,30 +226,32 @@ than one MIME-type database; it provides an interface similar to the one of the Dictionary mapping suffixes to suffixes. This is used to allow recognition of encoded files for which the encoding and the type are indicated by the same extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. + to allow the encoding and type to be recognized separately. + This is initialized with some predefined values. .. attribute:: MimeTypes.encodings_map - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. + Dictionary mapping filename extensions to encoding types. + This is initialized with some predefined values. .. attribute:: MimeTypes.types_map Tuple containing two dictionaries, mapping filename extensions to MIME types: the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. + the standard types. + They are initialized with some predefined values and MIME type + information loaded from files specified by the *filenames* argument. .. attribute:: MimeTypes.types_map_inv Tuple containing two dictionaries, mapping MIME types to a list of filename extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. + second one is for the standard types. + They are initialized with some predefined values and MIME type + information loaded from files specified by the *filenames* argument. .. method:: MimeTypes.guess_extension(type, strict=True) @@ -304,12 +317,16 @@ than one MIME-type database; it provides an interface similar to the one of the extension is already known, the new type will replace the old one. When the type is already known the extension will be added to the list of known extensions. + Registered lower-case extensions are matched case-insensitively. + When *strict* is ``True`` (the default), the mapping will be added to the official MIME types, otherwise to the non-standard ones. - .. deprecated-removed:: 3.14 3.16 - Invalid, undotted extensions will raise a - :exc:`ValueError` in Python 3.16. + .. deprecated:: 3.14 + *ext* values that do not start with ``'.'`` are deprecated. + + .. versionchanged:: next + *ext* now must start with ``'.'``. Otherwise :exc:`ValueError` is raised. .. _mimetypes-cli: @@ -348,7 +365,7 @@ it converts file extensions to MIME types. For each ``type`` entry, the script writes a line into the standard output stream. If an unknown type occurs, it writes an error message into the -standard error stream and exits with the return code ``1``. +standard output stream and exits with the return code ``1``. .. mimetypes-cli-example: @@ -375,7 +392,7 @@ interface: $ # get a MIME type for a rare file extension $ python -m mimetypes filename.pict - error: unknown extension of filename.pict + error: media type unknown for filename.pict $ # now look in the extended database built into Python $ python -m mimetypes --lenient filename.pict @@ -397,7 +414,8 @@ interface: $ python -m mimetypes filename.sh filename.nc filename.xxx filename.txt type: application/x-sh encoding: None type: application/x-netcdf encoding: None - error: unknown extension of filename.xxx + error: media type unknown for filename.xxx + type: text/plain encoding: None $ # try to feed an unknown MIME type $ python -m mimetypes --extension audio/aac audio/opus audio/future audio/x-wav diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 6b49c1a9ccd6e11..f4a1125554a0513 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -135,13 +135,23 @@ Console I/O .. function:: putch(char) - Print the byte string *char* to the console without buffering. + Print the byte string *char* to the console without buffering. Raises + :exc:`OSError` on failure, for example when the process has no console + attached. + + .. versionchanged:: next + Failures are now reported by raising :exc:`OSError` instead of being + silently ignored. .. function:: putwch(unicode_char) Wide char variant of :func:`putch`, accepting a Unicode value. + .. versionchanged:: next + Failures are now reported by raising :exc:`OSError` instead of being + silently ignored. + .. function:: ungetch(char) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 2b67d10d7bf1b7b..bedea46cb16d60d 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -100,10 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. -The arguments to :class:`Process` usually need to be unpickleable from within -the child process. If you tried typing the above example directly into a REPL it -could lead to an :exc:`AttributeError` in the child process trying to locate the -*f* function in the ``__main__`` module. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -932,7 +932,8 @@ For an example of the usage of queues for interprocess communication see standard library's :mod:`queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. + :meth:`~queue.Queue.task_done`, :meth:`~queue.Queue.join`, and + :meth:`~queue.Queue.shutdown`. .. method:: qsize() @@ -1335,12 +1336,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1360,18 +1361,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -1722,7 +1723,10 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. The :mod:`!multiprocessing.sharedctypes` module @@ -2532,7 +2536,7 @@ with the :class:`Pool` class. Callbacks should complete immediately since otherwise the thread which handles the results will get blocked. - .. method:: imap(func, iterable[, chunksize]) + .. method:: imap(func, iterable, chunksize=1, *, buffersize=None) A lazier version of :meth:`.map`. @@ -2546,12 +2550,27 @@ with the :class:`Pool` class. ``next(timeout)`` will raise :exc:`multiprocessing.TimeoutError` if the result cannot be returned within *timeout* seconds. - .. method:: imap_unordered(func, iterable[, chunksize]) + The *iterable* is collected immediately rather than lazily, unless a + *buffersize* is specified to limit the number of submitted tasks whose + results have not yet been yielded. If the buffer is full, iteration over + the *iterables* pauses until a result is yielded from the buffer. + To fully utilize pool's capacity when using this feature, + set *buffersize* at least to the number of processes in pool + (to consume *iterable* as you go), or even higher + (to prefetch the next ``N=buffersize-processes`` arguments). + + .. versionchanged:: next + Added the *buffersize* parameter. + + .. method:: imap_unordered(func, iterable, chunksize=1, *, buffersize=None) The same as :meth:`imap` except that the ordering of the results from the returned iterator should be considered arbitrary. (Only when there is only one worker process is the order guaranteed to be "correct".) + .. versionchanged:: next + Added the *buffersize* parameter. + .. method:: starmap(func, iterable[, chunksize]) Like :meth:`~multiprocessing.pool.Pool.map` except that the @@ -2916,6 +2935,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 57b35017072c97e..0ed24b4e6c5225d 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -90,20 +90,7 @@ Notes for type implementers Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different -extensions of the real numbers. For example, :class:`fractions.Fraction` -implements :func:`hash` as follows:: - - def __hash__(self): - if self.denominator == 1: - # Get integers right. - return hash(self.numerator) - # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) - else: - # Use tuple's hash to avoid a high collision rate on - # simple fractions. - return hash((self.numerator, self.denominator)) +extensions of the real numbers. See also :ref:`numeric-hash`. Adding More Numeric ABCs diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index c0dab83977e427f..3d1c8cda13be381 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -110,7 +110,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -134,13 +134,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -152,7 +152,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -172,25 +172,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -209,7 +209,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -403,13 +403,18 @@ Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 905212965bd70fe..09416f12ad45c2f 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1073,6 +1073,12 @@ As you can see, most actions involve storing or updating a value somewhere. and can be overridden by a custom subclass passed to the *values* argument of :meth:`OptionParser.parse_args` (as described in :ref:`optparse-parsing-arguments`). + :class:`!Values` objects support :func:`copy.replace`, + which returns a copy of the object with the specified attributes replaced. + + .. versionchanged:: next + Added support for :func:`copy.replace`. + Option arguments (and various other values) are stored as attributes of this object, according to the :attr:`~Option.dest` (destination) option attribute. diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 808187061733be0..f5ddb426c915f5a 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -59,6 +59,18 @@ the :mod:`glob` module.) Return a normalized absolutized version of the pathname *path*. On most platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``. + On Windows the path is normalized by the operating system, + therefore the result can differ from ``normpath(join(os.getcwd(), path))``. + A drive-relative path is resolved against the current directory + of the specified drive, and the drive letter is capitalized. + Trailing dots and spaces are stripped. + For example:: + + >>> os.path.abspath('c:spam') + 'C:\\Temp\\spam' + >>> os.path.abspath('c:/temp/spam. . .') + 'c:\\temp\\spam' + .. seealso:: :func:`os.path.join` and :func:`os.path.normpath`. .. versionchanged:: 3.6 @@ -435,6 +447,9 @@ the :mod:`glob` module.) links encountered in the path (if they are supported by the operating system). On Windows, this function will also resolve MS-DOS (also called 8.3) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. + The returned path uses the case reported by the operating system, + which can differ from the case of *path*, + in particular the drive letter is capitalized. By default, the path is evaluated up to the first component that does not exist, is a symlink loop, or whose evaluation raises :exc:`OSError`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7547967c6b32f0e..596597b1ab223ee 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -219,6 +219,14 @@ process and user. :data:`os.environ`, and when one of the :meth:`~dict.pop` or :meth:`~dict.clear` methods is called. + If the :manpage:`clearenv(3)` function is available, the :meth:`~dict.clear` method + uses it and emits a single ``os._clearenv`` audit event. Otherwise, it emits + an ``os.unsetenv`` event on each deleted variable. + + .. audit-event:: os.unsetenv key os.unsetenv + + .. audit-event:: os._clearenv "" os._clearenv + .. seealso:: The :func:`os.reload_environ` function. @@ -226,6 +234,10 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.15 + The :meth:`~dict.clear` method can now emit an ``os._clearenv`` audit + event. + .. data:: environb @@ -440,7 +452,7 @@ process and user. process. For most purposes, it is more useful to use :func:`getpass.getuser` since the latter checks the environment variables :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user is, and - falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the + falls back to ``pwd.getpwuid(os.getuid()).pw_name`` to get the login name of the current real user id. .. availability:: Unix, Windows, not WASI. @@ -529,7 +541,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -539,7 +551,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -713,7 +725,7 @@ process and user. Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -722,7 +734,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -788,29 +800,19 @@ process and user. single: gethostbyaddr() (in module socket) Returns information identifying the current operating system. - The return value is an object with five attributes: + The return value is a :class:`uname_result`. - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. - - On macOS, iOS and Android, this returns the *kernel* name and version (i.e., + On macOS, iOS and Android, this returns the *kernel* name and release (i.e., ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` - can be used to get the user-facing operating system name and version on iOS and + can be used to get the user-facing operating system name and release on iOS and Android. + .. seealso:: + :data:`sys.platform` which has finer granularity. + + The :mod:`platform` module provides detailed checks for the + system's identity. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -818,6 +820,41 @@ process and user. with named attributes. +.. class:: uname_result + + Name and information about the system returned by :func:`os.uname`. + These attributes correspond to the members described in :manpage:`uname(2)`. + + For backwards compatibility, this object is also iterable, behaving + like a five-tuple containing :attr:`~uname_result.sysname`, + :attr:`~uname_result.nodename`, :attr:`~uname_result.release`, + :attr:`~uname_result.version`, and :attr:`~uname_result.machine` + in that order. + + .. attribute:: sysname + + Operating system name. + + .. attribute:: nodename + + Name of machine on network. Some systems truncate + :attr:`~uname_result.nodename` to 8 characters or to the leading + component; a better way to get the hostname is :func:`socket.gethostname` + or even ``socket.gethostbyaddr(socket.gethostname())``. + + .. attribute:: release + + Operating system release. + + .. attribute:: version + + Operating system version. + + .. attribute:: machine + + Hardware identifier. + + .. function:: unsetenv(key, /) .. index:: single: environment variables; deleting @@ -974,10 +1011,14 @@ as internal buffering of data. It will always copy no bytes and return 0 as if the file was empty because of a known Linux kernel issue. - .. availability:: Linux >= 4.5 with glibc >= 2.27. + .. availability:: Linux >= 4.5. .. versionadded:: 3.8 + .. versionchanged:: 3.16 + The function is now also available when Python is built against a libc + that lacks ``copy_file_range()``, such as glibc older than 2.27. + .. function:: device_encoding(fd) @@ -1059,10 +1100,7 @@ as internal buffering of data. Force write of file with filedescriptor *fd* to disk. Does not force update of metadata. - .. availability:: Unix. - - .. note:: - This function is not available on MacOS. + .. availability:: Unix, not macOS, not iOS. .. function:: fpathconf(fd, name, /) @@ -1100,8 +1138,8 @@ as internal buffering of data. .. function:: fstatvfs(fd, /) Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. + file descriptor *fd* in a :class:`statvfs_result`, like :func:`statvfs`. + As of Python 3.3, this is equivalent to ``os.statvfs(fd)``. .. availability:: Unix. @@ -1414,7 +1452,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not WASI. + .. availability:: Unix, macOS >= 27.0, not WASI, not iOS. .. versionadded:: 3.3 @@ -1424,7 +1462,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1439,7 +1477,7 @@ or `the MSDN `_ on Windo :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1608,6 +1646,7 @@ or `the MSDN `_ on Windo - :data:`RWF_APPEND` - :data:`RWF_DONTCACHE` - :data:`RWF_ATOMIC` + - :data:`RWF_NOSIGNAL` Return the total number of bytes actually written. @@ -1657,6 +1696,16 @@ or `the MSDN `_ on Windo .. versionadded:: 3.10 +.. data:: RWF_NOSIGNAL + + Prevent pipe and socket writes from raising :const:`~signal.SIGPIPE`. + This flag is meaningful only for :func:`os.pwritev`. + + .. availability:: Linux >= 6.18. + + .. versionadded:: 3.16 + + .. function:: read(fd, n, /) Read at most *n* bytes from file descriptor *fd*. @@ -2146,6 +2195,11 @@ features: .. audit-event:: os.chdir path os.chdir + .. seealso:: + + The :func:`contextlib.chdir` context manager, which changes the current + working directory on entering and restores the previous one on exit. + .. versionchanged:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -2549,7 +2603,8 @@ features: Windows now handles a *mode* of ``0o700``. -.. function:: makedirs(name, mode=0o777, exist_ok=False) +.. function:: makedirs(name, mode=0o777, exist_ok=False, *, \ + parent_mode=None) .. index:: single: directory; creating @@ -2567,6 +2622,12 @@ features: If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is raised if the target directory already exists. + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories. Like *mode*, it is + combined with the process's umask value; see :ref:`the mkdir() + description `. Otherwise, intermediate directories are + created with the default mode, which is also subject to the umask. + .. note:: :func:`makedirs` will become confused if the path elements to create @@ -2593,6 +2654,11 @@ features: The *mode* argument no longer affects the file permission bits of newly created intermediate-level directories. + .. versionadded:: 3.15 + The *parent_mode* parameter. To match the behavior from Python 3.6 and + earlier (where *mode* was applied to all created directories), pass + ``parent_mode=mode``. + .. function:: mkfifo(path, mode=0o666, *, dir_fd=None) @@ -2894,6 +2960,11 @@ features: .. audit-event:: os.scandir path os.scandir + Sharing a :func:`scandir` iterator between threads will not corrupt the + iterator, but it is subject to :term:`race conditions `: + which entries each thread receives is unspecified, and closing the iterator + while another thread is iterating ends that iteration early. + The :func:`scandir` iterator supports the :term:`context manager` protocol and has the following method: @@ -2970,6 +3041,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name @@ -2983,10 +3057,16 @@ features: .. attribute:: path - The entry's full path name: equivalent to ``os.path.join(scandir_path, - entry.name)`` where *scandir_path* is the :func:`scandir` *path* - argument. The path is only absolute if the :func:`scandir` *path* - argument was absolute. If the :func:`scandir` *path* + The entry's path name: equivalent to ``os.path.join(scandir_path, + entry.name)`` where *scandir_path* is the original :func:`scandir` + *path* argument. Apart from the filename, the path preserves the + original :func:`scandir` argument. If the :func:`scandir` *path* + argument was relative, the :attr:`path` attribute is also relative. + Changing the current working directory after creating the + :func:`scandir` iterator may cause later uses of :attr:`path` to resolve + differently. On some platforms, the constructed path may not be valid + if the original :func:`scandir` argument was usable for enumeration but + not for joining with the entry name. If the :func:`scandir` *path* argument was a :ref:`file descriptor `, the :attr:`path` attribute is the same as the :attr:`name` attribute. @@ -3757,48 +3837,174 @@ features: .. function:: statvfs(path) - Perform a :c:func:`!statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:struct:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). + Perform a :manpage:`statvfs(3)` system call on the given path. The return value + is a :class:`statvfs_result` whose attributes describe the filesystem + on the given path and correspond to the members of the :c:struct:`statvfs` + structure. This function can support :ref:`specifying a file descriptor `. .. availability:: Unix. - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.7 - Added the :attr:`f_fsid` attribute. + +.. class:: statvfs_result + + Filesystem statistics returned by :func:`os.statvfs` and :func:`os.fstatvfs`. + See :manpage:`statvfs(3)` for more details. + + .. attribute:: f_bsize + + Block size. + + .. attribute:: f_frsize + + Fragment size. + + .. attribute:: f_blocks + + Number of :attr:`~statvfs_result.f_frsize` sized blocks the filesystem + can contain. + + .. attribute:: f_bfree + + Number of free blocks. + + .. attribute:: f_bavail + + Number of free blocks for unprivileged users. + + .. attribute:: f_files + + Number of file entries, inodes, the filesystem can contain. + + .. attribute:: f_ffree + + Number of free files entries. + + .. attribute:: f_favail + + Number of free file entries for unprivileged users. + + .. attribute:: f_flag + + Bit-mask of mount flags. The following flags are defined: + :data:`ST_RDONLY`, :data:`ST_NOSUID`, :data:`ST_NODEV`, + :data:`ST_NOEXEC`, :data:`ST_SYNCHRONOUS`, :data:`ST_MANDLOCK`, + :data:`ST_WRITE`, :data:`ST_APPEND`, :data:`ST_IMMUTABLE`, + :data:`ST_NOATIME`, :data:`ST_NODIRATIME`, and :data:`ST_RELATIME`. + + .. attribute:: f_namemax + + Filesystem max filename length. OS specific limitations such as + :ref:`Windows MAX_PATH ` and those described in Linux + :manpage:`pathname(7)` may exist. + + .. attribute:: f_fsid + + Filesystem ID. + + .. versionadded:: 3.7 + + +The following flags are used in :attr:`statvfs_result.f_flag`. + +.. data:: ST_RDONLY + + Read-only filesystem. + + .. versionadded:: 3.2 + +.. data:: ST_NOSUID + + Setuid/setgid bits are disabled or not supported. + + .. versionadded:: 3.2 + +.. data:: ST_NODEV + + Disallow access to device special files. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOEXEC + + Disallow program execution. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_SYNCHRONOUS + + Writes are synced at once. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_MANDLOCK + + Allow mandatory locks on an FS. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_WRITE + + Write on file/directory/symlink. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_APPEND + + Append-only file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_IMMUTABLE + + Immutable file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOATIME + + Do not update access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NODIRATIME + + Do not update directory access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_RELATIME + + Update atime relative to mtime/ctime. + + .. availability:: Linux. + + .. versionadded:: 3.4 .. data:: supports_dir_fd @@ -4203,10 +4409,14 @@ features: the file descriptor, and as such multiple files can have the same name without any side effects. - .. availability:: Linux >= 3.17 with glibc >= 2.27. + .. availability:: Linux >= 3.17. .. versionadded:: 3.8 + .. versionchanged:: 3.16 + The function is now also available when Python is built against a libc + that lacks ``memfd_create()``, such as glibc older than 2.27. + .. data:: MFD_CLOEXEC MFD_ALLOW_SEALING @@ -4367,16 +4577,17 @@ Naturally, they are all only available on Linux. - :const:`time.CLOCK_BOOTTIME` (Since Linux 3.15 for timerfd_create) If *clockid* is :const:`time.CLOCK_REALTIME`, a settable system-wide - real-time clock is used. If system clock is changed, timer setting need - to be updated. To cancel timer when system clock is changed, see + real-time clock is used. If the system clock is changed, the timer setting + needs to be updated. To cancel the timer when the system clock is changed, see :const:`TFD_TIMER_CANCEL_ON_SET`. If *clockid* is :const:`time.CLOCK_MONOTONIC`, a non-settable monotonically increasing clock is used. Even if the system clock is changed, the timer setting will not be affected. - If *clockid* is :const:`time.CLOCK_BOOTTIME`, same as :const:`time.CLOCK_MONOTONIC` - except it includes any time that the system is suspended. + If *clockid* is :const:`time.CLOCK_BOOTTIME`, it is the same as + :const:`time.CLOCK_MONOTONIC` except it includes any time that the system + is suspended. The file descriptor's behaviour can be modified by specifying a *flags* value. Any of the following variables may be used, combined using bitwise OR @@ -4387,8 +4598,8 @@ Naturally, they are all only available on Linux. If :const:`TFD_NONBLOCK` is not set as a flag, :func:`read` blocks until the timer expires. If it is set as a flag, :func:`read` doesn't block, but - If there hasn't been an expiration since the last call to read, - :func:`read` raises :class:`OSError` with ``errno`` is set to + if there hasn't been an expiration since the last call to read, + :func:`read` raises :class:`OSError` with ``errno`` set to :const:`errno.EAGAIN`. :const:`TFD_CLOEXEC` is always set by Python automatically. @@ -4403,7 +4614,7 @@ Naturally, they are all only available on Linux. .. versionadded:: 3.13 -.. function:: timerfd_settime(fd, /, *, flags=flags, initial=0.0, interval=0.0) +.. function:: timerfd_settime(fd, /, *, flags=0, initial=0.0, interval=0.0) Alter a timer file descriptor's internal timer. This function operates the same interval timer as :func:`timerfd_settime_ns`. @@ -4418,12 +4629,11 @@ Naturally, they are all only available on Linux. - :const:`TFD_TIMER_CANCEL_ON_SET` The timer is disabled by setting *initial* to zero (``0``). - If *initial* is equal to or greater than zero, the timer is enabled. + If *initial* is greater than zero, the timer is enabled. If *initial* is less than zero, it raises an :class:`OSError` exception - with ``errno`` set to :const:`errno.EINVAL` + with ``errno`` set to :const:`errno.EINVAL`. By default the timer will fire when *initial* seconds have elapsed. - (If *initial* is zero, timer will fire immediately.) However, if the :const:`TFD_TIMER_ABSTIME` flag is set, the timer will fire when the timer's clock @@ -4434,13 +4644,13 @@ Naturally, they are all only available on Linux. If *interval* is greater than zero, the timer fires every time *interval* seconds have elapsed since the previous expiration. If *interval* is less than zero, it raises :class:`OSError` with ``errno`` - set to :const:`errno.EINVAL` + set to :const:`errno.EINVAL`. If the :const:`TFD_TIMER_CANCEL_ON_SET` flag is set along with :const:`TFD_TIMER_ABSTIME` and the clock for this timer is :const:`time.CLOCK_REALTIME`, the timer is marked as cancelable if the real-time clock is changed discontinuously. Reading the descriptor is - aborted with the error ECANCELED. + aborted with the error :const:`errno.ECANCELED`. Linux manages system clock as UTC. A daylight-savings time transition is done by changing time offset only and doesn't cause discontinuous system @@ -5062,13 +5272,25 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Linux >= 5.10 .. versionadded:: 3.12 +.. function:: pidfd_getfd(pidfd, targetfd, *, flags=0) + + Duplicate *targetfd* from the process referred to by the process file + descriptor *pidfd*, into the calling process. The returned file descriptor + is :ref:`non-inheritable `. + + *flags* is reserved, and currently must be ``0``. + + See the :manpage:`pidfd_getfd(2)` man page for more details. + + .. availability:: Linux >= 5.6, Android >= :func:`build-time ` API level 31 + .. versionadded:: next .. function:: plock(op, /) Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not WASI, not iOS. + .. availability:: Unix, not WASI, not macOS, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -5110,9 +5332,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -5340,9 +5561,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 2b4aa1ee2099975..ab92c142c37a4f6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1266,6 +1266,8 @@ Reading and writing files >>> p.read_text() 'Text file contents' + Return the number of characters written. + An existing file of the same name is overwritten. The optional parameters have the same meaning as in :func:`open`. @@ -1286,6 +1288,8 @@ Reading and writing files >>> p.read_bytes() b'Binary file contents' + Return the number of bytes written. + An existing file of the same name is overwritten. .. versionadded:: 3.5 @@ -1351,6 +1355,11 @@ Reading directories ``False``, this method follows symlinks except when expanding "``**``" wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 @@ -1377,6 +1386,11 @@ Reading directories The paths are returned in no particular order. If you need a specific order, sort the results. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. seealso:: :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. @@ -1504,7 +1518,8 @@ Creating files and directories :meth:`~Path.write_bytes` methods are often used to create files. -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) +.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False, *, \ + parent_mode=None) Create a new directory at this given path. If *mode* is given, it is combined with the process's ``umask`` value to determine the file mode @@ -1515,6 +1530,12 @@ Creating files and directories as needed; they are created with the default permissions without taking *mode* into account (mimicking the POSIX ``mkdir -p`` command). + If *parent_mode* is not ``None``, it is used as the mode for any + newly-created, intermediate-level directories when *parents* is true. + Like *mode*, it is combined with the process's ``umask`` value. + Otherwise, intermediate directories are created with the default + permissions (also subject to the umask). + If *parents* is false (the default), a missing parent raises :exc:`FileNotFoundError`. @@ -1528,6 +1549,9 @@ Creating files and directories .. versionchanged:: 3.5 The *exist_ok* parameter was added. + .. versionadded:: 3.15 + The *parent_mode* parameter. + .. method:: Path.symlink_to(target, target_is_directory=False) diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index f8975c2f4281d45..8eadc2cf2b1ef0d 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -56,7 +56,7 @@ The :mod:`!pickle` module differs from :mod:`marshal` in several significant way * :mod:`marshal` cannot be used to serialize user-defined classes and their instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support @@ -693,7 +693,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 7a771ea3ab93d41..769ca60af1837ea 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -79,6 +79,9 @@ Command-line options A pickle file to read, or ``-`` to indicate reading from standard input. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. Programmatic interface diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index aa7dd71c1329dfd..5473a367c49a3ab 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -194,7 +194,7 @@ support. The :mod:`importlib.resources` module provides structured access to module resources. -.. function:: resolve_name(name) +.. function:: resolve_name(name, *, strict=False) Resolve a name to an object. @@ -208,6 +208,7 @@ support. * ``W(.W)*`` * ``W(.W)*:(W(.W)*)?`` + * ``W(.W)*:(W(.W)*)`` The first form is intended for backward compatibility only. It assumes that some part of the dotted name is a package, and the rest is an object @@ -222,6 +223,11 @@ support. hierarchy within that package. Only one import is needed in this form. If it ends with the colon, then a module object is returned. + The first two forms are accepted when ``strict=False`` (the default). + + The third form requires both the module name and callable, separated by + a colon. Only this form is accepted when ``strict=True``. + The function will return an object (which might be a module), or raise one of the following exceptions: @@ -233,3 +239,7 @@ support. hierarchy within the imported package to get to the desired object. .. versionadded:: 3.9 + + .. versionchanged:: 3.15 + + The optional keyword-only ``strict`` flag was added. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 1d30966794fd1bf..f728a7b32e789f4 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -141,6 +141,8 @@ Cross platform Returns the system's release, e.g. ``'2.2.0'`` or ``'NT'``. An empty string is returned if the value cannot be determined. + On iOS and Android, this is the user-facing OS release. To obtain the + Darwin or Linux kernel release, use :func:`os.uname`. .. function:: system() @@ -163,9 +165,6 @@ Cross platform Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is returned if the value cannot be determined. - On iOS and Android, this is the user-facing OS version. To obtain the - Darwin or Linux kernel version, use :func:`os.uname`. - .. function:: uname() Fairly portable uname interface. Returns a :func:`~collections.namedtuple` diff --git a/Doc/library/profiling.sampling.rst b/Doc/library/profiling.sampling.rst index a6ce2f30eadb2c0..aeb1a429b58515a 100644 --- a/Doc/library/profiling.sampling.rst +++ b/Doc/library/profiling.sampling.rst @@ -17,7 +17,7 @@ -------------- -.. image:: tachyon-logo.png +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png :alt: Tachyon logo :align: center :width: 300px @@ -153,6 +153,10 @@ Attach to a running process by PID:: python -m profiling.sampling attach 12345 +Print a single snapshot of a running process's stack:: + + python -m profiling.sampling dump 12345 + Use live mode for real-time monitoring (press ``q`` to quit):: python -m profiling.sampling run --live script.py @@ -173,8 +177,9 @@ Enable opcode-level profiling to see which bytecode instructions are executing:: Commands ======== -Tachyon operates through two subcommands that determine how to obtain the -target process. +Tachyon operates through several subcommands. ``run`` and ``attach`` collect +samples over time; ``dump`` captures a single snapshot; ``replay`` converts +binary profiles to other formats. The ``run`` command @@ -217,6 +222,78 @@ On most systems, attaching to another process requires appropriate permissions. See :ref:`profiling-permissions` for platform-specific requirements. +.. _dump-command: + +The ``dump`` command +-------------------- + +The ``dump`` command prints a single snapshot of a running process's Python +stack and exits, similar to a traceback:: + + python -m profiling.sampling dump 12345 + +Unlike ``attach``, ``dump`` does not run a sampling loop: it reads the +stack once. This is useful for investigating hung or unresponsive +processes, or for answering "what is this process doing right now?". + +The output mirrors a traceback (most recent call last) and annotates each +thread with its current state (main thread, has GIL, on CPU, waiting for +GIL, has exception, or idle): + +.. code-block:: text + + Stack dump for PID 12345, thread 140735 (main thread, has GIL, on CPU; most recent call last): + File "server.py", line 28, in serve + await handle_request(req) + File "handler.py", line 91, in handle_request + result = expensive_call(req) + +When the target's source files are readable, ``dump`` prints the source +line for each frame and highlights the executing expression. + +Like ``attach``, ``dump`` requires permission to read the target process's +memory. See :ref:`profiling-permissions`. + +The ``dump`` command supports the following options: + +``-a``, ``--all-threads`` + Dump every thread in the target process. Without this flag only the main + thread is shown. + +``--native`` + Include synthetic ```` frames marking transitions into C + extensions or other non-Python code. + +``--no-gc`` + Hide the synthetic ```` frames that mark active garbage collection. + +``--opcodes`` + Annotate each frame with the bytecode opcode the thread is currently + executing (for example, ``opcode=CALL_KW``). Useful for + instruction-level investigation, including identifying specializations + chosen by the adaptive interpreter. + +``--async-aware`` + Reconstruct stacks across ``await`` boundaries. ``dump`` walks the task + graph and emits one section per task, with ```` markers separating + coroutines awaiting each other. + +``--async-mode {running,all}`` + Controls which tasks are included when ``--async-aware`` is enabled. + ``running`` shows only the task currently executing on each thread; + ``all`` (the default for ``dump``) also includes tasks suspended on a + wait. ``attach``'s default for this flag is ``running``; ``dump`` + defaults to ``all`` because a single snapshot is most useful when it + shows the full task graph. + +``--blocking`` + Pause every thread in the target while reading its stack and resume + them after. Guarantees a fully consistent snapshot at the cost of + briefly stopping the target. Without it, ``dump`` reads memory while + the target keeps running, which is faster but can occasionally produce + a torn stack. + + .. _replay-command: The ``replay`` command @@ -1441,11 +1518,52 @@ Global options Attach to and profile a running process by PID. +.. option:: dump + + Print a single one-shot snapshot of a running process's Python stack. + .. option:: replay Convert a binary profile file to another output format. +Dump options +------------ + +The following options apply to the ``dump`` subcommand: + +.. option:: -a, --all-threads + + Dump all threads in the target process instead of just the main thread. + +.. option:: --native + + Include ```` frames for non-Python code. + +.. option:: --no-gc + + Exclude ```` frames for active garbage collection. + +.. option:: --opcodes + + Show bytecode opcode names when available. + +.. option:: --async-aware + + Reconstruct the stack across ``await`` boundaries for asyncio + applications. + +.. option:: --async-mode + + Async stack mode: ``running`` (only the running task) or ``all`` + (all tasks including waiting). Defaults to ``all`` for ``dump``. + Requires :option:`--async-aware`. + +.. option:: --blocking + + Pause all threads in the target process while reading the stack. + + Sampling options ---------------- diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index f236eba84576575..a0cfb440a36ffa9 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -68,6 +68,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2e6938b5cf68602..638b8f2862b7d19 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -35,7 +35,7 @@ the XML document. This module uses the :mod:`pyexpat` module to provide access to the Expat parser. Direct use of the :mod:`pyexpat` module is deprecated. -This module provides one exception and one type object: +This module provides the following exception, type object and data items: .. exception:: ExpatError @@ -53,6 +53,29 @@ This module provides one exception and one type object: The type of the return values from the :func:`ParserCreate` function. + +.. data:: EXPAT_VERSION + + The version string of the Expat library loaded by the interpreter, + like ``'expat_2.8.4'``. + + +.. data:: version_info + + The version of the Expat library loaded by the interpreter, + as a tuple of three integers: major, minor and micro version. + + +.. data:: features + + The list of the features with which the loaded Expat library + was compiled, as ``(name, value)`` pairs. + The value is only meaningful for features which have one, + like ``'XML_CONTEXT_BYTES'`` or the default protection limits + ``'XML_BLAP_ACT_THRES'`` and ``'XML_AT_MAX_AMP'``; + for other features, like ``'XML_DTD'`` and ``'XML_NS'``, + the value is ``0`` and only the presence of the name is significant. + The :mod:`!xml.parsers.expat` module contains two functions: @@ -61,14 +84,28 @@ The :mod:`!xml.parsers.expat` module contains two functions: Returns an explanatory string for a given error number *errno*. -.. function:: ParserCreate(encoding=None, namespace_separator=None) +.. function:: ParserCreate(encoding=None, namespace_separator=None, intern=None) + + Creates and returns a new :class:`xmlparser` object. + *encoding* [1]_, if specified, must be a string naming the encoding + used by the XML data. + If it is given it will override the implicit or explicit encoding + of the document. + + .. impl-detail:: + + Expat natively understands and processes UTF-8, UTF-16, UTF-16BE, + UTF-16LE, ISO-8859-1, and US-ASCII. + For other encodings (including aliases like Latin1 and ASCII) it + falls back to Python. + It supports most of 8-bit encodings and many multi-byte encodings + like Shift_JIS, although only BMP characters (``U+0000-U+FFFF``) + are supported with non-native encodings (this restriction is also + applied to aliases like UTF8). + These restrictions only apply if *encoding* is not given. - Creates and returns a new :class:`xmlparser` object. *encoding*, if specified, - must be a string naming the encoding used by the XML data. Expat doesn't - support as many encodings as Python does, and its repertoire of encodings can't - be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII. If - *encoding* [1]_ is given it will override the implicit or explicit encoding of the - document. + .. versionchanged:: next + Added support for multi-byte encodings. .. _xmlparser-non-root: @@ -108,12 +145,16 @@ The :mod:`!xml.parsers.expat` module contains two functions: http://www.python.org/ns/ elem1 elem2 + *intern*, if given, must be a dictionary. + It is used to intern the names of elements and attributes, + and is available as the :attr:`~xmlparser.intern` attribute. + By default a new empty dictionary is created for every parser. + Due to limitations in the ``Expat`` library used by :mod:`pyexpat`, the :class:`xmlparser` instance returned can only be used to parse a single XML document. Call ``ParserCreate`` for each document to provide unique parser instances. - .. seealso:: `The Expat XML Parser `_ @@ -130,18 +171,24 @@ XMLParser Objects .. method:: xmlparser.Parse(data[, isfinal]) - Parses the contents of the string *data*, calling the appropriate handler - functions to process the parsed data. *isfinal* must be true on the final call - to this method; it allows the parsing of a single file in fragments, + Parses the contents of *data*, + calling the appropriate handler functions to process the parsed data. + *data* can be a :term:`bytes-like object` or a string. + If it is a string, the encoding declaration in the XML data is ignored, + and the data is parsed as already decoded text. + *isfinal* must be true on the final call to this method; + it allows the parsing of a single file in fragments, not the submission of multiple files. - *data* can be the empty string at any time. + *data* can be empty at any time. .. method:: xmlparser.ParseFile(file) - Parse XML data reading from the object *file*. *file* only needs to provide - the ``read(nbytes)`` method, returning the empty string when there's no more - data. + Parse XML data reading from the object *file*. + *file* only needs to provide the ``read(nbytes)`` method, + which returns bytes, and an empty bytes object when there's no more data. + Text files are not supported; + use :meth:`Parse` for data which is already decoded. .. method:: xmlparser.SetBase(base) @@ -159,11 +206,31 @@ XMLParser Objects or ``None`` if :meth:`SetBase` hasn't been called. +.. method:: xmlparser.GetSpecifiedAttributeCount() + + Return the index just past the attributes given in the start tag. + Attributes defaulted from the DTD follow the specified ones, + so attributes at lower indices in the list + passed to :attr:`StartElementHandler` were given in the start tag. + Each attribute takes two items in that list, + its name and its value. + Only meaningful inside a :attr:`StartElementHandler` call, + and only if :attr:`ordered_attributes` is true. + + .. versionadded:: next + + .. method:: xmlparser.GetInputContext() - Returns the input data that generated the current event as a string. The data is - in the encoding of the entity which contains the text. When called while an - event handler is not active, the return value is ``None``. + Returns the input data which generated the current event + as a :class:`bytes` object. + The data is in the encoding of the entity which contains the text. + It extends to the end of the currently buffered input, + therefore it can contain also the data of the following events, + and if the event was generated by a large amount of text, + not all of it may be available. + When called while an event handler is not active, + the return value is ``None``. .. method:: xmlparser.ExternalEntityParserCreate(context[, encoding]) @@ -410,6 +477,22 @@ against some common XML vulnerabilities. default, this attribute is false; it may be changed at any time. +.. attribute:: xmlparser.intern + + The dictionary used to intern the names of elements and attributes. + It is either the dictionary passed as the *intern* argument + of :func:`ParserCreate`, or a new dictionary created for this parser. + + +.. attribute:: xmlparser.namespace_prefixes + + If set to a true value, and namespace processing is enabled, + the namespace prefix is reported as the third part of the expanded name, + separated by the namespace separator. + Names which have no prefix are not affected. + By default, this attribute is false; it may be changed at any time. + + The following attributes contain values relating to the most recent error encountered by an :class:`xmlparser` object, and will only have correct values once a call to :meth:`Parse` or :meth:`ParseFile` has raised an @@ -473,8 +556,7 @@ otherwise stated. encoding of the document text, and an optional "standalone" declaration. *version* and *encoding* will be strings, and *standalone* will be ``1`` if the document is declared standalone, ``0`` if it is declared not to be standalone, - or ``-1`` if the standalone clause was omitted. This is only available with - Expat version 1.95.0 or newer. + or ``-1`` if the standalone clause was omitted. .. method:: xmlparser.StartDoctypeDeclHandler(doctypeName, systemId, publicId, has_internal_subset) @@ -483,14 +565,12 @@ otherwise stated. ...``). The *doctypeName* is provided exactly as presented. The *systemId* and *publicId* parameters give the system and public identifiers if specified, or ``None`` if omitted. *has_internal_subset* will be true if the document - contains an internal document declaration subset. This requires Expat version - 1.2 or newer. + contains an internal document declaration subset. .. method:: xmlparser.EndDoctypeDeclHandler() - Called when Expat is done parsing the document type declaration. This requires - Expat version 1.2 or newer. + Called when Expat is done parsing the document type declaration. .. method:: xmlparser.ElementDeclHandler(name, model) @@ -505,12 +585,16 @@ otherwise stated. declaration declares three attributes, this handler is called three times, once for each attribute. *elname* is the name of the element to which the declaration applies and *attname* is the name of the attribute declared. The - attribute type is a string passed as *type*; the possible values are - ``'CDATA'``, ``'ID'``, ``'IDREF'``, ... *default* gives the default value for + The attribute type is a string passed as *type*: + ``'CDATA'``, ``'ID'``, ``'IDREF'``, ``'IDREFS'``, ``'ENTITY'``, + ``'ENTITIES'``, ``'NMTOKEN'`` or ``'NMTOKENS'``, + an enumeration like ``'(x|y)'``, + or a notation list like ``'NOTATION(n1|n2)'``. + *default* gives the default value for the attribute used when the attribute is not specified by the document instance, or ``None`` if there is no default value (``#IMPLIED`` values). If the attribute is required to be given in the document instance, *required* will be - true. This requires Expat version 1.95.0 or newer. + true. .. method:: xmlparser.StartElementHandler(name, attributes) @@ -546,10 +630,10 @@ otherwise stated. .. method:: xmlparser.UnparsedEntityDeclHandler(entityName, base, systemId, publicId, notationName) - Called for unparsed (NDATA) entity declarations. This is only present for - version 1.2 of the Expat library; for more recent versions, use - :attr:`EntityDeclHandler` instead. (The underlying function in the Expat - library has been declared obsolete.) + Called for unparsed (NDATA) entity declarations. + If this handler is not set, such declarations are reported by + :attr:`EntityDeclHandler`, which is preferred for new code. + (The underlying function in the Expat library has been declared obsolete.) .. method:: xmlparser.EntityDeclHandler(entityName, is_parameter_entity, value, base, systemId, publicId, notationName) @@ -560,8 +644,7 @@ otherwise stated. ``None`` for parsed entities, and the name of the notation for unparsed entities. *is_parameter_entity* will be true if the entity is a parameter entity or false for general entities (most applications only need to be concerned with - general entities). This is only available starting with version 1.95.0 of the - Expat library. + general entities). .. method:: xmlparser.NotationDeclHandler(notationName, base, systemId, publicId) @@ -615,7 +698,7 @@ otherwise stated. .. method:: xmlparser.DefaultHandlerExpand(data) - This is the same as the :func:`DefaultHandler`, but doesn't inhibit expansion + This is the same as the :attr:`DefaultHandler`, but doesn't inhibit expansion of internal entities. The entity reference will not be passed to the default handler. @@ -659,6 +742,16 @@ otherwise stated. :attr:`DefaultHandler` callback, if provided. +.. method:: xmlparser.SkippedEntityHandler(entityName, is_parameter_entity) + + Called for entity references which are not expanded, + because the parser did not read the declaration of the entity. + This happens when the external DTD subset or an external parameter entity + is not parsed. + *is_parameter_entity* is true for a parameter entity + and false for a general entity. + + .. _expaterror-objects: ExpatError Exceptions @@ -759,35 +852,35 @@ The constants in the model type group are: .. data:: XML_CTYPE_ANY - :noindex: The element named by the model name was declared to have a content model of ``ANY``. .. data:: XML_CTYPE_CHOICE - :noindex: The named element allows a choice from a number of options; this is used for content models such as ``(A | B | C)``. .. data:: XML_CTYPE_EMPTY - :noindex: Elements which are declared to be ``EMPTY`` have this model type. .. data:: XML_CTYPE_MIXED - :noindex: + + The named element allows character data, optionally interspersed with + the named children; this is used for content models such as + ``(#PCDATA)`` and ``(#PCDATA | A | B)*``. .. data:: XML_CTYPE_NAME - :noindex: + + The model names a single element, as for ``A``. .. data:: XML_CTYPE_SEQ - :noindex: Models which represent a series of models which follow one after the other are indicated with this model type. This is used for models such as ``(A, B, C)``. @@ -796,25 +889,21 @@ The constants in the quantifier group are: .. data:: XML_CQUANT_NONE - :noindex: No modifier is given, so it can appear exactly once, as for ``A``. .. data:: XML_CQUANT_OPT - :noindex: The model is optional: it can appear once or not at all, as for ``A?``. .. data:: XML_CQUANT_PLUS - :noindex: The model must occur one or more times (like ``A+``). .. data:: XML_CQUANT_REP - :noindex: The model must occur zero or more times, as for ``A*``. @@ -898,7 +987,7 @@ The ``errors`` module has the following attributes: .. data:: XML_ERROR_NO_ELEMENTS The document contains no elements (XML requires all documents to contain exactly - one top-level element).. + one top-level element). .. data:: XML_ERROR_NO_MEMORY @@ -1001,7 +1090,7 @@ The ``errors`` module has the following attributes: .. data:: XML_ERROR_XML_DECL - The document contained no document element at all. + There was an error parsing the XML declaration. .. data:: XML_ERROR_TEXT_DECL @@ -1083,9 +1172,11 @@ The ``errors`` module has the following attributes: .. rubric:: Footnotes -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl +.. [1] The encoding string included in XML output should conform to + the appropriate standards. For example, "UTF-8" is valid, but + "UTF8" is not valid in an XML document's declaration, even though + Python accepts it as an encoding name. + See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 5ac72ef7604d50c..79da2a3db719e78 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -76,6 +76,8 @@ The :mod:`!queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 @@ -171,7 +173,7 @@ provide the public methods described below. .. method:: Queue.get_nowait() - Equivalent to ``get(False)``. + Equivalent to ``get(block=False)``. Two methods are offered to support tracking whether enqueued tasks have been fully processed by daemon consumer threads. diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 43fb7295876fe1a..1aefb91e0ceb405 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -52,7 +52,7 @@ fine-tuning parameters. .. _re-syntax: -Regular Expression Syntax +Regular expression syntax ------------------------- A regular expression (or RE) specifies a set of strings that matches it; the @@ -205,7 +205,7 @@ The special characters are: *without* establishing any backtracking points. This is the possessive version of the quantifier above. For example, on the 6-character string ``'aaaaaa'``, ``a{3,5}+aa`` - attempt to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, + attempts to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, will need more characters than available and thus fail, while ``a{3,5}aa`` will match with ``a{3,5}`` capturing 5, then 4 ``'a'``\ s by backtracking and then the final 2 ``'a'``\ s are matched by the final @@ -279,25 +279,47 @@ The special characters are: ``[]()[{}]`` will match a right bracket, as well as left bracket, braces, and parentheses. - .. .. index:: single: --; in regular expressions - .. .. index:: single: &&; in regular expressions - .. .. index:: single: ~~; in regular expressions - .. .. index:: single: ||; in regular expressions - - * Support of nested sets and set operations as in `Unicode Technical - Standard #18`_ might be added in the future. This would change the - syntax, so to facilitate this change a :exc:`FutureWarning` will be raised - in ambiguous cases for the time being. - That includes sets starting with a literal ``'['`` or containing literal - character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To - avoid a warning escape them with a backslash. + .. index:: + single: --; in regular expressions + single: &&; in regular expressions + single: ||; in regular expressions + + * A character set may contain a nested set written in square brackets, and + two sets may be combined with a set operator, as in `Unicode Technical + Standard #18`_: + + * ``[A--B]`` (*difference*) matches a character that is in *A* but not + in *B*; for example ``[a-z--[aeiou]]`` matches an ASCII lowercase + consonant. + * ``[A&&B]`` (*intersection*) matches a character that is in both *A* + and *B*; for example ``[\w&&[a-z]]`` matches an ASCII lowercase letter. + * ``[A||B]`` (*union*) matches a character that is in *A* or in *B*; this + is the same as listing the members of both sets in a single set, but + allows combining nested sets. + + Operators have no precedence and are applied from left to right. To + group, write a nested set as the operand after an operator, as in + ``[a-z--[aeiou]]``. A leading ``'^'`` complements the whole result. + A ``'['`` begins a nested set only immediately after a set operator; + anywhere else -- including at the start of a character set -- it is an + ordinary character, so existing patterns keep their meaning. Escape it + as ``'\['`` to include a literal ``'['`` right after an operator. .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ + .. note:: + + Symmetric difference (``A~~B``) is not yet supported; a literal ``'~~'`` + in a character set still raises a :exc:`FutureWarning`. + .. versionchanged:: 3.7 :exc:`FutureWarning` is raised if a character set contains constructs that will change semantically in the future. + .. versionchanged:: next + Added support for nested sets and the set operators ``--``, ``&&`` + and ``||``. + .. index:: single: | (vertical bar); in regular expressions ``|`` @@ -509,8 +531,9 @@ The special characters are: *name* exists, and with ``no-pattern`` if it doesn't. ``no-pattern`` is optional and can be omitted. For example, ``(<)?(\w+@\w+(?:\.\w+)+)(?(1)>|$)`` is a poor email matching pattern, which - will match with ``''`` as well as ``'user@host.com'``, but - not with ``''``. + matches ``''`` as well as ``'user@host.com'``, but does not + match ``''`` in their entirety + (:func:`re.search` finds only ``'user@host.com'`` in the former). .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. @@ -591,7 +614,7 @@ character ``'$'``. Matches ``[0-9]`` if the :py:const:`~re.ASCII` flag is used. - __ https://www.unicode.org/versions/Unicode15.0.0/ch04.pdf#G134153 + __ https://www.unicode.org/versions/Unicode17.0.0/core-spec/chapter-4/#G124142 For 8-bit (bytes) patterns: Matches any decimal digit in the ASCII character set; @@ -658,6 +681,51 @@ character ``'$'``. matches characters which are neither alphanumeric in the current locale nor the underscore. +.. index:: single: \p; in regular expressions + single: \P; in regular expressions + +``\p{property=value}``, ``\p{value}`` + Matches any character with the given Unicode property + (see `Unicode Technical Standard #18 + `_, requirement RL1.2 "Properties"). + Property and value names are matched loosely: + case, whitespace, ``'-'`` and ``'_'`` are ignored. + The following properties are supported: + + * The ``General_Category`` property (short name ``gc``), + spelled ``\p{Lu}``, ``\p{gc=Lu}`` or, for a one-letter group, ``\p{L}``. + The supported values are the groups ``L``, ``N``, ``Z`` and ``C`` and the + values ``Lu``, ``Lt``, ``Lm``, ``Nd``, ``Nl``, ``No``, ``Zs``, ``Zl``, + ``Zp``, ``Cc``, ``Cf``, ``Cs``, ``Co`` and ``Cn``. + * The binary properties ``XID_Start``, ``XID_Continue``, ``Alphabetic``, + ``Lowercase``, ``Uppercase``, ``Numeric``, ``Printable``, ``Cased`` and + ``Case_Ignorable``. A binary property may also be spelled + ``\p{name=yes}`` or ``\p{name=no}``. + * The POSIX compatibility classes ``alpha``, ``alnum``, ``blank``, + ``cntrl``, ``digit``, ``graph``, ``lower``, ``print``, ``space``, + ``upper``, ``word`` and ``xdigit``. + * The properties ``ASCII``, ``Any``, ``Assigned``, + ``Noncharacter_Code_Point``, ``Join_Control``, ``Regional_Indicator``, + ``ASCII_Hex_Digit``, ``Hex_Digit``, ``Pattern_Syntax`` and + ``Pattern_White_Space``. + + Where a supported property corresponds to a :mod:`unicodedata` accessor or + :class:`str` method, the set of characters it matches is exactly the one + they report. For consistency with these, ``space`` follows + :py:meth:`str.isspace` (like ``\s``) and ``xdigit`` matches only the ASCII + hexadecimal digits. + + This is only recognized in Unicode (str) patterns. + In bytes patterns it is an error. + + .. versionadded:: next + +``\P{...}`` + Matches any character which does *not* have the given Unicode property. + This is the opposite of ``\p``. + + .. versionadded:: next + .. index:: single: \z; in regular expressions single: \Z; in regular expressions @@ -717,7 +785,7 @@ three digits in length. .. _contents-of-module-re: -Module Contents +Module contents --------------- The module defines several functions, constants, and an exception. Some of the @@ -833,8 +901,8 @@ Flags will be conditionally ORed with other flags. Example of use as a default value:: - def myfunc(text, flag=re.NOFLAG): - return re.search(text, flag) + def myfunc(pattern, text, flag=re.NOFLAG): + return re.search(pattern, text, flag) .. versionadded:: 3.11 @@ -931,7 +999,6 @@ Functions .. function:: prefixmatch(pattern, string, flags=0) -.. function:: match(pattern, string, flags=0) If zero or more characters at the beginning of *string* match the regular expression *pattern*, return a corresponding :class:`~re.Match`. Return @@ -954,9 +1021,14 @@ Functions :func:`~re.match`. Use that name when you need to retain compatibility with older Python versions. - .. versionchanged:: 3.15 - The alternate :func:`~re.prefixmatch` name of this API was added as a - more explicitly descriptive name than :func:`~re.match`. Use it to better + .. versionadded:: 3.15 + +.. function:: match(pattern, string, flags=0) + + .. soft-deprecated:: 3.15 + :func:`~re.match` has been :term:`soft deprecated` in favor of + the alternate :func:`~re.prefixmatch` name of this API which is + more explicitly descriptive. Use it to better express intent. The norm in other languages and regular expression implementations is to use the term *match* to refer to the behavior of what Python has always called :func:`~re.search`. @@ -1246,13 +1318,16 @@ Exceptions .. _re-objects: -Regular Expression Objects +Regular expression objects -------------------------- .. class:: Pattern Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1284,7 +1359,6 @@ Regular Expression Objects .. method:: Pattern.prefixmatch(string[, pos[, endpos]]) -.. method:: Pattern.match(string[, pos[, endpos]]) If zero or more characters at the *beginning* of *string* match this regular expression, return a corresponding :class:`~re.Match`. Return ``None`` if the @@ -1309,9 +1383,14 @@ Regular Expression Objects :meth:`~Pattern.match`. Use that name when you need to retain compatibility with older Python versions. - .. versionchanged:: 3.15 - The alternate :meth:`~Pattern.prefixmatch` name of this API was added as - a more explicitly descriptive name than :meth:`~Pattern.match`. Use it to + .. versionadded:: 3.15 + +.. method:: Pattern.match(string[, pos[, endpos]]) + + .. soft-deprecated:: 3.15 + :meth:`~Pattern.match` has been :term:`soft deprecated` in favor of + the alternate :meth:`~Pattern.prefixmatch` name of this API which is + more explicitly descriptive. Use it to better express intent. The norm in other languages and regular expression implementations is to use the term *match* to refer to the behavior of what Python has always called :meth:`~Pattern.search`. @@ -1396,7 +1475,7 @@ Regular Expression Objects .. _match-objects: -Match Objects +Match objects ------------- Match objects always have a boolean value of ``True``. @@ -1411,6 +1490,9 @@ when there is no match, you can test whether there was a match with a simple Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. @@ -1615,11 +1697,11 @@ when there is no match, you can test whether there was a match with a simple .. _re-examples: -Regular Expression Examples +Regular expression examples --------------------------- -Checking for a Pair +Checking for a pair ^^^^^^^^^^^^^^^^^^^ In this example, we'll use the following helper function to display match @@ -1705,15 +1787,21 @@ expressions. | ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | +--------------------------------+---------------------------------------------+ -To extract the filename and numbers from a string like :: +To extract the filename and numbers from a string like: + +.. code-block:: text /usr/sbin/sendmail - 0 errors, 4 warnings -you would use a :c:func:`!scanf` format like :: +you would use a :c:func:`!scanf` format like: + +.. code-block:: text %s - %d errors, %d warnings -The equivalent regular expression would be :: +The equivalent regular expression would be: + +.. code-block:: text (\S+) - (\d+) errors, (\d+) warnings @@ -1772,18 +1860,24 @@ not familiar with the Python API's divergence from what otherwise become the industry norm. Quoting from the Zen Of Python (``python3 -m this``): *"Explicit is better than -implicit"*. Anyone reading the name :func:`~re.prefixmatch` is likely to -understand the intended semantics. When reading :func:`~re.match` there remains +implicit"*. Anyone reading the name :func:`!prefixmatch` is likely to +understand the intended semantics. When reading :func:`!match` there remains a seed of doubt about the intended behavior to anyone not already familiar with this old Python gotcha. -We **do not** plan to deprecate and remove the older *match* name, +We **do not** plan to remove the older :func:`!match` name, as it has been used in code for over 30 years. -Code supporting older versions of Python should continue to use *match*. +It has been :term:`soft deprecated`: +code supporting older versions of Python should continue to use :func:`!match`, +while new code should prefer :func:`!prefixmatch`. .. versionadded:: 3.15 + :func:`!prefixmatch` + +.. soft-deprecated:: 3.15 + :func:`!match` -Making a Phonebook +Making a phonebook ^^^^^^^^^^^^^^^^^^ :func:`split` splits a string into a list delimited by the passed pattern. The @@ -1844,7 +1938,7 @@ house number from the street name: ['Heather', 'Albrecht', '548.326.4584', '919', 'Park Place']] -Text Munging +Text munging ^^^^^^^^^^^^ :func:`sub` replaces every occurrence of a pattern with a string or the @@ -1864,7 +1958,7 @@ in each word of a sentence except for the first and last characters:: 'Pofsroser Aodlambelk, plasee reoprt yuor asnebces potlmrpy.' -Finding all Adverbs +Finding all adverbs ^^^^^^^^^^^^^^^^^^^ :func:`findall` matches *all* occurrences of a pattern, not just the first @@ -1877,7 +1971,7 @@ the following manner:: ['carefully', 'quickly'] -Finding all Adverbs and their Positions +Finding all adverbs and their positions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If one wants more information about all matches of a pattern than the matched @@ -1893,7 +1987,7 @@ to find all of the adverbs *and their positions* in some text, they would use 40-47: quickly -Raw String Notation +Raw string notation ^^^^^^^^^^^^^^^^^^^ Raw string notation (``r"text"``) keeps regular expressions sane. Without it, @@ -1917,7 +2011,7 @@ functionally identical:: -Writing a Tokenizer +Writing a tokenizer ^^^^^^^^^^^^^^^^^^^ A `tokenizer or scanner `_ @@ -1933,7 +2027,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index 536b5980f86a817..9e1ec4691872f05 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -97,7 +97,7 @@ The :mod:`!runpy` module provides two functions: :class:`~importlib.machinery.ModuleSpec` for alternatives. .. versionchanged:: 3.15 - ``__cached__`` is no longer set. + The global variable ``__cached__`` is no longer set. .. function:: run_path(path_name, init_globals=None, run_name=None) @@ -106,7 +106,7 @@ The :mod:`!runpy` module provides two functions: Execute the code at the named filesystem location and return the resulting module's globals dictionary. As with a script name supplied to the CPython - command line, *file_path* may refer to a Python source file, a + command line, *path_name* may refer to a Python source file, a compiled bytecode file or a valid :data:`sys.path` entry containing a :mod:`__main__` module (e.g. a zipfile containing a top-level :file:`__main__.py` file). @@ -134,12 +134,12 @@ The :mod:`!runpy` module provides two functions: ``__name__`` is set to *run_name* if this optional argument is not :const:`None` and to ``''`` otherwise. - If *file_path* directly references a script file (whether as source + If *path_name* directly references a script file (whether as source or as precompiled byte code), then ``__file__`` will be set to - *file_path*, and ``__spec__``, ``__loader__`` and + *path_name*, and ``__spec__``, ``__loader__`` and ``__package__`` will all be set to :const:`None`. - If *file_path* is a reference to a valid :data:`sys.path` entry, then + If *path_name* is a reference to a valid :data:`sys.path` entry, then ``__spec__`` will be set appropriately for the imported :mod:`__main__` module (that is, ``__spec__.name`` will always be ``__main__``). ``__file__``, ``__loader__`` and ``__package__`` will be @@ -147,7 +147,7 @@ The :mod:`!runpy` module provides two functions: A number of alterations are also made to the :mod:`sys` module. Firstly, :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated - with the value of *file_path* and ``sys.modules[__name__]`` is updated + with the value of *path_name* and ``sys.modules[__name__]`` is updated with a temporary module object for the module being executed. All modifications to items in :mod:`sys` are reverted before the function returns. @@ -175,7 +175,7 @@ The :mod:`!runpy` module provides two functions: ``__package__`` are deprecated. .. versionchanged:: 3.15 - ``__cached__`` is no longer set. + The global variable ``__cached__`` is no longer set. .. seealso:: diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 70541c5f3cb3676..037e27f031d0c82 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -117,9 +117,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 3b5b57fb1c21702..842e8c512def139 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -137,7 +137,7 @@ Other functions :term:`bytes-like objects ` *a* and *b* are equal, otherwise ``False``, using a "constant-time compare" to reduce the risk of - `timing attacks `_. + `timing attacks `__. See :func:`hmac.compare_digest` for additional details. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index 09563af14d018a4..6400005871746a5 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -94,18 +96,22 @@ The module defines the following: When CPython is built, this function may be disabled using :option:`--disable-epoll`. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -113,12 +119,16 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + .. function:: select(rlist, wlist, xlist, timeout=None) @@ -190,7 +200,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -285,52 +295,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -362,12 +372,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -396,7 +406,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -421,24 +431,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -489,7 +499,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -533,7 +543,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -553,66 +563,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permits control() to return the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disables event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -620,19 +630,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -640,22 +650,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -663,19 +673,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index 2ab12f2f6f9169e..2dfb0246d5d90c0 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -44,12 +44,15 @@ The :mod:`!shlex` module defines the following functions: .. versionadded:: 3.8 -.. function:: quote(s) +.. function:: quote(s, *, force=False) Return a shell-escaped version of the string *s*. The returned value is a string that can safely be used as one token in a shell command line, for cases where you cannot use a list. + If *force* is :const:`True`, then *s* is unconditionally quoted, + even if it is already safe for a shell without being quoted. + .. _shlex-quote-warning: .. warning:: @@ -91,8 +94,23 @@ The :mod:`!shlex` module defines the following functions: >>> command ['ls', '-l', 'somefile; rm -rf ~'] + The *force* keyword can be used to produce consistent behavior when + escaping multiple strings: + + >>> from shlex import quote + >>> filenames = ['my first file', 'file2', 'file 3'] + >>> filenames_some_escaped = [quote(f) for f in filenames] + >>> filenames_some_escaped + ["'my first file'", 'file2', "'file 3'"] + >>> filenames_all_escaped = [quote(f, force=True) for f in filenames] + >>> filenames_all_escaped + ["'my first file'", "'file2'", "'file 3'"] + .. versionadded:: 3.3 + .. versionchanged:: next + The *force* keyword was added. + The :mod:`!shlex` module defines the following class: diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index d289ba58c240658..2822e979990c4de 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -64,8 +64,8 @@ Directory and files operations The destination location must be writable; otherwise, an :exc:`OSError` exception will be raised. If *dst* already exists, it will be replaced. - Special files such as character or block devices and pipes cannot be - copied with this function. + Special files such as character or block devices, pipes, and sockets cannot + be copied with this function. If *follow_symlinks* is false and *src* is a symbolic link, a new symbolic link will be created instead of copying the @@ -87,10 +87,13 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. + .. versionchanged:: 3.15 + :exc:`SpecialFileError` is now also raised for sockets and device files. + .. exception:: SpecialFileError This exception is raised when :func:`copyfile` or :func:`copytree` attempt - to copy a named pipe. + to copy a named pipe, socket, or device file. .. versionadded:: 2.7 @@ -385,10 +388,14 @@ Directory and files operations If *dst* already exists but is not a directory, it may be overwritten depending on :func:`os.rename` semantics. - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to the destination using *copy_function* - and then removed. In case of symlinks, a new symlink pointing to the target - of *src* will be created as the destination and *src* will be removed. + :func:`os.rename` is preferably used internally when *src* and the destination are on + the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError` + (e.g. the user has write permission to the destination file but not to its parent + directory), this method falls back to using *copy_function*, in which case + *src* is copied to the destination using *copy_function* and then removed. + + In case of symlinks, a new symlink pointing to the target of *src* will be + created in or as the destination, and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments, *src* and the destination, and will be used to copy *src* to the destination @@ -696,7 +703,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -749,8 +756,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 04895ae4ec524b3..090c4ef3ba03fc2 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -17,7 +17,7 @@ import can be suppressed using the interpreter's :option:`-S` option. Importing this module normally appends site-specific paths to the module search path and adds :ref:`callables `, including :func:`help` to the built-in -namespace. However, Python startup option :option:`-S` blocks this and this module +namespace. However, Python startup option :option:`-S` blocks this, and this module can be safely imported with no automatic modifications to the module search path or additions to the builtins. To explicitly trigger the usual site-specific additions, call the :func:`main` function. @@ -71,40 +71,121 @@ the user site prefixes are also implicitly not searched for site-packages. single: # (hash); comment pair: statement; import -A path configuration file is a file whose name has the form :file:`{name}.pth` -and exists in one of the four directories mentioned above; its contents are -additional items (one per line) to be added to ``sys.path``. Non-existing items -are never added to ``sys.path``, and no check is made that the item refers to a -directory rather than a file. No item is added to ``sys.path`` more than -once. Blank lines and lines beginning with ``#`` are skipped. Lines starting -with ``import`` (followed by space or tab) are executed. +The :mod:`!site` module recognizes two startup configuration files of the form +:file:`{name}.pth` for path configurations, and :file:`{name}.start` for +pre-first-line code execution. Both files can exist in one of the four +directories mentioned above. Within each directory, these files are sorted +alphabetically by filename, then parsed in sorted order. -.. note:: +.. _site-pth-files: - An executable line in a :file:`.pth` file is run at every Python startup, - regardless of whether a particular module is actually going to be used. - Its impact should thus be kept to a minimum. - The primary intended purpose of executable lines is to make the - corresponding module(s) importable - (load 3rd-party import hooks, adjust :envvar:`PATH` etc). - Any other initialization is supposed to be done upon a module's - actual import, if and when it happens. - Limiting a code chunk to a single line is a deliberate measure - to discourage putting anything more complex here. +Path extensions (:file:`.pth` files) +------------------------------------ + +:file:`{name}.pth` contains additional items (one per line) to be appended to +``sys.path``. Items that name non-existing directories are never added to +``sys.path``, and no check is made that the item refers to a directory rather +than a file. No item is added to ``sys.path`` more than once. Blank lines +and lines beginning with ``#`` are skipped. + +For backward compatibility, lines starting with ``import`` (followed by space +or tab) are executed with :func:`exec`. .. versionchanged:: 3.13 + The :file:`.pth` files are now decoded by UTF-8 at first and then by the :term:`locale encoding` if it fails. +.. versionchanged:: 3.15 + + :file:`.pth` file lines starting with ``import`` are deprecated. During + the deprecation period, such lines are still executed (except in the case + below), but a diagnostic message is emitted only when the :option:`-v` flag + is given. + + ``import`` lines in :file:`{name}.pth` are silently ignored when a + :ref:`matching ` :file:`{name}.start` file exists. + + Errors on individual lines no longer abort processing of the rest of the + file. Each error is reported and the remaining lines continue to be + processed. + +.. deprecated-removed:: 3.15 3.20 + + Decoding :file:`{name}.pth` files in any encoding other than ``utf-8-sig`` + is deprecated in Python 3.15, and support for decoding from the locale + encoding will be removed in Python 3.20. + + ``import`` lines in :file:`{name}.pth` files are deprecated and will be + silently ignored in Python 3.18 and 3.19. In Python 3.20 a warning will be + produced for ``import`` lines in :file:`{name}.pth` files. + + +.. _site-start-files: + +Startup entry points (:file:`.start` files) +------------------------------------------- + +.. versionadded:: 3.15 + +A startup entry point file is a file whose name has the form +:file:`{name}.start` and exists in one of the site-packages directories +described above. Each file specifies entry points to be called during +interpreter startup, using the ``pkg.mod:callable`` syntax understood by +:func:`pkgutil.resolve_name`. + +Each non-blank line that does not begin with ``#`` must contain an entry +point reference in the form ``pkg.mod:callable``. The colon and callable +portion are mandatory. Each callable is invoked with no arguments, and +any return value is discarded. + +:file:`.start` files are processed after all :file:`.pth` path extensions +have been applied to :data:`sys.path`, ensuring that paths are available +before any startup code runs. + +Unlike :data:`sys.path` extensions from :file:`.pth` files, duplicate entry +points are **not** de-duplicated --- if an entry point appears more than once, +it will be called more than once. + +If an exception occurs during resolution or invocation of an entry point, +a traceback is printed to :data:`sys.stderr` and processing continues with +the remaining entry points. + +:file:`.start` files must be encoded in UTF-8. + +:pep:`829` defined the original specification for these features. + +.. note:: + + If a :file:`{name}.start` file exists alongside a :file:`{name}.pth` file + with the same base name, any ``import`` lines in the :file:`.pth` file are + ignored in favor of the entry points in the :file:`.start` file. + +.. note:: + + Executable lines (``import`` lines in :file:`{name}.pth` files and + :file:`{name}.start` file entry points) are always run at Python startup + (unless :option:`-S` is given to disable the ``site.py`` module entirely), + regardless of whether a particular module is actually going to be used. + +.. note:: + + :file:`{name}.start` files invoke :func:`pkgutil.resolve_name` with + ``strict=True``, which requires the full ``pkg.mod:callable`` form. + .. index:: single: package triple: path; configuration; file + +Startup file examples +--------------------- + For example, suppose ``sys.prefix`` and ``sys.exec_prefix`` are set to :file:`/usr/local`. The Python X.Y library is then installed in :file:`/usr/local/lib/python{X.Y}`. Suppose this has a subdirectory :file:`/usr/local/lib/python{X.Y}/site-packages` with three -subsubdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path +sub-subdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path configuration files, :file:`foo.pth` and :file:`bar.pth`. Assume :file:`foo.pth` contains the following:: @@ -131,6 +212,45 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. +Let's say that there is also a :file:`foo.start` file containing the +following:: + + # foo package startup code + + foo.submod:initialize + +Now, after ``sys.path`` has been extended as above, and before Python turns +control over to user code, the ``foo.submod`` module is imported and the +``initialize()`` function from that module is called. + + +.. _site-migration-guide: + +Migrating from ``import`` lines in ``.pth`` files to ``.start`` files +--------------------------------------------------------------------- + +If your package currently ships a :file:`{name}.pth` file, you can keep all +``sys.path`` extension lines unchanged. Only ``import`` lines need to be +migrated. + +To migrate, create a callable (taking zero arguments) within an importable +module in your package. Reference it as a ``pkg.mod:callable`` entry point +in a matching :file:`{name}.start` file. Move everything on your ``import`` +line after the first semi-colon into the ``callable()`` function. + +If your package must straddle older Pythons that do not support :pep:`829` +and newer Pythons that do, change the ``import`` lines in your +:file:`{name}.pth` to use the following form: + +.. code-block:: python + + import pkg.mod; pkg.mod.callable() + +Older Pythons will execute these ``import`` lines, while newer Pythons will +ignore them in favor of the :file:`{name}.start` file. After the straddling +period, remove all ``import`` lines from your :file:`.pth` files. + + :mod:`!sitecustomize` --------------------- @@ -236,18 +356,113 @@ Module contents This function used to be called unconditionally. +.. function:: makepath(*paths) + + Join *paths* with :func:`os.path.join`, attempt to make the result + absolute with :func:`os.path.abspath`, and return a 2-tuple containing + the absolute path and its case-normalized form as produced by + :func:`os.path.normcase`. If :func:`os.path.abspath` raises + :exc:`OSError`, the joined path is used unchanged for the + case-normalization step. + + The second element of the returned tuple is the form used throughout the + :mod:`!site` module to compare paths on case-insensitive file systems, and + is what populates the ``known_paths`` sets that prevent duplicate + :data:`sys.path` entries in various APIs within this module. + + +.. class:: StartupState(known_paths=None) + + Instances of this class accumulate interpreter startup configuration data + from one or more site directories. They are the preferred interface for + batching the processing of :file:`.pth` and :file:`.start` files across + multiple site directories, so that every :data:`sys.path` extension is + visible before any startup code runs. + + The optional *known_paths* argument is a set of case-normalized paths + (which can be produced by :func:`makepath`) used to prevent duplicate + :data:`sys.path` entries. When ``None`` (the default), the set is built + from the current :data:`sys.path`. :func:`main` implicitly uses an + instance of this class. + + Typical use: + + .. code-block:: python + + state = site.StartupState() + for sitedir in site_dirs: + state.addsitedir(sitedir) + state.process() + + .. versionadded:: 3.15 + + .. method:: addsitedir(sitedir) + + Read the :file:`.pth` and :file:`.start` files in *sitedir* and + record their :data:`sys.path` extensions, deprecated :file:`.pth` + ``import`` lines, and :file:`.start` entry points on this state. + The recorded data is not applied until :meth:`process` is called. + + .. method:: addusersitepackages() + + Add the per-user site-packages directory, if enabled and if it exists. + The directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: addsitepackages(prefixes=None) + + Add global site-packages directories, computed from *prefixes* or from + the global :data:`PREFIXES` when *prefixes* is ``None``. Each + directory's startup data is accumulated for later processing by + :meth:`process`. + + .. method:: process() + + Apply the accumulated state by first adding the path extensions to + :data:`sys.path`, then executing the :file:`.start` file entry points + and :file:`.pth` file ``import`` lines (:ref:`deprecated + `). + + This method is not idempotent and must not be called more than once + on the same instance. Doing so will apply the accumulated state + more than once, re-running entry points and ``import`` lines. + + .. function:: addsitedir(sitedir, known_paths=None) - Add a directory to sys.path and process its :file:`.pth` files. Typically - used in :mod:`sitecustomize` or :mod:`usercustomize` (see above). + Add a directory to sys.path and parse the :file:`.pth` and :file:`.start` + files found in that directory. Typically used in :mod:`sitecustomize` or + :mod:`usercustomize` (see above). + + The *known_paths* argument is an optional set of case-normalized paths + used to prevent duplicate :data:`sys.path` entries. When ``None`` (the + default), the set is built from the current :data:`sys.path`. + + For batched processing across multiple site directories, build a + :class:`StartupState` explicitly and call :meth:`StartupState.addsitedir` + on it; that defers :file:`.pth` and :file:`.start` processing until a + single :meth:`StartupState.process` call, ensuring every :data:`sys.path` + extension is visible before any startup code runs. + + .. versionchanged:: 3.15 + + Also processes :file:`.start` files. See :ref:`site-start-files`. -.. function:: getsitepackages() +.. function:: getsitepackages(prefixes=None) Return a list containing all global site-packages directories. + For each directory given in *prefixes* (or :data:`PREFIXES` if *prefixes* + is ``None``), this function will compute its site-packages subdirectory + depending on the system environment, and will return a list of full paths, + which are not checked for existence. + .. versionadded:: 3.2 + .. versionchanged:: 3.3 + Added the optional *prefixes* parameter. + .. function:: getuserbase() @@ -308,5 +523,5 @@ value greater than 2 if there is an error. .. seealso:: * :pep:`370` -- Per user site-packages directory + * :pep:`829` -- Startup entry points and the deprecation of import lines in ``.pth`` files * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. - diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 71747d5f515a065..836aa91bb0885b1 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -486,6 +486,7 @@ The AF_* and SOCK_* constants are now :class:`AddressFamily` and .. versionchanged:: 3.15 ``IPV6_HDRINCL`` was added. + Added support for ``SO_PASSRIGHTS`` on Linux platforms when available. .. data:: AF_CAN @@ -841,65 +842,8 @@ Creating sockets The following functions all create :ref:`socket objects `. -.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, - or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or - :const:`CAN_J1939`. - - If *fileno* is specified, the values for *family*, *type*, and *proto* are - auto-detected from the specified file descriptor. Auto-detection can be - overruled by calling the function with explicit *family*, *type*, or *proto* - arguments. This only affects how Python represents e.g. the return value - of :meth:`socket.getpeername` but not the actual OS resource. Unlike - :func:`socket.fromfd`, *fileno* will return the same socket and not a - duplicate. This may help close a detached socket using - :meth:`socket.close`. - - The newly created socket is :ref:`non-inheritable `. - - .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - .. versionchanged:: 3.7 - The CAN_ISOTP protocol was added. - - .. versionchanged:: 3.7 - When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` - bit flags are applied to *type* they are cleared, and - :attr:`socket.type` will not reflect them. They are still passed - to the underlying system ``socket()`` call. Therefore, - - :: - - sock = socket.socket( - socket.AF_INET, - socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - - will still create a non-blocking socket on OSes that support - ``SOCK_NONBLOCK``, but ``sock.type`` will be set to - ``socket.SOCK_STREAM``. - - .. versionchanged:: 3.9 - The CAN_J1939 protocol was added. - - .. versionchanged:: 3.10 - The IPPROTO_MPTCP protocol was added. +The :class:`socket ` class constructor creates a new socket +directly; see :ref:`socket-objects` for its parameters and full description. .. function:: socketpair([family[, type[, proto]]]) @@ -1024,12 +968,6 @@ The following functions all create :ref:`socket objects `. .. versionadded:: 3.3 -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - Other functions ''''''''''''''' @@ -1537,642 +1475,709 @@ The :mod:`!socket` module also offers various network-related services: Socket Objects -------------- -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. +.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. + Create a new socket using the given address family, socket type and protocol + number. The address family should be :const:`AF_INET` (the default), + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, + or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the + default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other + ``SOCK_`` constants. The protocol number is usually zero and may be omitted + or in the case where the address family is :const:`AF_CAN` the protocol + should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or + :const:`CAN_J1939`. + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using + :meth:`socket.close`. -.. method:: socket.accept() + The newly created socket is :ref:`non-inheritable `. - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.3 + The AF_CAN family was added. + The AF_RDS family was added. .. versionchanged:: 3.4 - The socket is now non-inheritable. + The CAN_BCM protocol was added. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.4 + The returned socket is now non-inheritable. + .. versionchanged:: 3.7 + The CAN_ISOTP protocol was added. -.. method:: socket.bind(address) + .. versionchanged:: 3.7 + When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` + bit flags are applied to *type* they are cleared, and + :attr:`socket.type` will not reflect them. They are still passed + to the underlying system ``socket()`` call. Therefore, - Bind the socket to *address*. The socket must not already be bound. The format - of *address* depends on the address family --- see :ref:`socket-addresses`. + :: - .. audit-event:: socket.bind self,address socket.socket.bind + sock = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - .. availability:: not WASI. + will still create a non-blocking socket on OSes that support + ``SOCK_NONBLOCK``, but ``sock.type`` will be set to + ``socket.SOCK_STREAM``. + .. versionchanged:: 3.9 + The CAN_J1939 protocol was added. -.. method:: socket.close() + .. versionchanged:: 3.10 + The IPPROTO_MPTCP protocol was added. - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). + Socket objects have the following methods. Except for + :meth:`~socket.makefile`, these correspond to Unix system calls applicable + to sockets. - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. + .. versionchanged:: 3.2 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`~socket.close`. - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. - .. note:: + .. method:: accept() - :meth:`close` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown` - before :meth:`close`. + Accept a connection. The socket must be bound to an address and listening for + connections. The return value is a pair ``(conn, address)`` where *conn* is a + *new* socket object usable to send and receive data on the connection, and + *address* is the address bound to the socket on the other end of the connection. + The newly created socket is :ref:`non-inheritable `. -.. method:: socket.connect(address) + .. versionchanged:: 3.4 + The socket is now non-inheritable. - Connect to a remote socket at *address*. The format of *address* depends on the - address family --- see :ref:`socket-addresses`. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - If the connection is interrupted by a signal, the method waits until the - connection completes, or raises a :exc:`TimeoutError` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - .. audit-event:: socket.connect self,address socket.socket.connect + .. method:: bind(address) - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. + + .. audit-event:: socket.bind self,address socket.socket.bind + + .. availability:: not WASI. + + + .. method:: close() + + Mark the socket closed. The underlying system resource (e.g. a file + descriptor) is also closed when all file objects from :meth:`makefile` + are closed. Once that happens, all future operations on the socket + object will fail. The remote end will receive no more data (after + queued data is flushed). + + Sockets are automatically closed when they are garbage-collected, but + it is recommended to :meth:`close` them explicitly, or to use a + :keyword:`with` statement around them. + + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`!close` call is made. + + .. note:: + + :meth:`close` releases the resource associated with a connection but + does not necessarily close the connection immediately. If you want + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. + + + .. method:: connect(address) + + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. + + If the connection is interrupted by a signal, the method waits until the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the + signal handler doesn't raise an exception and the socket is blocking or has + a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). + signal (or the exception raised by the signal handler). - .. availability:: not WASI. + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 + The method now waits until the connection completes instead of raising an + :exc:`InterruptedError` exception if the connection is interrupted by a + signal, the signal handler doesn't raise an exception and the socket is + blocking or has a timeout (see the :pep:`475` for the rationale). -.. method:: socket.connect_ex(address) + .. availability:: not WASI. - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - .. audit-event:: socket.connect self,address socket.socket.connect_ex + .. method:: connect_ex(address) - .. availability:: not WASI. + Like ``connect(address)``, but return an error indicator instead of raising an + exception for errors returned by the C-level :c:func:`!connect` call (other + problems, such as "host not found," can still raise exceptions). The error + indicator is ``0`` if the operation succeeded, otherwise the value of the + :c:data:`errno` variable. This is useful to support, for example, asynchronous + connects. -.. method:: socket.detach() + .. audit-event:: socket.connect self,address socket.socket.connect_ex - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. + .. availability:: not WASI. - .. versionadded:: 3.2 + .. method:: detach() + Put the socket object into closed state without actually closing the + underlying file descriptor. The file descriptor is returned, and can + be reused for other purposes. -.. method:: socket.dup() + .. versionadded:: 3.2 - Duplicate the socket. - The newly created socket is :ref:`non-inheritable `. + .. method:: dup() - .. versionchanged:: 3.4 - The socket is now non-inheritable. + Duplicate the socket. - .. availability:: not WASI. + The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The socket is now non-inheritable. -.. method:: socket.fileno() + .. availability:: not WASI. - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. + .. method:: fileno() -.. method:: socket.get_inheritable() + Return the socket's file descriptor (a small integer), or -1 on failure. This + is useful with :func:`select.select`. - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. + Under Windows the small integer returned by this method cannot be used where a + file descriptor can be used (such as :func:`os.fdopen`). Unix does not have + this limitation. - .. versionadded:: 3.4 + .. method:: get_inheritable() + Get the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle: ``True`` if the socket can be inherited in + child processes, ``False`` if it cannot. -.. method:: socket.getpeername() + .. versionadded:: 3.4 - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. The format - of the address returned depends on the address family --- see :ref:`socket-addresses`. - On some systems this function is not supported. + .. method:: getpeername() -.. method:: socket.getsockname() + Return the remote address to which the socket is connected. This is useful to + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. The format of the address returned depends on - the address family --- see :ref:`socket-addresses`. + .. method:: getsockname() -.. method:: socket.getsockopt(level, optname[, buflen]) + Return the socket's own address. This is useful to find out the port number of + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - .. availability:: not WASI. + .. method:: getsockopt(level, optname[, buflen]) + Return the value of the given socket option (see the Unix man page + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) + are defined in this module. If *buflen* is absent, an integer option is assumed + and its integer value is returned by the function. If *buflen* is present, it + specifies the maximum length of the buffer used to receive the option in, and + this buffer is returned as a bytes object. It is up to the caller to decode the + contents of the buffer (see the optional built-in module :mod:`struct` for a way + to decode C structures encoded as byte strings). -.. method:: socket.getblocking() + .. availability:: not WASI. - Return ``True`` if socket is in blocking mode, ``False`` if in - non-blocking. - This is equivalent to checking ``socket.gettimeout() != 0``. + .. method:: getblocking() - .. versionadded:: 3.7 + Return ``True`` if socket is in blocking mode, ``False`` if in + non-blocking. + This is equivalent to checking ``socket.gettimeout() != 0``. -.. method:: socket.gettimeout() + .. versionadded:: 3.7 - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. + .. method:: gettimeout() -.. method:: socket.ioctl(control, option) + Return the timeout in seconds (float) associated with socket operations, + or ``None`` if no timeout is set. This reflects the last call to + :meth:`setblocking` or :meth:`settimeout`. - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. + .. method:: ioctl(control, option) - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + The :meth:`ioctl` method is a limited interface to the WSAIoctl system + interface. Please refer to the `Win32 documentation + `_ for more + information. - .. availability:: Windows + On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` + functions may be used; they accept a socket object as their first argument. - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. + Currently only the following control codes are supported: + ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows -.. method:: socket.listen([backlog]) + .. versionchanged:: 3.6 + ``SIO_LOOPBACK_FAST_PATH`` was added. - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - .. availability:: not WASI. + .. method:: listen([backlog]) - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. + Enable a server to accept connections. If *backlog* is specified, it must + be at least 0 (if it is lower, it is set to 0); it specifies the number of + unaccepted connections that the system will allow before refusing new + connections. If not specified, a default reasonable value is chosen. + .. availability:: not WASI. -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) + .. versionchanged:: 3.5 + The *backlog* parameter is now optional. - .. index:: single: I/O control; buffering - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or - a combination of those. + .. method:: makefile(mode='r', buffering=None, *, encoding=None, \ + errors=None, newline=None) - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. + .. index:: single: I/O control; buffering - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. + Return a :term:`file object` associated with the socket. The exact returned + type depends on the arguments given to :meth:`makefile`. These arguments are + interpreted the same way as by the built-in :func:`open` function, except + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. - .. note:: + The socket must be in blocking mode; it can have a timeout, but the file + object's internal buffer may end up in an inconsistent state if a timeout + occurs. - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. + Closing the file object returned by :meth:`makefile` won't close the + original socket unless all other file objects have been closed and + :meth:`socket.close` has been called on the socket object. + .. note:: -.. method:: socket.recv(bufsize[, flags]) + On Windows, the file-like object created by :meth:`makefile` cannot be + used where a file object with a file descriptor is expected, such as the + stream arguments of :meth:`subprocess.Popen`. - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. A returned empty bytes object indicates that the client has disconnected. - See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument - *flags*; it defaults to zero. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recv(bufsize[, flags]) + Receive data from the socket. The return value is a bytes object representing the + data received. The maximum amount of data to be received at once is specified + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. -.. method:: socket.recvfrom(bufsize[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. The format of *address* depends on the address family --- see - :ref:`socket-addresses`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recvfrom(bufsize[, flags]) - .. versionchanged:: 3.7 - For multicast IPv6 address, first item of *address* does not contain - ``%scope_id`` part anymore. In order to get full IPv6 address use - :func:`getnameinfo`. - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:expr:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: - # Append data, ignoring any truncated integers at the end. - fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - .. availability:: Unix. + Receive data from the socket. The return value is a pair ``(bytes, address)`` + where *bytes* is a bytes object representing the data received and *address* is the + address of the socket sending the data. See the Unix manual page + :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. - Most Unix platforms. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. versionchanged:: 3.7 + For multicast IPv6 address, first item of *address* does not contain + ``%scope_id`` part anymore. In order to get full IPv6 address use + :func:`getnameinfo`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - .. availability:: Unix. + .. method:: recvmsg(bufsize[, ancbufsize[, flags]]) + + Receive normal data (up to *bufsize* bytes) and ancillary data from + the socket. The *ancbufsize* argument sets the size in bytes of + the internal buffer used to receive the ancillary data; it defaults + to 0, meaning that no ancillary data will be received. Appropriate + buffer sizes for ancillary data can be calculated using + :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit + into the buffer might be truncated or discarded. The *flags* + argument defaults to 0 and has the same meaning as for + :meth:`recv`. + + The return value is a 4-tuple: ``(data, ancdata, msg_flags, + address)``. The *data* item is a :class:`bytes` object holding the + non-ancillary data received. The *ancdata* item is a list of zero + or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing + the ancillary data (control messages) received: *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + :class:`bytes` object holding the associated data. The *msg_flags* + item is the bitwise OR of various flags indicating conditions on + the received message; see your system documentation for details. + If the receiving socket is unconnected, *address* is the address of + the sending socket, if available; otherwise, its value is + unspecified. + + On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to + pass file descriptors between processes over an :const:`AF_UNIX` + socket. When this facility is used (it is often restricted to + :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its + ancillary data, items of the form ``(socket.SOL_SOCKET, + socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object + representing the new file descriptors as a binary array of the + native C :c:expr:`int` type. If :meth:`recvmsg` raises an + exception after the system call returns, it will first attempt to + close any file descriptors received via this mechanism. + + Some systems do not indicate the truncated length of ancillary data + items which have been only partially received. If an item appears + to extend beyond the end of the buffer, :meth:`recvmsg` will issue + a :exc:`RuntimeWarning`, and will return the part of it which is + inside the buffer provided it has not been truncated before the + start of its associated data. + + On systems which support the :const:`!SCM_RIGHTS` mechanism, the + following function will receive up to *maxfds* file descriptors, + returning the message data and a list containing the descriptors + (while ignoring unexpected conditions such as unrelated control + messages being received). See also :meth:`sendmsg`. :: + + import socket, array + + def recv_fds(sock, msglen, maxfds): + fds = array.array("i") # Array of ints + msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: + # Append data, ignoring any truncated integers at the end. + fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + return msg, list(fds) + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + + + .. method:: recvmsg_into(buffers[, ancbufsize[, flags]]) + + Receive normal data and ancillary data from the socket, behaving as + :meth:`recvmsg` would, but scatter the non-ancillary data into a + series of buffers instead of returning a new bytes object. The + *buffers* argument must be an iterable of objects that export + writable buffers (e.g. :class:`bytearray` objects); these will be + filled with successive chunks of the non-ancillary data until it + has all been written or there are no more buffers. The operating + system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) + on the number of buffers that can be used. The *ancbufsize* and + *flags* arguments have the same meaning as for :meth:`recvmsg`. + + The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, + address)``, where *nbytes* is the total number of bytes of + non-ancillary data written into the buffers, and *ancdata*, + *msg_flags* and *address* are the same as for :meth:`recvmsg`. + + Example:: + + >>> import socket + >>> s1, s2 = socket.socketpair() + >>> b1 = bytearray(b'----') + >>> b2 = bytearray(b'0123456789') + >>> b3 = bytearray(b'--------------') + >>> s1.send(b'Mary had a little lamb') + 22 + >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) + (22, [], 0, None) + >>> [b1, b2, b3] + [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + + .. method:: recvfrom_into(buffer[, nbytes[, flags]]) + + Receive data from the socket, writing it into *buffer* instead of creating a + new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is + the number of bytes received and *address* is the address of the socket sending + the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. + + + .. method:: recv_into(buffer[, nbytes[, flags]]) + + Receive up to *nbytes* bytes from the socket, storing the data into a buffer + rather than creating a new bytestring. If *nbytes* is not specified (or 0), + receive up to the size available in the given buffer. Returns the number of + bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning + of the optional argument *flags*; it defaults to zero. - Most Unix platforms. - .. versionadded:: 3.3 + .. method:: send(bytes[, flags]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Returns the number of bytes sent. Applications are responsible for checking that + all data has been sent; if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. For further + information on this topic, consult the :ref:`socket-howto`. -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. The format of *address* - depends on the address family --- see :ref:`socket-addresses`. + .. method:: sendall(bytes[, flags]) -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Unlike :meth:`send`, this method continues to send data from *bytes* until + either all data has been sent or an error occurs. ``None`` is returned on + success. On error, an exception is raised, and there is no way to determine how + much data, if any, was successfully sent. - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. + .. versionchanged:: 3.5 + The socket timeout is no longer reset each time data is sent successfully. + The socket timeout is now the maximum total duration to send all data. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). -.. method:: socket.send(bytes[, flags]) - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv`. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. + .. method:: sendto(bytes, address) + sendto(bytes, flags, address) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send data to the socket. The socket should not be connected to a remote socket, + since the destination socket is specified by *address*. The optional *flags* + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. + .. audit-event:: socket.sendto self,address socket.socket.sendto -.. method:: socket.sendall(bytes[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv`. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. + .. method:: sendmsg(buffers[, ancdata[, flags[, address]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send normal and ancillary data to the socket, gathering the + non-ancillary data from a series of buffers and concatenating it + into a single message. The *buffers* argument specifies the + non-ancillary data as an iterable of + :term:`bytes-like objects ` + (e.g. :class:`bytes` objects); the operating system may set a limit + (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers + that can be used. The *ancdata* argument specifies the ancillary + data (control messages) as an iterable of zero or more tuples + ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + bytes-like object holding the associated data. Note that + some systems (in particular, systems without :func:`CMSG_SPACE`) + might support sending only one control message per call. The + *flags* argument defaults to 0 and has the same meaning as for + :meth:`send`. If *address* is supplied and not ``None``, it sets a + destination address for the message. The return value is the + number of bytes of non-ancillary data sent. + The following function sends the list of file descriptors *fds* + over an :const:`AF_UNIX` socket, on systems which support the + :const:`!SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) + import socket, array - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv`. Return the number of - bytes sent. The format of *address* depends on the address family --- see - :ref:`socket-addresses`. + def send_fds(sock, msg, fds): + return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - .. audit-event:: socket.sendto self,address socket.socket.sendto + .. availability:: Unix, not WASI. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) + Most Unix platforms. - .. availability:: Unix, not WASI. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg - Most Unix platforms. + .. versionadded:: 3.3 - .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. method:: sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. + Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) + .. availability:: Linux >= 2.6.38. - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. + .. versionadded:: 3.6 - .. availability:: Linux >= 2.6.38. + .. method:: sendfile(file, offset=0, count=None) - .. versionadded:: 3.6 + Send a file until EOF is reached by using high-performance + :mod:`os.sendfile` and return the total number of bytes which were sent. + *file* must be a regular file object opened in binary mode. If + :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a + regular file :meth:`send` will be used instead. *offset* tells from where to + start reading the file. If specified, *count* is the total number of bytes + to transmit as opposed to sending the file until EOF is reached. File + position is updated on return or also in case of error in which case + :meth:`file.tell() ` can be used to figure out the number of + bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. + Non-blocking sockets are not supported. -.. method:: socket.sendfile(file, offset=0, count=None) + .. versionadded:: 3.5 - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. + .. method:: set_inheritable(inheritable) - .. versionadded:: 3.5 + Set the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle. -.. method:: socket.set_inheritable(inheritable) + .. versionadded:: 3.4 - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - .. versionadded:: 3.4 + .. method:: setblocking(flag) + Set blocking or non-blocking mode of the socket: if *flag* is false, the + socket is set to non-blocking, else to blocking mode. -.. method:: socket.setblocking(flag) + This method is a shorthand for certain :meth:`~socket.settimeout` calls: - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. + * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - This method is a shorthand for certain :meth:`~socket.settimeout` calls: + * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` + .. versionchanged:: 3.7 + The method no longer applies :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - .. versionchanged:: 3.7 - The method no longer applies :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: settimeout(value) + Set a timeout on blocking socket operations. The *value* argument can be a + nonnegative real number expressing seconds, or ``None``. + If a non-zero value is given, subsequent socket operations will raise a + :exc:`timeout` exception if the timeout period *value* has elapsed before + the operation has completed. If zero is given, the socket is put in + non-blocking mode. If ``None`` is given, the socket is put in blocking mode. -.. method:: socket.settimeout(value) + For further information, please consult the :ref:`notes on socket timeouts `. - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative real number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. + .. versionchanged:: 3.7 + The method no longer toggles :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - For further information, please consult the :ref:`notes on socket timeouts `. + .. versionchanged:: 3.15 + Accepts any real number, not only integer or float. - .. versionchanged:: 3.7 - The method no longer toggles :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. - .. versionchanged:: 3.15 - Accepts any real number, not only integer or float. + .. method:: setsockopt(level, optname, value: int | Buffer) + setsockopt(level, optname, None, optlen: int) + .. index:: pair: module; struct -.. method:: socket.setsockopt(level, optname, value: int | Buffer) - socket.setsockopt(level, optname, None, optlen: int) + Set the value of the given socket option (see the Unix manual page + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter + case it is up to the caller to ensure that the bytestring contains the + proper bits (see the optional built-in module :mod:`struct` for a way to + encode C structures as bytestrings). When *value* is set to ``None``, + *optlen* argument is required. It's equivalent to calling :c:func:`!setsockopt` C + function with ``optval=NULL`` and ``optlen=optlen``. - .. index:: pair: module; struct + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this - module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the latter - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to calling :c:func:`setsockopt` C - function with ``optval=NULL`` and ``optlen=optlen``. + .. versionchanged:: 3.6 + setsockopt(level, optname, None, optlen: int) form added. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. availability:: not WASI. - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. - .. availability:: not WASI. + .. method:: shutdown(how) + Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, + further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends + are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are + disallowed. -.. method:: socket.shutdown(how) + .. availability:: not WASI. - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. - .. availability:: not WASI. + .. method:: share(process_id) + Duplicate a socket and prepare it for sharing with a target process. The + target process must be provided with *process_id*. The resulting bytes object + can then be passed to the target process using some form of interprocess + communication and the socket can be recreated there using :func:`fromshare`. + Once this method has been called, it is safe to close the socket since + the operating system has already duplicated it for the target process. -.. method:: socket.share(process_id) + .. availability:: Windows. - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. + .. versionadded:: 3.3 - .. availability:: Windows. - .. versionadded:: 3.3 + Note that there are no methods :meth:`!read` or :meth:`!write`; use + :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. + Socket objects also have these (read-only) attributes that correspond to the + values given to the :class:`~socket.socket` constructor. -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. + .. attribute:: family + The socket family. -.. attribute:: socket.family - The socket family. + .. attribute:: type + The socket type. -.. attribute:: socket.type - The socket type. + .. attribute:: proto + The socket protocol. -.. attribute:: socket.proto - The socket protocol. +.. class:: SocketType + The base class of the :class:`~socket.socket` type, re-exported from + :mod:`!_socket`. An instance check such as + ``isinstance(socket(...), SocketType)`` is true, but ``SocketType`` is not + the same as ``type(socket(...))``, which is :class:`~socket.socket` itself. .. _socket-timeouts: diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ff1676c5bfb5619..5aa3d2b0adcbf6e 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -55,7 +55,7 @@ This document includes four main sections: PEP written by Marc-André Lemburg. -.. We use the following practises for SQL code: +.. We use the following practices for SQL code: - UPPERCASE for keywords - snake_case for schema - single quotes for string literals @@ -1417,6 +1417,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1426,6 +1429,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: next + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1709,6 +1715,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: next + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break @@ -1751,6 +1760,10 @@ Blob objects .. versionadded:: 3.11 + .. versionchanged:: next + :class:`Blob` now supports negative-step slices + (e.g. ``blob[9:0:-2]``) for both reading and writing. + A :class:`Blob` instance is a :term:`file-like object` that can read and write data in an SQLite :abbr:`BLOB (Binary Large OBject)`. Call :func:`len(blob) ` to get the size (number of bytes) of the blob. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index f2c35d1897a77f8..66fe6c7aee48626 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -146,9 +146,9 @@ purposes. *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load default CA certificates. - When :attr:`~SSLContext.keylog_filename` is supported and the environment - variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` - enables key logging. + When the environment variable :envvar:`!SSLKEYLOGFILE` is set, + :func:`create_default_context` enables key logging by setting + :attr:`~SSLContext.keylog_filename` to the variable's value. The default settings for this context include :data:`VERIFY_X509_PARTIAL_CHAIN` and :data:`VERIFY_X509_STRICT`. @@ -1121,7 +1121,7 @@ SSL sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -2076,7 +2076,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2879,11 +2879,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index cbb131855dc664b..dba0e26787d9516 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -713,7 +713,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: covariance(x, y, /) - Return the sample covariance of two inputs *x* and *y*. Covariance + Return the sample covariance of two sequence inputs *x* and *y*. Covariance is a measure of the joint variability of two inputs. Both inputs must be of the same length (no less than two), otherwise @@ -739,7 +739,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the `Pearson's correlation coefficient `_ - for two inputs. Pearson's correlation coefficient *r* takes values + for two sequence inputs. Pearson's correlation coefficient *r* takes values between -1 and +1. It measures the strength and direction of a linear relationship. @@ -802,7 +802,7 @@ However, for reading convenience, most of the examples show sorted sequences. (it is equal to the difference between predicted and actual values of the dependent variable). - Both inputs must be of the same length (no less than two), and + Both inputs must be sequences of the same length (no less than two), and the independent variable *x* cannot be constant; otherwise a :exc:`StatisticsError` is raised. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index a10551975edb731..ec37d29ef283ad0 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -706,7 +706,7 @@ A hexadecimal string takes the form:: [sign] ['0x'] integer ['.' fraction] ['p' exponent] -where the optional ``sign`` may by either ``+`` or ``-``, ``integer`` +where the optional ``sign`` may be either ``+`` or ``-``, ``integer`` and ``fraction`` are strings of hexadecimal digits, and ``exponent`` is a decimal integer with an optional leading sign. Case is not significant, and there must be at least one hexadecimal digit in @@ -891,10 +891,12 @@ many numeric contexts, ``False`` and ``True`` behave like the integers 0 and 1, However, relying on this is discouraged; explicitly convert using :func:`int` instead. +.. _iterator-types: + .. _typeiter: -Iterator Types -============== +Iteration-related types +======================= .. index:: single: iterator protocol @@ -907,6 +909,9 @@ using two distinct methods; these are used to allow user-defined classes to support iteration. Sequences, described below in more detail, always support the iteration methods. +Iterables +--------- + One method needs to be defined for container objects to provide :term:`iterable` support: @@ -923,10 +928,14 @@ support: :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C API. +.. _stdtypes-iterators: + +Iterators +--------- + The iterator objects themselves are required to support the following two methods, which together form the :dfn:`iterator protocol`: - .. method:: iterator.__iter__() Return the :term:`iterator` object itself. This is required to allow both @@ -955,16 +964,278 @@ Implementations that do not obey this property are deemed broken. .. _generator-types: -Generator Types +Generator types --------------- -Python's :term:`generator`\s provide a convenient way to implement the iterator -protocol. If a container object's :meth:`~object.__iter__` method is implemented as a -generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` -methods. -More information about generators can be found in :ref:`the documentation for -the yield expression `. +Python's :term:`generators ` -- or more precisely, +:term:`generator functions ` and +:term:`generator iterators ` -- provide a convenient way +to implement the iterator protocol. + +A function that contains one or more :ref:`yield expressions ` +is a :term:`generator function`. +For example:: + + >>> def count_to_three(): + ... yield 0 + ... yield 1 + ... yield 2 + ... yield 3 + +Generator functions behave as regular +:ref:`user-defined functions ` +(for example, they have the same attributes), except that calling a generator +function returns a :ref:`generator iterator `:: + + >>> count_to_three() + + +Iterating a generator iterator executes code of the underlying +generator function, producing each :keyword:`yield`\ed value in turn:: + + >>> for number in count_to_three(): + ... print(number) + 0 + 1 + 2 + 3 + + >>> list(count_to_three()) + [0, 1, 2, 3] + +One common use for generator functions is implementing the +:meth:`~object.__iter__` method of custom iterable objects. +For example:: + + >>> class CardDeck: + ... def __iter__(self): + ... yield 'three of clubs' + ... yield 'ace of hearts' + + >>> list(CardDeck()) + ['three of clubs', 'ace of hearts'] + + +.. index:: pair: object; generator +.. _generator-methods: + +Generator iterators +^^^^^^^^^^^^^^^^^^^ + +Generator iterators implement the +:ref:`iterator protocol `. +Iterating them drives execution of the underlying generator function. + +.. index:: pair: exception; StopIteration + +.. method:: generator.__next__() + + Starts the execution of a generator function or resumes it at the + :ref:`yield expression ` where the function is currently suspended. + When a generator function is resumed with a :meth:`~generator.__next__` + method, the current yield expression always evaluates to :const:`None`. + The execution then continues to the next yield expression, where the + generator is suspended again, and the value of the expression after the + :keyword:`yield` keyword is returned to :meth:`~generator.__next__`'s + caller. + If the generator exits without yielding another value, + :meth:`~generator.__next__` raises a :exc:`StopIteration` exception, + signalling that iteration has completed. + + This method is normally called implicitly, for example by a :keyword:`for` + loop, or by the built-in :func:`next` function. + +Generator iterators have a few more methods than generic iterators, which +can be used to control the execution of the underlying generator function: + +.. method:: generator.send(value) + + "Sends" a value into the generator function: the *value* argument becomes + the result of the current yield expression. + + Otherwise, this method behaves like :meth:`~generator.__next__`: it resumes + the underlying function and either returns the next yielded value or raises + :exc:`StopIteration`. + + When :meth:`send` is called to start the generator, it must be called + with :const:`None` as the argument, because there is no current yield + expression that could receive the value. + + +.. method:: generator.throw(value) + generator.throw(type[, value[, traceback]]) + + Raises an exception at the point where the generator is currently suspended. + + Otherwise, this method behaves like :meth:`~generator.__next__`: it resumes + the underlying function and either returns the next yielded value or raises + :exc:`StopIteration`. + If the generator function does not catch the passed-in exception, or + raises a different exception, then that exception propagates to the caller. + + When :meth:`throw` is called to start the generator, the generator + immediately exits (that is, subsequent calls to :meth:`~generator.__next__` + will raise :exc:`StopIteration`) and the thrown exception is propagated to + :meth:`throw`'s caller. + + In typical use, this is called with a single argument, an exception instance, + similar to the way the :keyword:`raise` keyword is used. + + For backwards compatibility, however, the second signature is + supported, following a convention from older versions of Python. + The *type* argument should be an exception class, and *value* + should be an exception instance. If the *value* is not provided, the + *type* constructor is called to get an instance. If *traceback* + is provided, it is set on the exception, otherwise any existing + :attr:`~BaseException.__traceback__` attribute stored in *value* may + be cleared. + + .. versionchanged:: 3.12 + + The second signature \(type\[, value\[, traceback\]\]\) is deprecated and + may be removed in a future version of Python. + +.. index:: pair: exception; GeneratorExit + +.. method:: generator.close() + + Raises a :exc:`GeneratorExit` exception at the point where the generator + function is currently suspended (equivalent to calling ``throw(GeneratorExit)``). + + If the generator function has already exited (due to an exception or + normal return), or raises :exc:`GeneratorExit` (by not catching the + exception), :meth:`close` returns :const:`None`. + If the generator yields a value, a :exc:`RuntimeError` is raised. + If the generator raises any other exception, it is propagated to the caller. + If a generator returns a value upon being closed, that value is returned + by :meth:`close`. + + When a generator iterator is garbage collected before it has exited, + :meth:`~generator.close` is called automatically. + + .. versionchanged:: 3.13 + + If a generator returns a value upon being closed, the value is returned + by :meth:`close`. + Previously, it returned ``None``. + + +Calling any of the generator methods (:meth:`~generator.__next__`, +:meth:`~generator.send`, :meth:`~generator.throw`, :meth:`~generator.close`) +while one of these methods is already executing +raises a :exc:`ValueError` exception. + + +.. index:: pair: object; asynchronous-generator +.. _asynchronous-generator-methods: + +Asynchronous generator iterators +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This subsection describes the methods of an asynchronous generator iterator, +which are used to control the execution of an asynchronous generator function. + + +.. index:: pair: exception; StopAsyncIteration + +.. method:: agen.__anext__() + :async: + + Returns an :term:`awaitable` which when run starts to execute the + asynchronous generator function or resumes it at the + :ref:`yield expression ` where the function is currently suspended. + When an asynchronous generator function is resumed with an + :meth:`~agen.__anext__` method, the current yield expression always + evaluates to :const:`None` in the returned awaitable, which when run will + continue to the next yield expression. + The value of the expression after the :keyword:`yield` keyword is the value + of the :exc:`StopIteration` exception raised by the completing coroutine. + If the asynchronous generator exits without yielding another value, the + awaitable instead raises a :exc:`StopAsyncIteration` exception, + signalling that the asynchronous iteration has completed. + + This method is normally called implicitly by an :keyword:`async for` loop, + or by the built-in :func:`anext` function. + + +Asynchronous generator-iterators have a few more methods than generic +asynchronous iterators, which can be used to control the execution of +the underlying generator function: + +.. method:: agen.asend(value) + :async: + + Returns an awaitable which, when run, "sends" a value into the underlying + asynchronous generator function: the *value* argument becomes + the result of the current yield expression. + + Otherwise, this method behaves like :meth:`~agen.__anext__`: when the + returned awaitable runs, it resumes the underlying function and either + returns the next yielded value as the value of the raised + :exc:`StopIteration`, or raises :exc:`StopAsyncIteration`. + + When :meth:`asend` is called to start the asynchronous + generator, it must be called with :const:`None` as the argument, + because there is no yield expression that could receive the value. + + +.. method:: agen.athrow(value) + agen.athrow(type[, value[, traceback]]) + :async: + + Returns an awaitable that, when run, raises an exception at the point where + the underlying asynchronous generator function is currently suspended. + + Otherwise, this method behaves like :meth:`~agen.__anext__`: when the + returned awaitable runs, it resumes the underlying function (with an + exception raised) and either returns the next yielded value as the value of + the raised :exc:`StopIteration`, or raises :exc:`StopAsyncIteration`. + If the underlying function does not catch the passed-in exception, or + raises a different exception, then when the awaitable is run, that + exception propagates to the caller of the awaitable. + + When :meth:`~agen.athrow` is called to start the generator, the generator + exits when the awaitable runs (that is, subsequent results from + :meth:`~agen.__anext__` will raise :exc:`StopAsyncIteration` when run) + and the thrown exception is propagated to the awaitable's caller. + + In typical use, this is called with a single argument, an exception instance, + similar to the way the :keyword:`raise` keyword is used. + + For backwards compatibility, however, the second signature is + supported. + An exception instance is created from three arguments in the same way as in + :meth:`generator.throw`. + + .. versionchanged:: 3.12 + + The second signature \(type\[, value\[, traceback\]\]\) is deprecated and + may be removed in a future version of Python. + + +.. index:: pair: exception; GeneratorExit + +.. method:: agen.aclose() + :async: + + Returns an awaitable that when run will throw a :exc:`GeneratorExit` into + the underlying asynchronous generator function at the point where it is + currently suspended (equivalent to calling ``athrow(GeneratorExit)``). + + If the asynchronous generator function then exits gracefully, is already + closed, or raises :exc:`GeneratorExit` (by not catching the exception), + then the returned awaitable will raise a :exc:`StopIteration` exception. + Any further awaitables returned by subsequent calls to the asynchronous + generator will raise a :exc:`StopAsyncIteration` exception. + + If the asynchronous generator yields a value, a :exc:`RuntimeError` is + raised by the awaitable. + If the asynchronous generator raises any other exception, that exception + is propagated to the caller of the awaitable. + + If the asynchronous generator has already exited due to an exception or + normal exit, then further calls to :meth:`aclose` will return an awaitable + that does nothing. .. _typesseq: @@ -999,6 +1270,9 @@ The ``in`` and ``not in`` operations have the same priorities as the comparison operations. The ``+`` (concatenation) and ``*`` (repetition) operations have the same priority as the corresponding numeric operations. [3]_ +See :ref:`time-complexity` for the costs of the various sequence +operations. + .. index:: triple: operations on; sequence; types pair: built-in function; len @@ -1121,6 +1395,8 @@ Notes: "end" values (which end depends on the sign of *k*). Note, *k* cannot be zero. If *k* is ``None``, it is treated like ``1``. +.. _typesseq-repeated-concatenation: + (6) Concatenating immutable sequences always results in a new object. This means that building up a sequence by repeated concatenation will have a @@ -1345,7 +1621,7 @@ Mutable sequence types also support the following methods: :no-typesetting: .. method:: sequence.pop(index=-1, /) - Retrieve the item at *index* and also removes it from *sequence*. + Retrieve the item at *index* and also remove it from *sequence*. By default, the last item in *sequence* is removed and returned. .. method:: bytearray.remove(value, /) @@ -1403,6 +1679,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1494,6 +1772,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -1714,7 +1996,11 @@ category. | | :meth:`str.strip` | :meth:`bytes.strip` | | +--------------------+----------------------+----------------------+----------------------------+ | | :meth:`str.lstrip` | :meth:`str.rstrip` | :meth:`bytes.lstrip` | :meth:`bytes.rstrip` | -+--------------------------+--------------------+----------------------+----------------------+----------------------------+ +| +--------------------+----------------------+----------------------+----------------------------+ +| | :meth:`str.removeprefix` | :meth:`bytes.removeprefix` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.removesuffix` | :meth:`bytes.removesuffix` | ++--------------------------+-------------------------------------------+---------------------------------------------------+ | Translation and Encoding | :meth:`str.translate` | :meth:`bytes.translate` | | +-------------------------------------------+---------------------------------------------------+ | | :meth:`str.maketrans` | :meth:`bytes.maketrans` | @@ -2110,7 +2396,7 @@ expression support in the :mod:`re` module). one character, ``False`` otherwise. Alphabetic characters are those characters defined in the Unicode character database as "Letter", i.e., those with general category property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different - from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and + from the `Alphabetic property defined in section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard `__. For example: @@ -2168,9 +2454,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2211,15 +2513,14 @@ expression support in the :mod:`re` module). >>> '0123456789'.isnumeric() True - >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-indic digit zero to nine + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine True >>> '⅕'.isnumeric() # Vulgar fraction one fifth True >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() (False, True, True) - See also :meth:`isdecimal` and :meth:`isdigit`. Numeric characters are - a superset of decimal numbers. + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() @@ -2370,7 +2671,8 @@ expression support in the :mod:`re` module). Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* argument is not a prefix; rather, all combinations of its values are stripped:: >>> ' spacious '.lstrip() @@ -2564,7 +2866,8 @@ expression support in the :mod:`re` module). Return a list of the words in the string, using *sep* as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* - ones. If *sep* is not specified or ``None``, any whitespace string is a + ones. If *sep* is not specified or ``None``, any + :meth:`whitespace ` string is a separator. Except for splitting from the right, :meth:`rsplit` behaves like :meth:`split` which is described in detail below. @@ -2573,7 +2876,8 @@ expression support in the :mod:`re` module). Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* + or ``None``, the *chars* argument defaults to removing whitespace, that is + characters for which :meth:`str.isspace` is true. The *chars* argument is not a suffix; rather, all combinations of its values are stripped. For example: @@ -2610,7 +2914,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2622,13 +2928,16 @@ expression support in the :mod:`re` module). ['1', '2', '3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm is - applied: runs of consecutive whitespace are regarded as a single separator, + applied: runs of consecutive :meth:`whitespace ` are regarded + as a single separator, and the result will contain no empty strings at the start or end if the string has leading or trailing whitespace. Consequently, splitting an empty string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2640,7 +2949,9 @@ expression support in the :mod:`re` module). If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only leading runs of consecutive whitespace are considered. - For example:: + For example: + + .. doctest:: >>> "".split(None, 0) [] @@ -2649,7 +2960,7 @@ expression support in the :mod:`re` module). >>> " foo ".split(maxsplit=0) ['foo '] - See also :meth:`join`. + See also :meth:`join` and :meth:`rsplit`. .. index:: @@ -2743,9 +3054,9 @@ expression support in the :mod:`re` module). Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. - If omitted or ``None``, the *chars* argument defaults to removing whitespace. - The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped. + If omitted or ``None``, the *chars* argument defaults to removing whitespace, + that is characters for which :meth:`str.isspace` is true. The *chars* argument + is not a prefix or suffix; rather, all combinations of its values are stripped. For example: @@ -2775,8 +3086,22 @@ expression support in the :mod:`re` module). .. method:: str.swapcase() Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. + vice versa. For example: + + .. doctest:: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that ``s.swapcase().swapcase() == s``. + For example: + + .. doctest:: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also :meth:`str.lower` and :meth:`str.upper`. .. method:: str.title() @@ -2995,7 +3320,7 @@ replacement field. For example:: '0.333333' >>> f'{one_third:_^+10}' '___+1/3___' - >>> >>> f'{one_third!r:_^20}' + >>> f'{one_third!r:_^20}' '___Fraction(1, 3)___' >>> f'{one_third = :~>10}~' 'one_third = ~~~~~~~1/3~' @@ -3005,12 +3330,12 @@ replacement field. For example:: Template String Literals (t-strings) ------------------------------------ -An :dfn:`t-string` (formally a :dfn:`template string literal`) is +A :dfn:`t-string` (formally a :dfn:`template string literal`) is a string literal that is prefixed with ``t`` or ``T``. These strings follow the same syntax and evaluation rules as :ref:`formatted string literals `, -with for the following differences: +with the following differences: * Rather than evaluating to a ``str`` object, template string literals evaluate to a :class:`string.templatelib.Template` object. @@ -3037,7 +3362,7 @@ with for the following differences: The :class:`!Interpolation` instance for the expression will be created as normal, except that :attr:`~string.templatelib.Interpolation.conversion` will be set to '``r``' (:func:`repr`) by default. - If an explicit conversion or format specifier are provided, + If an explicit conversion or format specifier is provided, this will override the default behaviour. @@ -3414,7 +3739,7 @@ objects. .. classmethod:: fromhex(string, /) - This :class:`bytearray` class method returns bytearray object, decoding + This :class:`bytearray` class method returns a bytearray object, decoding the given string object. The string must contain two hexadecimal digits per byte, with ASCII whitespace being ignored. @@ -3868,7 +4193,8 @@ produce new objects. Return a copy of the sequence with specified leading bytes removed. The *bytes* argument is a binary sequence specifying the set of byte values to be removed. If omitted or ``None``, the *bytes* argument defaults - to removing ASCII whitespace. The *bytes* argument is not a prefix; + to removing :meth:`ASCII whitespace `. + The *bytes* argument is not a prefix; rather, all combinations of its values are stripped:: >>> b' spacious '.lstrip() @@ -3912,7 +4238,8 @@ produce new objects. Split the binary sequence into subsequences of the same type, using *sep* as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* ones. If *sep* is not specified or ``None``, - any subsequence consisting solely of ASCII whitespace is a separator. + any subsequence consisting solely of + :meth:`ASCII whitespace ` is a separator. Except for splitting from the right, :meth:`rsplit` behaves like :meth:`split` which is described in detail below. @@ -3923,7 +4250,8 @@ produce new objects. Return a copy of the sequence with specified trailing bytes removed. The *bytes* argument is a binary sequence specifying the set of byte values to be removed. If omitted or ``None``, the *bytes* argument defaults to - removing ASCII whitespace. The *bytes* argument is not a suffix; rather, + removing :meth:`ASCII whitespace `. + The *bytes* argument is not a suffix; rather, all combinations of its values are stripped:: >>> b' spacious '.rstrip() @@ -3976,7 +4304,8 @@ produce new objects. [b'1', b'2', b'3<4'] If *sep* is not specified or is ``None``, a different splitting algorithm - is applied: runs of consecutive ASCII whitespace are regarded as a single + is applied: runs of consecutive :meth:`ASCII whitespace ` + are regarded as a single separator, and the result will contain no empty strings at the start or end if the sequence has leading or trailing whitespace. Consequently, splitting an empty sequence or a sequence consisting solely of ASCII @@ -3999,7 +4328,8 @@ produce new objects. Return a copy of the sequence with specified leading and trailing bytes removed. The *bytes* argument is a binary sequence specifying the set of byte values to be removed. If omitted or ``None``, the *bytes* - argument defaults to removing ASCII whitespace. The *bytes* argument is + argument defaults to removing :meth:`ASCII whitespace `. + The *bytes* argument is not a prefix or suffix; rather, all combinations of its values are stripped:: @@ -4373,7 +4703,7 @@ the ``%`` operator (modulo). This is also known as the bytes *formatting* or *interpolation* operator. Given ``format % values`` (where *format* is a bytes object), ``%`` conversion specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. +The effect is similar to using the :c:func:`sprintf` function in the C language. If *format* requires a single argument, *values* may be a single non-tuple object. [5]_ Otherwise, *values* must be a tuple with exactly the number of @@ -4570,8 +4900,11 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. + :class:`!memoryview`\s are :ref:`generic ` over the type of their + underlying data. + ``len(view)`` is equal to the length of :meth:`~memoryview.tolist`, which - is the nested list representation of the view. If ``view.ndim = 1``, + is the nested list representation of the view. If ``view.ndim == 1``, this is equal to the number of elements in the view. .. versionchanged:: 3.12 @@ -4656,7 +4989,7 @@ copying. :class:`collections.abc.Sequence` .. versionchanged:: 3.5 - memoryviews can now be indexed with tuple of integers. + memoryviews can now be indexed with a tuple of integers. .. versionchanged:: 3.14 memoryview is now a :term:`generic type`. @@ -4821,13 +5154,20 @@ copying. .. versionadded:: 3.2 .. method:: cast(format, /) - cast(format, shape, /) + cast(format, shape, /, *, order='C') Cast a memoryview to a new format or shape. *shape* defaults to ``[byte_length//new_itemsize]``, which means that the result view will be one-dimensional. The return value is a new memoryview, but - the buffer itself is not copied. Supported casts are 1D -> C-:term:`contiguous` - and C-contiguous -> 1D. + the buffer itself is not copied. Supported casts are + 1D -> C-:term:`contiguous`, C-contiguous -> 1D, and + F-contiguous -> 1D. + + With a multidimensional *shape*, *order* selects the memory layout of + the result: ``'C'`` for C-contiguous (row-major, the default) or ``'F'`` + for Fortran-contiguous (column-major). The buffer is still not copied, + so ``order='F'`` gives a zero-copy view over a buffer holding + column-major data. The destination format is restricted to a single element native format in :mod:`struct` syntax. One of the formats must be a byte format @@ -4914,6 +5254,10 @@ copying. .. versionchanged:: 3.5 The source format is no longer restricted when casting to a byte view. + .. versionchanged:: next + Casting a multi-dimensional F-contiguous view to a one-dimensional + view is now supported. + .. method:: count(value, /) Count the number of occurrences of *value*. @@ -4976,8 +5320,20 @@ copying. >>> y.nbytes 96 + Interpret a flat buffer as a Fortran-contiguous (column-major) array:: + + >>> buf = bytes(range(6)) + >>> y = memoryview(buf).cast('B', shape=[3, 2], order='F') + >>> y.f_contiguous + True + >>> y.tolist() + [[0, 3], [1, 4], [2, 5]] + .. versionadded:: 3.3 + .. versionchanged:: next + Added the *order* parameter. + .. attribute:: readonly A bool indicating whether the memory is read only. @@ -5066,6 +5422,7 @@ computing mathematical operations such as intersection, union, difference, and symmetric difference. (For other containers see the built-in :class:`dict`, :class:`list`, and :class:`tuple` classes, and the :mod:`collections` module.) +See :ref:`time-complexity` for the costs of the various set operations. Like other collections, sets support ``x in set``, ``len(set)``, and ``for x in set``. Being an unordered collection, sets do not record element position or @@ -5263,6 +5620,8 @@ Note, the *elem* argument to the :meth:`~object.__contains__`, :meth:`~set.discard` methods may be a set. To support searching for an equivalent frozenset, a temporary one is created from *elem*. +Sets and frozensets are :ref:`generic ` over the type of their elements. + .. seealso:: For detailed information on thread-safety guarantees for :class:`set` @@ -5288,6 +5647,8 @@ There are currently two standard mapping types, the :dfn:`dictionary` and (For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, and the :mod:`collections` module.) +See :ref:`time-complexity` for the costs of the various dictionary +operations. A dictionary's keys are *almost* arbitrary values. Values that are not :term:`hashable`, that is, values containing lists, dictionaries or other @@ -5366,6 +5727,9 @@ can be used interchangeably to index the same dictionary entry. Dictionary order is guaranteed to be insertion order. This behavior was an implementation detail of CPython from 3.6. + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -5703,6 +6067,16 @@ Frozen dictionaries :class:`!frozendict` is not a :class:`!dict` subclass but inherits directly from ``object``. + Like dictionaries, frozendicts are :ref:`generic ` over two types, + signifying (respectively) the types of the frozendict's keys and values. + + .. classmethod:: fromkeys(iterable, value=None, /) + + Similar to :meth:`dict.fromkeys`, but call again the type constructor + with an initialized :class:`frozendict` if the type is a + :class:`frozendict` subclass or if the constructor returned a + :class:`frozendict`. + .. versionadded:: 3.15 @@ -5844,7 +6218,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5893,6 +6268,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) @@ -6077,7 +6461,7 @@ enables cleaner type hinting syntax compared to subscripting :class:`typing.Unio .. note:: - The ``|`` operand cannot be used at runtime to define unions where one or + The ``|`` operator cannot be used at runtime to define unions where one or more members is a forward reference. For example, ``int | "Foo"``, where ``"Foo"`` is a reference to a class not yet defined, will fail at runtime. For unions which include forward references, present the @@ -6236,7 +6620,7 @@ Methods Methods are functions that are called using the attribute notation. There are two flavors: :ref:`built-in methods ` (such as :meth:`~list.append` on lists) -and :ref:`class instance method `. +and :ref:`class instance methods `. Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 08ccdfa3f454f8d..f1abed91d20ed51 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -472,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -518,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -603,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | @@ -959,7 +971,8 @@ attributes: Alternatively, you can provide the entire regular expression pattern by overriding the class attribute *pattern*. If you do this, the value must be a -regular expression object with four named capturing groups. The capturing +regular expression pattern string, or a compiled regular expression +object, with four named capturing groups. The capturing groups correspond to the rules given above, along with the invalid placeholder rule: diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst index a5b2d796aaf4b83..6e91850fdf59ca4 100644 --- a/Doc/library/string.templatelib.rst +++ b/Doc/library/string.templatelib.rst @@ -245,6 +245,8 @@ Types ... 3.0 | 1. + 2. | None | .2f + Interpolations are :ref:`generic ` over the types of their values. + .. rubric:: Attributes .. attribute:: value diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index fa0fb19d852f862..07e4c82375fc1ee 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -68,6 +68,7 @@ The module defines the following exception and functions: Pack the values *v1*, *v2*, ... according to the format string *format* and write the packed bytes into the writable buffer *buffer* starting at position *offset*. Note that *offset* is a required argument. + A negative *offset* counts from the end of *buffer*. .. function:: unpack(format, buffer) @@ -84,6 +85,7 @@ The module defines the following exception and functions: string *format*. The result is a tuple even if it contains exactly one item. The buffer's size in bytes, starting at position *offset*, must be at least the size required by the format, as reflected by :func:`calcsize`. + A negative *offset* counts from the end of *buffer*. .. function:: iter_unpack(format, buffer) @@ -111,7 +113,7 @@ Format Strings -------------- Format strings describe the data layout when -packing and unpacking data. They are built up from :ref:`format characters`, +packing and unpacking data. They are built up from :ref:`type codes `, which specify the type of data being packed/unpacked. In addition, special characters control the :ref:`byte order, size and alignment`. Each format string consists of an optional prefix character which @@ -183,8 +185,8 @@ Use :data:`sys.byteorder` to check the endianness of your system. Native size and alignment are determined using the C compiler's ``sizeof`` expression. This is always combined with native byte order. -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. +Standard size depends only on the type code; see the table in +the :ref:`type-codes` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. @@ -208,12 +210,13 @@ Notes: count of zero. See :ref:`struct-examples`. +.. _type-codes: .. _format-characters: -Format Characters -^^^^^^^^^^^^^^^^^ +Type Codes +^^^^^^^^^^ -Format characters have the following meaning; the conversion between C and +Type codes (or format codes) have the following meaning; the conversion between C and Python values should be obvious given their types. The 'Standard size' column refers to the size of the packed value in bytes when using standard size; that is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or @@ -227,32 +230,32 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ | ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ @@ -260,15 +263,15 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``d`` | :c:expr:`double` | float | 8 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ -| ``F`` | :c:expr:`float complex` | complex | 8 | \(10) | +| ``Zf`` | :c:expr:`float complex` | complex | 8 | \(10) | +--------+--------------------------+--------------------+----------------+------------+ -| ``D`` | :c:expr:`double complex` | complex | 16 | \(10) | +| ``Zd`` | :c:expr:`double complex` | complex | 16 | \(10) | +--------+--------------------------+--------------------+----------------+------------+ | ``s`` | :c:expr:`char[]` | bytes | | \(9) | +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -280,6 +283,12 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. +.. versionchanged:: 3.15 + Added support for the ``'Zf'`` and ``'Zd'`` formats. + +.. versionchanged:: 3.16 + ``'F'`` and ``'D'`` formats are deprecated. + .. seealso:: The :mod:`array` and :ref:`ctypes ` modules, @@ -317,7 +326,7 @@ Notes: format used by the platform. (5) - The ``'P'`` format character is only available for the native byte ordering + The ``'P'`` type code is only available for the native byte ordering (selected as the default or with the ``'@'`` byte order character). The byte order character ``'='`` chooses to use little- or big-endian ordering based on the host system. The struct module does not interpret this as native @@ -339,42 +348,48 @@ Notes: When packing, ``'x'`` inserts one NUL byte. (8) - The ``'p'`` format character encodes a "Pascal string", meaning a short + The ``'p'`` type code encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all - are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + are used. Note that for :func:`unpack`, the ``'p'`` type code consumes + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) - For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + For the ``'s'`` type code, the count is interpreted as the length of the + byte string, not a repeat count like for the other type codes; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (10) - For the ``'F'`` and ``'D'`` format characters, the packed representation uses + For the ``'Zf'`` and ``'Zd'`` type codes, the packed representation uses the IEEE 754 binary32 and binary64 format for components of the complex number, regardless of the floating-point format used by the platform. - Note that complex types (``F`` and ``D``) are available unconditionally, + Note that complex types are available unconditionally, despite complex types being an optional feature in C. As specified in the C11 standard, each complex type is represented by a two-element C array containing, respectively, the real and imaginary parts. + The ``'F'`` and ``'D'`` (for ``'Zf'`` and ``'Zd'``, respectively) format + characters are supported for compatibility. -A format character may be preceded by an integral repeat count. For example, +A type code may be preceded by an integral repeat count. For example, the format string ``'4h'`` means exactly the same as ``'hhhh'``. Whitespace characters between formats are ignored; a count and its format must @@ -391,7 +406,7 @@ then :exc:`struct.error` is raised. .. index:: single: ? (question mark); in struct format strings -For the ``'?'`` format character, the return value is either :const:`True` or +For the ``'?'`` type code, the return value is either :const:`True` or :const:`False`. When packing, the truth value of the argument object is used. Either 0 or 1 in the native or standard bool representation will be packed, and any non-zero value will be ``True`` when unpacking. @@ -446,7 +461,7 @@ the result in a named tuple:: >>> Student._make(unpack('<10sHHb', record)) Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) -The ordering of format characters may have an impact on size in native +The ordering of type codes may have an impact on size in native mode since padding is implicit. In standard mode, the user is responsible for inserting any desired padding. Note in @@ -504,7 +519,7 @@ When constructing format strings which mimic native layouts, the compiler and machine architecture determine byte ordering and padding. In such cases, the ``@`` format character should be used to specify native byte ordering and data sizes. Internal pad bytes are normally inserted -automatically. It is possible that a zero-repeat format code will be +automatically. It is possible that a zero-repeat type code will be needed at the end of a format string to round up to the correct byte boundary for proper alignment of consecutive chunks of data. @@ -523,7 +538,7 @@ code solves that problem:: >>> calcsize('@llh0l') 24 -The ``'x'`` format code can be used to specify the repeat, but for +The ``'x'`` type code can be used to specify the repeat, but for native formats it is better to use a zero-repeat format like ``'0l'``. By default, native byte ordering and alignment is used, but it is diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index fe64daa3291d670..2a31213560c92d3 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -236,8 +236,8 @@ underlying :class:`Popen` interface can be used directly. .. attribute:: returncode - Exit status of the child process. If the process exited due to a - signal, this will be the negative signal number. + Exit status of the child process, an integer. If the process + exited due to a signal, this will be the negative signal number. .. attribute:: cmd diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 52a722608db4312..859687340882de9 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -20,6 +20,8 @@ Generating Symbol Tables .. function:: symtable(code, filename, compile_type, *, module=None) Return the toplevel :class:`SymbolTable` for the Python source *code*. + *code* can be a string, a bytes object, or an AST object, + as for the builtin :func:`compile`. *filename* is the name of the file containing the code. *compile_type* is like the *mode* argument to :func:`compile`. The optional argument *module* specifies the module name. @@ -29,6 +31,9 @@ Generating Symbol Tables .. versionadded:: 3.15 Added the *module* parameter. + .. versionchanged:: next + *code* can now be an AST object. + Examining Symbol Tables ----------------------- @@ -187,57 +192,6 @@ Examining Symbol Tables A namespace of a class. This class inherits from :class:`SymbolTable`. - .. method:: get_methods() - - Return a tuple containing the names of method-like functions declared - in the class. - - Here, the term 'method' designates *any* function defined in the class - body via :keyword:`def` or :keyword:`async def`. - - Functions defined in a deeper scope (e.g., in an inner class) are not - picked up by :meth:`get_methods`. - - For example: - - .. testsetup:: symtable.Class.get_methods - - import warnings - context = warnings.catch_warnings() - context.__enter__() - warnings.simplefilter("ignore", category=DeprecationWarning) - - .. testcleanup:: symtable.Class.get_methods - - context.__exit__() - - .. doctest:: symtable.Class.get_methods - - >>> import symtable - >>> st = symtable.symtable(''' - ... def outer(): pass - ... - ... class A: - ... def f(): - ... def w(): pass - ... - ... def g(self): pass - ... - ... @classmethod - ... async def h(cls): pass - ... - ... global outer - ... def outer(self): pass - ... ''', 'test', 'exec') - >>> class_A = st.get_children()[2] - >>> class_A.get_methods() - ('f', 'g', 'h') - - Although ``A().f()`` raises :exc:`TypeError` at runtime, ``A.f`` is still - considered as a method-like function. - - .. deprecated-removed:: 3.14 3.16 - .. class:: Symbol diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 4a460479e4afd74..7cca6f2bcdae912 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -11,9 +11,9 @@ .. note:: :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, - not an independent module, so there is no need to - ``import sys.monitoring``, simply ``import sys`` and then use - ``sys.monitoring``. + not an independent module, and ``import sys.monitoring`` would fail + with a :exc:`ModuleNotFoundError`. Instead, simply ``import sys`` + and then use ``sys.monitoring``. This namespace provides access to the functions and constants necessary to @@ -180,8 +180,8 @@ Local events '''''''''''' Local events are associated with normal execution of the program and happen -at clearly defined locations. All local events can be disabled. -The local events are: +at clearly defined locations. All local events can be disabled +per location. The local events are: * :monitoring-event:`PY_START` * :monitoring-event:`PY_RESUME` @@ -205,6 +205,8 @@ Using :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` events will give much better performance as they can be disabled independently. +.. _monitoring-ancillary-events: + Ancillary events '''''''''''''''' @@ -226,7 +228,7 @@ Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled via :data:`DISABLE`. +program and cannot be individually disabled per location. The other events that can be monitored are: @@ -234,6 +236,12 @@ The other events that can be monitored are: * :monitoring-event:`PY_UNWIND` * :monitoring-event:`RAISE` * :monitoring-event:`EXCEPTION_HANDLED` +* :monitoring-event:`RERAISE` + +.. versionchanged:: 3.15 + Other events can now be turned on and disabled on a per code object + basis. Returning :data:`DISABLE` from a callback disables the event + for the entire code object (for the current tool). The STOP_ITERATION event @@ -247,8 +255,7 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike -:monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled. Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are @@ -314,15 +321,14 @@ location by returning :data:`sys.monitoring.DISABLE` from a callback function. This does not change which events are set, or any other code locations for the same event. -Disabling events for specific locations is very important for high -performance monitoring. For example, a program can be run under a -debugger with no overhead if the debugger disables all monitoring -except for a few breakpoints. +:ref:`Other events ` can be disabled on a per code +object basis by returning :data:`sys.monitoring.DISABLE` from a callback +function. This disables the event for the entire code object (for the current +tool). -If :data:`DISABLE` is returned by a callback for a -:ref:`global event `, :exc:`ValueError` will be raised -by the interpreter in a non-specific location (that is, no traceback will be -provided). +Disabling events for specific locations is very important for high performance +monitoring. For example, a program can be run under a debugger with no overhead +if the debugger disables all monitoring except for a few breakpoints. .. function:: restart_events() -> None diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 6946eb6eeaa5fae..bcc8f855c06f71d 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -233,15 +233,15 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. function:: _clear_type_cache() - Clear the internal type cache. The type cache is used to speed up attribute - and method lookups. Use the function *only* to drop unnecessary references - during reference leak debugging. - - This function should be used for internal and specialized purposes only. + This function is a no-op. It used to clear the internal type cache, which + is now implemented per-type. .. deprecated:: 3.13 Use the more general :func:`_clear_internal_caches` function instead. + .. versionchanged:: 3.16 + This function is now a no-op. + .. function:: _clear_internal_caches() @@ -250,6 +250,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.13 + .. versionchanged:: 3.16 + The type cache is no longer cleared, as it is now implemented per-type. + .. function:: _current_frames() @@ -879,7 +882,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionchanged:: 3.6 Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` - and :func:`_enablelegacywindowsfsencoding` for more information. + for more information. .. versionchanged:: 3.7 Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is @@ -919,8 +922,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy * ``"all"``: All top-level imports are potentially lazy - * ``"none"``: All lazy imports are suppressed (even explicitly marked - ones) See also :func:`set_lazy_imports` and :pep:`810`. @@ -1483,6 +1484,21 @@ always available. Unless explicitly noted otherwise, all variables are read-only They hold the legacy representation of ``sys.last_exc``, as returned from :func:`exc_info` above. + +.. data:: lazy_modules + + A :class:`set` of fully qualified module name strings that have been lazily + imported in the current interpreter but not yet loaded. When a + lazily imported module is accessed for the first time, its name is removed + from this set. + + This attribute is intended for debugging and introspection. + + See also :func:`set_lazy_imports` and :pep:`810`. + + .. versionadded:: 3.15 + + .. data:: maxsize An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can @@ -1757,8 +1773,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only * ``"normal"``: Only imports explicitly marked with the ``lazy`` keyword are lazy * ``"all"``: All top-level imports become potentially lazy - * ``"none"``: All lazy imports are suppressed (even explicitly marked - ones) This function is intended for advanced users who need to control lazy imports across their entire application. Library developers should @@ -1779,13 +1793,13 @@ always available. Unless explicitly noted otherwise, all variables are read-only callable or ``None`` to clear the filter. The filter function is called for every potentially lazy import to - determine whether it should actually be lazy. It must have the following + determine whether it should actually be lazy. It should have the following signature:: def filter(importing_module: str, imported_module: str, fromlist: tuple[str, ...] | None) -> bool - Where: + The function is called with three positional arguments: * *importing_module* is the name of the module doing the import * *imported_module* is the resolved name of the module being imported @@ -1930,7 +1944,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only The interpreter is about to execute a new line of code or re-execute the condition of a loop. The local trace function is called; *arg* is ``None``; the return value specifies the new local trace function. See - :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this + :source:`InternalDocs/code_objects.md` for a detailed explanation of how this works. Per-line events may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to :const:`False` on that @@ -2107,31 +2121,6 @@ always available. Unless explicitly noted otherwise, all variables are read-only See :pep:`768` for more details. -.. function:: _enablelegacywindowsfsencoding() - - Changes the :term:`filesystem encoding and error handler` to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to - 3.6. - - This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` - environment variable before launching Python. - - See also :func:`sys.getfilesystemencoding` and - :func:`sys.getfilesystemencodeerrors`. - - .. availability:: Windows. - - .. note:: - Changing the filesystem encoding after Python startup is risky because - the old fsencoding or paths encoded by the old fsencoding may be cached - somewhere. Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - - .. versionadded:: 3.6 - See :pep:`529` for more details. - - .. deprecated-removed:: 3.13 3.16 - Use :envvar:`PYTHONLEGACYWINDOWSFSENCODING` instead. - .. data:: stdin stdout stderr @@ -2149,7 +2138,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only returned by the :func:`open` function. Their parameters are chosen as follows: - * The encoding and error handling are is initialized from + * The encoding and error handling are initialized from :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character @@ -2254,8 +2243,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only The name of the lock implementation: - * ``"semaphore"``: a lock uses a semaphore - * ``"mutex+cond"``: a lock uses a mutex and a condition variable + * ``"semaphore"``: a lock uses a semaphore (Python 3.14 and older) + * ``"mutex+cond"``: a lock uses a mutex and a condition variable (Python 3.14 and older) + * ``"pymutex"``: a lock uses the :c:type:`PyMutex` implementation (Python 3.15 and newer) * ``None`` if this information is unknown .. attribute:: thread_info.version diff --git a/Doc/library/sys_path_init.rst b/Doc/library/sys_path_init.rst index e6c2cddbe842487..329e49253e91aa4 100644 --- a/Doc/library/sys_path_init.rst +++ b/Doc/library/sys_path_init.rst @@ -105,6 +105,8 @@ Please refer to :mod:`site`'s mechanism, such as :mod:`venv`, that many virtual environment implementations follow. +.. _sys-path-init-_pth-files: + _pth files ---------- diff --git a/Doc/library/tachyon-logo.png b/Doc/library/tachyon-logo.png deleted file mode 100644 index bf0901ec9f313e0..000000000000000 Binary files a/Doc/library/tachyon-logo.png and /dev/null differ diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index a86469bb9ad7041..0d8ce6e497915fd 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -142,6 +142,12 @@ Some facts and figures: a Zstandard dictionary used to improve compression of smaller amounts of data. + For modes ``'w:gz'`` and ``'w|gz'``, :func:`tarfile.open` accepts the + keyword argument *mtime* to create a gzip archive header with that mtime. By + default, the mtime is set to the time of creation of the archive. Use + *mtime* ``0`` to generate a compressed stream that does not depend on + creation time, for reproducible output. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -664,7 +670,7 @@ be finalized; only the internally used file object will be closed. See the it is best practice to only do so in top-level applications or :mod:`site configuration `. To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod` to prevent injection of a ``self`` argument. + :deco:`staticmethod` to prevent injection of a ``self`` argument. .. versionchanged:: 3.14 @@ -957,6 +963,11 @@ A ``TarInfo`` object has the following public data attributes: If *deep* is false, the copy is shallow, i.e. ``pax_headers`` and any custom attributes are shared with the original ``TarInfo`` object. + This method is also used by :func:`copy.replace`. + + .. versionchanged:: next + Added support for :func:`copy.replace`. + A :class:`TarInfo` object also provides some convenient query methods: @@ -1101,6 +1112,10 @@ reused in custom filters: paths (in case the name is absolute even after stripping slashes, e.g. ``C:/foo`` on Windows). This raises :class:`~tarfile.AbsolutePathError`. + - Normalize filenames (:attr:`TarInfo.name`) that contain ``..`` components + using :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the name if it traverses symbolic links. - :ref:`Refuse ` to extract files whose absolute path (after following symlinks) would end up outside the destination. This raises :class:`~tarfile.OutsideDestinationError`. @@ -1109,6 +1124,10 @@ reused in custom filters: Return the modified ``TarInfo`` member. + .. versionchanged:: next + + Filenames containing ``..`` components are now normalized. + .. function:: data_filter(member, path) Implements the ``'data'`` filter. @@ -1189,6 +1208,8 @@ Here is an incomplete list of things to consider: * Check filenames against an allow-list of characters (to filter out control characters, confusables, foreign path separators, and so on). +* Check for platform-specific filename semantics. For example, on Windows + some names can have reserved meanings. * Check that filenames have expected extensions (discouraging files that execute when you “click on them”, or extension-less files like Windows special device names). diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index bf9198e175a0e11..316d82775cf626e 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -14,10 +14,11 @@ This module creates temporary files and directories. It works on all supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -:term:`context managers `. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. +:class:`TemporaryFileWrapper`, :class:`TemporaryDirectory`, and +:class:`SpooledTemporaryFile` are high-level interfaces which provide +automatic cleanup and can be used as :term:`context managers +`. :func:`mkstemp` and :func:`mkdtemp` are lower-level +functions which require manual cleanup. All the user-callable functions and constructors take additional arguments which allow direct control over the location and name of temporary files and @@ -84,13 +85,13 @@ The module defines the following user-callable items: :func:`TemporaryFile` with *delete* and *delete_on_close* parameters that determine whether and how the named file should be automatically deleted. - The returned object is always a :term:`file-like object` whose :attr:`!file` - attribute is the underlying true file object. This file-like object - can be used in a :keyword:`with` statement, just like a normal file. The - name of the temporary file can be retrieved from the :attr:`!name` attribute - of the returned file-like object. On Unix, unlike with the - :func:`TemporaryFile`, the directory entry does not get unlinked immediately - after the file creation. + The returned object is always a :class:`TemporaryFileWrapper` instance + (a :term:`file-like object`) whose :attr:`~TemporaryFileWrapper.file` attribute is the underlying + true file object. This file-like object can be used in a :keyword:`with` + statement, just like a normal file. The name of the temporary file can be + retrieved from the :attr:`!name` attribute of the returned file-like object. + On Unix, unlike with the :func:`TemporaryFile`, the directory entry does not + get unlinked immediately after the file creation. If *delete* is true (the default) and *delete_on_close* is true (the default), the file is deleted as soon as it is closed. If *delete* is true @@ -140,6 +141,34 @@ The module defines the following user-callable items: .. versionchanged:: 3.12 Added *delete_on_close* parameter. +.. class:: TemporaryFileWrapper(file, name, delete=True, delete_on_close=True) + + A mutable wrapper returned by :func:`NamedTemporaryFile`. It wraps the + underlying file object, delegating attribute access to it, and ensures + the temporary file is deleted when appropriate. + + .. attribute:: file + + The underlying :term:`file-like object`. + + .. attribute:: name + + The file name of the temporary file. + + .. method:: close() + + Close the temporary file, possibly deleting it depending on the + *delete* and *delete_on_close* arguments passed to + :func:`NamedTemporaryFile`. + + .. note:: + + ``tempfile._TemporaryFileWrapper`` is kept as a backwards compatible + deprecated alias for this class. + It will be removed in Python 3.21 + + .. versionadded:: next + .. class:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) @@ -228,7 +257,8 @@ The module defines the following user-callable items: The file descriptor is :ref:`not inherited by child processes `. Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible - for deleting the temporary file when done with it. + for closing the file descriptor (for example, using :func:`os.close`) and + deleting the temporary file (for example, using :func:`os.remove`). If *suffix* is not ``None``, the file name will end with that suffix, otherwise there will be no suffix. :func:`mkstemp` does not put a dot @@ -318,8 +348,11 @@ The module defines the following user-callable items: #. A platform-specific location: - * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, - :file:`\\TEMP`, and :file:`\\TMP`, in that order. + * On Windows, the directories + :file:`%USERPROFILE%\\AppData\\Local\\Temp`, + :file:`%SYSTEMROOT%\\Temp`, :file:`C:\\TEMP`, + :file:`C:\\TMP`, :file:`\\TEMP`, and + :file:`\\TMP`, in that order. * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and :file:`/usr/tmp`, in that order. diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 7ae3fabf1cec644..765811eba32fd40 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -232,9 +232,10 @@ The :mod:`!test.support` module defines the following constants: .. data:: verbose - ``True`` when verbose output is enabled. Should be checked when more - detailed information is desired about a running test. *verbose* is set by - :mod:`test.regrtest`. + How verbose the output is: the number of :option:`!-v` options which + :mod:`test.regrtest` was run with, and therefore ``0`` when verbose output + is not enabled. Should be checked when more detailed information is + desired about a running test. .. data:: is_jython @@ -285,7 +286,7 @@ The :mod:`!test.support` module defines the following constants: :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` methods of :class:`socket.socket`. - Its default value is 5 seconds. + Its default value is 10 seconds. See also :data:`INTERNET_TIMEOUT`. @@ -718,7 +719,7 @@ The :mod:`!test.support` module defines the following functions: .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with - :func:`unittest.expectedFailure`. Any use of this decorator should + :deco:`unittest.expectedFailure`. Any use of this decorator should have an associated comment identifying the relevant tracker issue. @@ -961,6 +962,72 @@ The :mod:`!test.support` module defines the following functions: :mod:`tracemalloc` is enabled. +.. currentmodule:: test.support.isolation + +.. decorator:: runInSubprocess(*, options=(), env=None, timeout=None) + + Decorator that runs the decorated test in a fresh interpreter subprocess, in + isolation, so that it does not share global or interpreter state with the + rest of the test run. It can decorate a test method or a whole + :class:`~unittest.TestCase` subclass. Decorated methods must take no extra + arguments. A failure, error or skip in the subprocess is reported for the + corresponding test, and individual :meth:`subtests + ` that fail or are skipped are reported + individually. A reported failure or error shows the original subprocess + traceback as the cause of the exception. + + When a **method** is decorated, only that method runs in a subprocess; all + fixtures (:meth:`~unittest.TestCase.setUp` / :meth:`~unittest.TestCase.tearDown`, + :meth:`~unittest.TestCase.setUpClass` / :meth:`~unittest.TestCase.tearDownClass` + and ``setUpModule()`` / ``tearDownModule()``) run both in the parent process + (as usual) and in the subprocess around the method. + + When a **class** is decorated, the whole class runs in a single subprocess, + and :meth:`~unittest.TestCase.setUpClass`, + :meth:`~unittest.TestCase.tearDownClass`, :meth:`~unittest.TestCase.setUp` + and :meth:`~unittest.TestCase.tearDown` run once each in the subprocess and + are skipped in the parent process. A failure or skip of + :meth:`~unittest.TestCase.setUpClass` in the subprocess is reported for the + whole class. ``setUpModule()`` cannot be controlled by a class decorator, + so it still runs in the parent process too; test it with + :data:`runningInSubprocess` if needed. + + The subprocess inherits the enabled resources (``-u``), memory limit + (``-M``) and verbosity (``-v``) of the parent test run, so that + :func:`~test.support.requires_resource`, :func:`~test.support.requires`, + :func:`~test.support.bigmemtest` and the like behave consistently in both + processes. + + *options* is a sequence of interpreter command line options + to run the subprocess with, + and *env* is a mapping of environment variables to set in it, + on top of the inherited environment. + A value of ``None`` in *env* unsets the variable. + Note that :option:`-E` and :option:`-I` make the subprocess ignore + the ``PYTHON*`` environment variables, including :envvar:`PYTHONPATH`. + + *timeout* is the number of seconds to wait for the subprocess; + the test is reported as an error if it does not complete in time. + By default there is no timeout, + and a hung test is left to the timeout of the test runner. + + The test is skipped on platforms without subprocess support. + + +.. data:: runningInSubprocess + + ``True`` while the code runs in the isolated subprocess spawned by + :func:`runInSubprocess`, and ``False`` otherwise (including in the parent + process and in a normal, non-isolated test run). Fixtures such as + :meth:`~unittest.TestCase.setUp`, :meth:`~unittest.TestCase.tearDown`, + :meth:`~unittest.TestCase.setUpClass`, :meth:`~unittest.TestCase.tearDownClass`, + ``setUpModule()`` and ``tearDownModule()`` can test it to choose which code + to run in the subprocess. + + +.. currentmodule:: test.support + + .. function:: check_free_after_iterating(test, iter, cls, args=()) Assert instances of *cls* are deallocated after iterating. @@ -1590,6 +1657,32 @@ The :mod:`!test.support.os_helper` module provides support for os tests. wrapped with a wait loop that checks for the existence of the file. +.. decorator:: with_source_date_epoch(*, epoch=123456789) + + A decorator for running tests with the :envvar:`SOURCE_DATE_EPOCH` + environment variable set to *epoch*. + + +.. decorator:: without_source_date_epoch + + A decorator for running tests with the :envvar:`SOURCE_DATE_EPOCH` + environment variable unset. + + +.. class:: SourceDateEpochTestMeta + + Metaclass wrapping all test methods of the class with + :func:`with_source_date_epoch` if the *source_date_epoch* keyword class + argument is true, or with :func:`without_source_date_epoch` otherwise. + For example:: + + class TestsWithSourceEpoch(Tests, + metaclass=SourceDateEpochTestMeta, + source_date_epoch=True): + pass + + + :mod:`!test.support.import_helper` --- Utilities for import tests ================================================================= diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 19cc4f191dff8d9..5d9a7b6314b1668 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -515,7 +515,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. @@ -1436,3 +1436,159 @@ is equivalent to:: Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, :class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as :keyword:`with` statement context managers. + + +Iterator synchronization +------------------------ + +By default, Python iterators do not support concurrent access. Most iterators make +no guarantees when accessed simultaneously from multiple threads. Generator +iterators, for example, raise :exc:`ValueError` if one of their iterator methods +is called while the generator is already executing. The tools in this section +allow reliable concurrency support to be added to ordinary iterators and +iterator-producing callables. + +The :class:`serialize_iterator` wrapper lets multiple threads share a single iterator and +take turns consuming from it. While one thread is running ``__next__()``, the +others block until the iterator becomes available. Each value produced by the +underlying iterator is delivered to exactly one caller. + +The :func:`concurrent_tee` function lets multiple threads each receive the full +stream of values from one underlying iterator. It creates independent iterators +that all draw from the same source. Values are buffered until consumed by all +of the derived iterators. + +.. class:: serialize_iterator(iterable) + + Return an iterator wrapper that serializes concurrent calls to + :meth:`~iterator.__next__` using a lock. + + If the wrapped iterator also defines :meth:`~generator.send`, + :meth:`~generator.throw`, or :meth:`~generator.close`, those calls + are serialized as well. + + This makes it possible to share a single iterator, including a generator + iterator, between multiple threads. A lock ensures that calls are handled + one at a time. No values are duplicated or skipped by the wrapper itself. + Each item from the underlying iterator is given to exactly one caller. + + This wrapper does not copy or buffer values. Threads that call + :func:`next` while another thread is already advancing the iterator will + block until the active call completes. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = threading.serialize_iterator(squares(5)) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, each number is printed exactly once, but the work is shared + between the two threads. + + .. versionadded:: 3.15 + + +.. function:: synchronized_iterator(func) + + Wrap an iterator-producing callable so that each iterator it returns is + automatically passed through :class:`serialize_iterator`. + + This is especially useful as a :term:`decorator` for generator functions, + allowing their generator-iterators to be consumed from multiple threads. + + Example: + + .. code-block:: python + + import threading + + @threading.synchronized_iterator + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + + t1 = threading.Thread(target=consume, args=("left", source)) + t2 = threading.Thread(target=consume, args=("right", source)) + t1.start() + t2.start() + t1.join() + t2.join() + + The returned wrapper preserves the metadata of *func*, such as its name and + wrapped function reference. + + .. versionadded:: 3.15 + + +.. function:: concurrent_tee(iterable, n=2) + + Return *n* independent iterators from a single input *iterable*, with + guaranteed behavior when the derived iterators are consumed concurrently. + + This function is similar to :func:`itertools.tee`, but is intended for cases + where the source iterator may feed consumers running in different threads. + Each returned iterator yields every value from the underlying iterable, in + the same order. + + Internally, values are buffered until every derived iterator has consumed + them. + + The returned iterators share the same underlying synchronization lock. Each + individual derived iterator is intended to be consumed by one thread at a + time. If a single derived iterator must itself be shared by multiple + threads, wrap it with :class:`serialize_iterator`. + + If *n* is ``0``, return an empty tuple. If *n* is negative, raise + :exc:`ValueError`. + + Example: + + .. code-block:: python + + import threading + + def squares(n): + for x in range(n): + yield x * x + + def consume(name, iterable): + for item in iterable: + print(name, item) + + source = squares(5) + left, right = threading.concurrent_tee(source) + + t1 = threading.Thread(target=consume, args=("left", left)) + t2 = threading.Thread(target=consume, args=("right", right)) + t1.start() + t2.start() + t1.join() + t2.join() + + In this example, both consumer threads see the full sequence of squares + from a single generator expression. + + .. versionadded:: 3.15 diff --git a/Doc/library/time-complexity.rst b/Doc/library/time-complexity.rst new file mode 100644 index 000000000000000..5ce02ac2761b621 --- /dev/null +++ b/Doc/library/time-complexity.rst @@ -0,0 +1,337 @@ +.. _time-complexity: + +=============================================== +Time complexity of operations on built-in types +=============================================== + +This page documents the time complexity of various operations on built-in types +in CPython. Other Python implementations may have different performance +characteristics. Additionally, the listed costs assume exact built-in types, as +instances of subclasses may have different costs. + +We use |big O notation|_ to describe how the running time of an operation grows +with the size of its inputs. Unless stated otherwise, *n* denotes the number of +elements currently in the container, and *k* is the value of a numeric +parameter, such as an index or a repeat count. + +.. |big O notation| replace:: Big *O* notation +.. _big O notation: https://en.wikipedia.org/wiki/Big_O_notation + + +:class:`!list` +============== + +Lists are mutable sequences; for more detail on the implementation see +:ref:`how-are-lists-implemented`. The largest costs come from growing beyond the +current allocation size (because everything must move), or from inserting or +deleting somewhere near the beginning (because everything after that must move). +If you need to add or remove at both ends, consider using a +:class:`collections.deque` instead. + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - Copy (``l.copy()``) + - *O*\ (*n*) + * - Append (``l.append(x)``) [1]_ + - *O*\ (1) + * - Pop (``l.pop(k)``) [1]_ [2]_ + - *O*\ (*n* - *k*) + * - Insert (``l.insert(k, x)``) [1]_ [2]_ + - *O*\ (*n* - *k*) + * - Get item (``l[k]``) + - *O*\ (1) + * - Set item (``l[k] = x``) + - *O*\ (1) + * - Delete item (``del l[k]``) [2]_ + - *O*\ (*n* - *k*) + * - Iteration + - *O*\ (*n*) + * - Get slice (``l[i:j]``) + - *O*\ (*j* - *i*) + * - Set slice (``l[i:j] = t``) [1]_ + - *O*\ (*j* - *i*) if len(*t*) == *j* - *i*, + otherwise *O*\ (*n* - *i* + len(*t*)) + * - Delete slice (``del l[i:j]``) + - *O*\ (*n* - *i*) + * - Extend (``l.extend(t)``) [1]_ [3]_ + - *O*\ (len(*t*)) + * - Sort (``l.sort()``) [4]_ + - *O*\ (*n* log *n*) + * - Concatenate (``l1 + l2``) + - *O*\ (len(*l1*) + len(*l2*)) + * - Multiply (``l * k``) + - *O*\ (*nk*) + * - ``x in l`` + - *O*\ (*n*) + * - ``min(l)``, ``max(l)`` + - *O*\ (*n*) + * - Get length (``len(l)``) [5]_ + - *O*\ (1) + + +:class:`!tuple` +=============== + +A :class:`tuple` is an :term:`immutable` sequence. Because a tuple can never +change, there are no insertion or deletion costs, and making a copy simply +returns the same object, so is constant time (*O*\ (1)). + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - Copy (``tuple(t)``) + - *O*\ (1) + * - Get item (``t[k]``) + - *O*\ (1) + * - Get slice (``t[i:j]``) + - *O*\ (*j* - *i*) + * - Concatenate (``t1 + t2``) + - *O*\ (len(*t1*) + len(*t2*)) + * - Multiply (``t * k``) + - *O*\ (*nk*) + * - Iteration + - *O*\ (*n*) + * - ``x in t`` + - *O*\ (*n*) + * - ``min(t)``, ``max(t)`` + - *O*\ (*n*) + * - Get length (``len(t)``) [5]_ + - *O*\ (1) + + +:class:`!dict`, :class:`!frozendict` +==================================== + +The times listed for dict objects are average-case times, as they assume the +hash function for the objects is sufficiently robust to make collisions +uncommon. They also assume the keys are well-distributed among the set of +possible keys. In the worst case, when every key hashes to the same value, +each of the *O*\ (1) operations below instead takes *O*\ (*n*) time. They also +assume that hashing and comparing a key is *O*\ (1). For more detail on the +implementation, see :ref:`how-are-dictionaries-implemented`. + +A :class:`frozendict` is immutable, so it does not support setting, deleting, +or updating items. The other operations below apply to it at the same costs. + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - ``key in d`` + - *O*\ (1) + * - Copy (``d.copy()``) [6]_ [7]_ + - *O*\ (*n*) + * - Get item (``d[key]``, ``d.get(key)``) + - *O*\ (1) + * - Set item (``d[key] = value``) [1]_ + - *O*\ (1) + * - Delete item (``del d[key]``, ``d.pop(key)``) + - *O*\ (1) + * - Update (``d.update(t)``, ``d |= t``) [1]_ [3]_ [7]_ + - *O*\ (len(*t*)) + * - Iteration [7]_ + - *O*\ (*n*) + * - Get length (``len(d)``) [5]_ + - *O*\ (1) + + +:class:`!set`, :class:`!frozenset` +================================== + +See :class:`dict` as the :class:`set` and :class:`frozenset` implementations are +similar, and the same caveats apply. +In the worst case, *O*\ (1) operations instead take *O*\ (*n*) time, +and operations that look up every element degrade accordingly. + +A :class:`frozenset` is :term:`immutable`, so it does not support adding, +discarding, or the in-place update operations. The others below apply to it at +the same costs. + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - ``x in s`` + - *O*\ (1) + * - Copy (``s.copy()``) [6]_ [7]_ + - *O*\ (*n*) + * - Add (``s.add(x)``) [1]_ + - *O*\ (1) + * - Discard (``s.discard(x)``, ``s.remove(x)``) + - *O*\ (1) + * - Union (``s1 | s2``, ``s1.union(s2)``) [7]_ + - *O*\ (len(*s1*) + len(*s2*)) + * - Update (``s1 |= s2``, ``s1.update(s2)``) [1]_ [7]_ + - *O*\ (len(*s2*)) + * - Intersection (``s1 & s2``, ``s1.intersection(s2)``) [7]_ [8]_ + - *O*\ (min(len(*s1*), len(*s2*))) + * - Intersection update (``s1 &= s2``, ``s1.intersection_update(s2)``) [1]_ [7]_ [8]_ + - *O*\ (min(len(*s1*), len(*s2*))) + * - Difference (``s1 - s2``, ``s1.difference(s2)``) [7]_ [9]_ + - *O*\ (len(*s1*)) + * - Difference update (``s1 -= s2``, ``s1.difference_update(s2)``) [1]_ [7]_ [8]_ + - *O*\ (min(len(*s1*), len(*s2*))) + * - Symmetric difference (``s1 ^ s2``, ``s1.symmetric_difference(s2)``) [7]_ + - *O*\ (len(*s1*) + len(*s2*)) + * - Symmetric difference update (``s1 ^= s2``, ``s1.symmetric_difference_update(s2)``) [1]_ [7]_ + - *O*\ (len(*s2*)) + * - Get length (``len(s)``) [5]_ + - *O*\ (1) + + +:class:`!str`, :class:`!bytes`, :class:`!bytearray` +=================================================== + +:class:`str` and :class:`bytes` objects are immutable sequences of characters and +bytes, respectively. As with tuples, copying one returns the original object. +A :class:`bytearray` is mutable, and additionally supports the mutating operations +of :class:`list` (except :meth:`!sort`), at the same costs. However, deleting at +the front with ``del`` (``del b[0]``, ``del b[:k]``) only advances the start of +the buffer instead of moving the remaining bytes, and is amortized *O*\ (1). + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - Get item (``s[k]``) + - *O*\ (1) + * - Get slice (``s[i:j]``) + - *O*\ (*j* - *i*) + * - Concatenate (``s + t``) [10]_ + - *O*\ (len(*s*) + len(*t*)) + * - Multiply (``s * k``) + - *O*\ (*nk*) + * - Substring search (``x in s``, ``s.find(x)``, ``s.index(x)``) [11]_ + - *O*\ (*n*) + * - Reverse substring search (``s.rfind(x)``, ``s.rindex(x)``) [11]_ [12]_ + - *O*\ (*n* × len(*x*)) + * - Encode or decode [13]_ + - *O*\ (*n*) + * - Iteration + - *O*\ (*n*) + * - Get length (``len(s)``) [5]_ + - *O*\ (1) + + +:class:`!memoryview` +==================== + +:class:`memoryview` objects allow Python code to access the internal data +of an object that supports the :ref:`buffer protocol ` without +copying. In particular, slicing a memory view returns a new view onto the same +buffer. + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - Create (``memoryview(obj)``) + - *O*\ (1) + * - Get item (``v[k]``) + - *O*\ (1) + * - Get slice (``v[i:j]``) + - *O*\ (1) + * - Index (``v.index(x)``) [11]_ [14]_ + - *O*\ (*n*) + * - Count (``v.count(x)``) [14]_ + - *O*\ (*n*) + * - Convert to bytes (``v.tobytes()``, ``bytes(v)``) + - *O*\ (*n*) + * - Get length (``len(v)``) [5]_ + - *O*\ (1) + + +:class:`!range` +=============== + +A :class:`range` object computes its items on demand from its *start*, *stop* and +*step* values, so most operations do not depend on the length of the range. + +.. list-table:: + :header-rows: 1 + + * - Operation + - Complexity + * - Get item (``r[k]``) + - *O*\ (1) + * - Get slice (``r[i:j]``) + - *O*\ (1) + * - ``x in r`` [15]_ + - *O*\ (1) + * - Index and count (``r.index(x)``, ``r.count(x)``) [15]_ + - *O*\ (1) + * - Iteration + - *O*\ (*n*) + * - ``min(r)``, ``max(r)`` + - *O*\ (*n*) + * - Get length (``len(r)``) [5]_ + - *O*\ (1) + + +Notes +===== + +.. [1] Amortized. An individual operation may occasionally be *O*\ (*n*) + when the underlying storage is resized, but this cost is spread over + many operations, depending on the history of the container. + +.. [2] Popping or deleting the element at index *k* of a list of size *n* + shifts all elements after *k* one slot to the left, moving *n* - *k* - 1 + elements; inserting at index *k* shifts the elements from *k* onwards one + slot to the right, moving *n* - *k* elements. The worst case is index 0, + where the whole rest of the list has to be moved; the average case, an + index in the middle of the list, takes *O*\ (*n*/2) = *O*\ (*n*) + operations; and operating at the end of the list moves nothing and is + *O*\ (1). + +.. [3] Plus the cost of iterating over *t*, which may be expensive for an + arbitrary iterable. + +.. [4] This is the worst case scenario. Sorting is adaptive and input that is + already sorted or reverse-sorted takes only *O*\ (*n*) comparisons. + See :source:`Objects/listsort.txt` for more information. + +.. [5] The number of elements is stored in the object, so ``len()`` does + not need to count them. + +.. [6] Copying a :class:`frozendict` or a :class:`frozenset` is *O*\ (1) as it + returns the original object. + +.. [7] These operations scan the container's internal hash table, which is + not shrunk when elements are removed. After removing most elements, they + still take time proportional to the container's former size, until a + later insertion triggers a resize. + +.. [8] *O*\ (len(*t*)) if *t* is not a set. + +.. [9] *O*\ (len(*s*) + len(*t*)) if *t* is not a set. + +.. [10] Each concatenation builds a new object, so building a string by + concatenating many pieces in a loop is quadratic in the total length. + See the :ref:`note on concatenating immutable sequences + ` for alternatives. + +.. [11] With *start* and *end* arguments, *n* is the length of the region + searched rather than of *s*, and unlike slicing nothing is copied. + +.. [12] This is the worst case. Reverse searches are *O*\ (*n*) on typical + input. Forward searches instead use a more elaborate algorithm with a + linear worst case, described in + :source:`Objects/stringlib/stringlib_find_two_way_notes.txt`. + +.. [13] This assumes a codec that does a constant amount of work per character. + +.. [14] These unpack and compare each element individually, so they are much + slower than the equivalent :class:`bytes` methods. + +.. [15] Assuming :class:`int` or :class:`bool` arguments. For other types, + the range is searched like any other sequence in *O*\ (*n*) time. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index a931134331f0a5c..d7ca0628c662ea0 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -122,23 +122,24 @@ An explanation of some terminology and conventions is in order. Functions --------- -.. function:: asctime([t]) +.. function:: asctime([time_tuple]) Convert a tuple or :class:`struct_time` representing a time as returned by :func:`gmtime` or :func:`localtime` to a string of the following form: ``'Sun Jun 20 23:21:05 1993'``. The day field is two characters long and is space padded if the day is a single digit, - e.g.: ``'Wed Jun 9 04:26:40 1993'``. + for example: ``'Wed Jun 9 04:26:40 1993'``. - If *t* is not provided, the current time as returned by :func:`localtime` - is used. Locale information is not used by :func:`asctime`. + If *time_tuple* is not provided, + the current time as returned by :func:`localtime` is used. + Locale information is not used by :func:`asctime`. .. note:: Unlike the C function of the same name, :func:`asctime` does not add a trailing newline. -.. function:: pthread_getcpuclockid(thread_id) +.. function:: pthread_getcpuclockid(thread_id, /) Return the *clk_id* of the thread-specific CPU-time clock for the specified *thread_id*. @@ -157,7 +158,7 @@ Functions .. versionadded:: 3.7 -.. function:: clock_getres(clk_id) +.. function:: clock_getres(clk_id, /) Return the resolution (precision) of the specified clock *clk_id*. Refer to :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. @@ -167,7 +168,7 @@ Functions .. versionadded:: 3.3 -.. function:: clock_gettime(clk_id) -> float +.. function:: clock_gettime(clk_id, /) -> float Return the time of the specified clock *clk_id*. Refer to :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. @@ -180,7 +181,7 @@ Functions .. versionadded:: 3.3 -.. function:: clock_gettime_ns(clk_id) -> int +.. function:: clock_gettime_ns(clk_id, /) -> int Similar to :func:`clock_gettime` but return time as nanoseconds. @@ -189,7 +190,7 @@ Functions .. versionadded:: 3.7 -.. function:: clock_settime(clk_id, time) +.. function:: clock_settime(clk_id, time, /) Set the time of the specified clock *clk_id*. Currently, :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. @@ -205,7 +206,7 @@ Functions Accepts any real number as *time*, not only integer or float. -.. function:: clock_settime_ns(clk_id, time: int) +.. function:: clock_settime_ns(clk_id, time: int, /) Similar to :func:`clock_settime` but set time with nanoseconds. @@ -214,23 +215,23 @@ Functions .. versionadded:: 3.7 -.. function:: ctime([secs]) +.. function:: ctime(seconds=None, /) Convert a time expressed in seconds since the epoch_ to a string of a form: ``'Sun Jun 20 23:21:05 1993'`` representing local time. The day field is two characters long and is space padded if the day is a single digit, - e.g.: ``'Wed Jun 9 04:26:40 1993'``. + for example: ``'Wed Jun 9 04:26:40 1993'``. - If *secs* is not provided or :const:`None`, the current time as - returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to - ``asctime(localtime(secs))``. Locale information is not used by + If *seconds* is not provided or :const:`None`, the current time as + returned by :func:`.time` is used. ``ctime(seconds)`` is equivalent to + ``asctime(localtime(seconds))``. Locale information is not used by :func:`ctime`. .. versionchanged:: 3.15 Accepts any real number, not only integer or float. -.. function:: get_clock_info(name) +.. function:: get_clock_info(name, /) Get information on the specified clock as a namespace object. Supported clock names and the corresponding functions to read their value @@ -255,10 +256,10 @@ Functions .. versionadded:: 3.3 -.. function:: gmtime([secs]) +.. function:: gmtime(seconds=None, /) Convert a time expressed in seconds since the epoch_ to a :class:`struct_time` in - UTC in which the dst flag is always zero. If *secs* is not provided or + UTC in which the dst flag is always zero. If *seconds* is not provided or :const:`None`, the current time as returned by :func:`.time` is used. Fractions of a second are ignored. See above for a description of the :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this @@ -268,11 +269,12 @@ Functions Accepts any real number, not only integer or float. -.. function:: localtime([secs]) +.. function:: localtime(seconds=None, /) - Like :func:`gmtime` but converts to local time. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. The dst - flag is set to ``1`` when DST applies to the given time. + Like :func:`gmtime` but converts to local time. + If *seconds* is not provided or :const:`None`, + the current time as returned by :func:`.time` is used. + The dst flag is set to ``1`` when DST applies to the given time. :func:`localtime` may raise :exc:`OverflowError`, if the timestamp is outside the range of values supported by the platform C :c:func:`localtime` @@ -284,7 +286,7 @@ Functions Accepts any real number, not only integer or float. -.. function:: mktime(t) +.. function:: mktime(time_tuple, /) This is the inverse function of :func:`localtime`. Its argument is the :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` @@ -391,7 +393,7 @@ Functions .. versionadded:: 3.7 -.. function:: sleep(secs) +.. function:: sleep(seconds, /) Suspend execution of the calling thread for the given number of seconds. The argument may be a non-integer to indicate a more precise sleep time. @@ -404,13 +406,16 @@ Functions .. rubric:: Windows implementation - On Windows, if *secs* is zero, the thread relinquishes the remainder of its - time slice to any other thread that is ready to run. If there are no other - threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 10 and newer the implementation uses + On Windows, if *seconds* is zero, + the thread relinquishes the remainder of its time slice + to any other thread that is ready to run. + If there are no other threads ready to run, + the function returns immediately, and the thread continues execution. + On Windows 10 and newer the implementation uses a `high-resolution timer `_ - which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. + which provides resolution of 100 nanoseconds. + If *seconds* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation @@ -425,12 +430,13 @@ Functions To voluntarily relinquish the CPU, specify a real-time :ref:`scheduling policy ` and use :func:`os.sched_yield` instead. - .. audit-event:: time.sleep secs + .. audit-event:: time.sleep seconds .. versionchanged:: 3.5 - The function now sleeps at least *secs* even if the sleep is interrupted - by a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale). + The function now sleeps at least *seconds* + even if the sleep is interrupted by a signal, + except if the signal handler raises an exception + (see :pep:`475` for the rationale). .. versionchanged:: 3.11 On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now @@ -445,13 +451,13 @@ Functions .. index:: single: % (percent); datetime format -.. function:: strftime(format[, t]) +.. function:: strftime(format[, time_tuple]) Convert a tuple or :class:`struct_time` representing a time as returned by :func:`gmtime` or :func:`localtime` to a string as specified by the *format* - argument. If *t* is not provided, the current time as returned by + argument. If *time_tuple* is not provided, the current time as returned by :func:`localtime` is used. *format* must be a string. :exc:`ValueError` is - raised if any field in *t* is outside of the allowed range. + raised if any field in *time_tuple* is outside of the allowed range. 0 is a legal argument for any position in the time tuple; if it is normally illegal the value is forced to a correct one. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index fd67c5c0a0f5133..dd574fce09f5fc8 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -19,7 +19,7 @@ See also Tim Peters' introduction to the "Algorithms" chapter in the second edition of *Python Cookbook*, published by O'Reilly. -Basic Examples +Basic examples -------------- The following example shows how the :ref:`timeit-command-line-interface` @@ -56,7 +56,7 @@ repetitions only when the command-line interface is used. In the .. _python-interface: -Python Interface +Python interface ---------------- The module defines three convenience functions and a public class: @@ -206,7 +206,7 @@ The module defines three convenience functions and a public class: .. _timeit-command-line-interface: -Command-Line Interface +Command-line interface ---------------------- When called as a program from the command line, the following form is used:: @@ -279,6 +279,9 @@ most cases. You can use :func:`time.process_time` to measure CPU time. baseline overhead can be measured by invoking the program without arguments, and it might differ between Python versions. +.. versionadded:: 3.15 + Output is in color by default and can be + :ref:`controlled using environment variables `. .. _timeit-examples: diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index fa3c7e910ce21f0..e27af48ba7ac485 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -33,9 +33,11 @@ alternative `GUI frameworks and tools .dnd_leave(source, event)* -#. Call to *.dnd_enter(source, event)* -#. Call to *.dnd_commit(source, event)* to notify of drop -#. Call to *.dnd_end(target, event)* to signal end of drag-and-drop +#. Call to ``.dnd_leave(source, event)``. +#. Call to ``.dnd_enter(source, event)``. +#. Call to ``.dnd_commit(source, event)`` to notify of the drop. +#. Call to ``.dnd_end(target, event)`` to signal the end of drag-and-drop. .. class:: DndHandler(source, event) @@ -48,7 +49,8 @@ Selection of a target object occurs as follows: .. method:: on_motion(event) - Inspect area below mouse for target objects while drag is performed. + Inspect area below mouse for target objects while a drag + is performed. .. method:: on_release(event) @@ -56,7 +58,9 @@ Selection of a target object occurs as follows: .. function:: dnd_start(source, event) - Factory function for drag-and-drop process. + Factory function for the drag-and-drop process. + Return the :class:`DndHandler` instance managing the drag, or ``None`` if a + drag could not be started. .. seealso:: diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index 9eecb803c3aedcb..c1cab7802da3889 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -20,55 +20,103 @@ The different font weights and slants are: .. class:: Font(root=None, font=None, name=None, exists=False, **options) - The :class:`Font` class represents a named font. *Font* instances are given - unique names and can be specified by their family, size, and style - configuration. Named fonts are Tk's method of creating and identifying - fonts as a single object, rather than specifying a font by its attributes - with each occurrence. - - arguments: - - | *font* - font specifier tuple (family, size, options) - | *name* - unique font name - | *exists* - self points to existing named font if true - - additional keyword options (ignored if *font* is specified): - - | *family* - font family i.e. Courier, Times - | *size* - font size - | If *size* is positive it is interpreted as size in points. - | If *size* is a negative number its absolute value is treated - | as size in pixels. - | *weight* - font emphasis (NORMAL, BOLD) - | *slant* - ROMAN, ITALIC - | *underline* - font underlining (0 - none, 1 - underline) - | *overstrike* - font strikeout (0 - none, 1 - strikeout) + The :class:`Font` class represents a font used by Tk widgets. + It either creates a new *named font* or refers to an existing font. + A named font is Tk's way of identifying a font as a single object that can + be referred to by name and reconfigured in place, + rather than respecifying its attributes at each use. + + With *exists* false (the default), a new named font is created. + Its attributes are taken from the font description *font* if it is given, + overridden by any keyword *options*. + The new font is named *name*, or a generated unique name if *name* is + omitted. + + With *exists* true, an existing font is referred to instead of being + created. + If *name* is given, it is the name of the font, + which is reconfigured by *font* and *options* if either is given. + If *name* is omitted, the font description *font* is wrapped as is, + without creating a named font, + so that it is used without loss of precision by :meth:`actual`, + :meth:`measure` and :meth:`metrics`. + In this case no keyword options are accepted, + and the :attr:`!name` attribute is the description itself rather than a + string. + + The font description *font* is a tuple of the family name, the size and + zero or more styles, + or any other form accepted by Tk, such as the name of a named font. + + The keyword *options* are: + + | *family* - font family, for example, Courier, Times + | *size* - font size + | If *size* is positive it is interpreted as size in points. + | If *size* is a negative number its absolute value is treated + | as size in pixels. + | *weight* - font emphasis (NORMAL, BOLD) + | *slant* - ROMAN, ITALIC + | *underline* - font underlining (0 - none, 1 - underline) + | *overstrike* - font strikeout (0 - none, 1 - strikeout) + + .. versionchanged:: 3.10 + Two fonts now compare equal (``==``) only when both are :class:`Font` + instances with the same name belonging to the same Tcl interpreter. + + .. versionchanged:: next + A font description can now be wrapped without creating a new named font, + and keyword options now override the attributes of the specified *font*. .. method:: actual(option=None, displayof=None) - Return the attributes of the font. + Return the actual attributes of the font, which may differ from the + requested ones because of platform limitations. + With no *option*, return a dictionary of all the attributes; if *option* + is given, return the value of that single attribute. + The attributes are resolved on the display of the *displayof* widget, + or the main application window if it is not specified. .. method:: cget(option) Retrieve an attribute of the font. + .. note:: + + :meth:`!cget` and :meth:`configure` operate on a named font and raise + :exc:`~tkinter.TclError` for a wrapped font description. + Use :meth:`actual` to query the attributes of the latter. + .. method:: config(**options) + :no-typesetting: + + .. method:: configure(**options) + + Modify one or more attributes of the font. + With no arguments, return a dictionary of the current attributes. - Modify attributes of the font. + :meth:`config` is an alias of :meth:`!configure`. .. method:: copy() - Return new instance of the current font. + Return a distinct copy of the current font: + a new named font with the same attributes but a different name, + which can be reconfigured independently of the original. + If the current font wraps a font description, + the copy is instead a named font with its resolved attributes. .. method:: measure(text, displayof=None) Return amount of space the text would occupy on the specified display - when formatted in the current font. If no display is specified then the - main application window is assumed. + when formatted in the current font, as an integer number of pixels. + If no display is specified then the main application window is assumed. .. method:: metrics(*options, **kw) Return font-specific data. + With no options, return a dictionary mapping each metric name to its + integer value; if one option name is given, return that metric's value as + an integer. Options include: *ascent* - distance between baseline and highest point that a @@ -84,15 +132,17 @@ The different font weights and slants are: .. function:: families(root=None, displayof=None) - Return the different font families. + Return a tuple of the names of the available font families. .. function:: names(root=None) - Return the names of defined fonts. + Return a tuple of the names of all the defined fonts. .. function:: nametofont(name, root=None) - Return a :class:`Font` representation of a tk named font. + Return a :class:`Font` representation of the existing named font *name*. + *root* is the widget whose Tcl interpreter owns the font; if omitted, the + default root window is used. .. versionchanged:: 3.10 The *root* parameter was added. diff --git a/Doc/library/tkinter.fontchooser.rst b/Doc/library/tkinter.fontchooser.rst new file mode 100644 index 000000000000000..969495f4914a9e9 --- /dev/null +++ b/Doc/library/tkinter.fontchooser.rst @@ -0,0 +1,90 @@ +:mod:`!tkinter.fontchooser` --- Font selection dialog +===================================================== + +.. module:: tkinter.fontchooser + :synopsis: Font selection dialog + +.. versionadded:: next + +**Source code:** :source:`Lib/tkinter/fontchooser.py` + +-------------- + +The :mod:`!tkinter.fontchooser` module provides the :class:`FontChooser` class +as an interface to the native font selection dialog. + +The font dialog is application-global: +there is a single font dialog per Tcl interpreter, +and all :class:`FontChooser` instances configure the same dialog. + +Depending on the platform, the dialog may be modal or modeless, +so :meth:`~FontChooser.show` may return immediately. +The selected font is not returned: +it is passed as a :class:`~tkinter.font.Font` object to the callback +specified with the *command* option. + +The dialog also generates two virtual events on the parent window +(see the *parent* option): + +``<>`` + Generated when the dialog is shown or hidden. + Query the *visible* option to tell which. + +``<>`` + Generated when the selected font changes. + +.. note:: + + The *command* callback is the only reliable way to obtain the selected font. + On some platforms the *font* option is not updated to the user's choice. + +.. class:: FontChooser(master=None, **options) + + The class implementing the font selection dialog. + + *master* is the widget whose Tcl interpreter owns the dialog. + If omitted, it defaults to *parent* if that is given, + or to the default root window otherwise. + + The supported configuration options are: + + * *parent* --- the window to which the dialog and its virtual events are related. + It defaults to the main window; + on macOS the dialog is shown as a sheet attached to it, + rather than as a free-standing panel. + * *title* --- the title of the dialog. + * *font* --- the font that is currently selected in the dialog. + * *command* --- a callback that is called + with a :class:`~tkinter.font.Font` object wrapping the selected font + when the user selects a font. + * *visible* --- whether the dialog is currently displayed (read-only). + + The *font* option accepts the forms supported by :class:`tkinter.font.Font`. + + .. method:: configure(**options) + config(**options) + + Query or modify the options of the font dialog. + With no arguments, return a dict of all option values. + With a string argument, return the value of that option. + Otherwise, set the given options. + + .. method:: cget(option) + + Return the value of the given option of the font dialog. + + .. method:: show() + + Display the font dialog. + Depending on the platform, this method may return immediately + or only once the dialog has been withdrawn. + + .. method:: hide() + + Hide the font dialog if it is displayed. + + +.. seealso:: + + Module :mod:`tkinter.font` + Tkinter font-handling utilities diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 2a69d282638529d..47fe680bac9107e 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -9,11 +9,12 @@ -------------- The :mod:`!tkinter.messagebox` module provides a template base class as well as -a variety of convenience methods for commonly used configurations. The message -boxes are modal and will return a subset of (``True``, ``False``, ``None``, -:data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on -the user's selection. Common message box styles and layouts include but are not -limited to: +a variety of convenience methods for commonly used configurations. +The message boxes are modal: each blocks until the user responds, then returns +a value that depends on the function. +The ``show*`` functions and :meth:`Message.show` return the symbolic name of +the button the user pressed, as a string (such as :data:`OK` or :data:`YES`). +Common message box styles and layouts include but are not limited to: .. figure:: tk_msg.png @@ -66,6 +67,10 @@ limited to: Arranges for a :ref:`predefined set of buttons ` to be displayed. + .. note:: + + Tk 8.6 added the *command* option. + .. method:: show(**options) Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. @@ -106,7 +111,7 @@ limited to: .. function:: askretrycancel(title=None, message=None, **options) Ask if operation should be retried. Shows buttons :data:`RETRY` and :data:`CANCEL`. - Return ``True`` if the answer is yes and ``False`` otherwise. + Return ``True`` if the answer is retry and ``False`` otherwise. .. function:: askyesno(title=None, message=None, **options) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index a34b74a088874fe..44f688ad0d6422b 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -17,10 +17,15 @@ demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. -Tkinter supports a range of Tcl/Tk versions, built either with or -without thread support. The official Python binary release bundles Tcl/Tk 8.6 -threaded. See the source code for the :mod:`_tkinter` module -for more information about supported versions. +Tkinter supports a range of Tcl/Tk versions, built either with or without +thread support. +Tcl/Tk 8.5.12 is the minimum supported version; the official Python binary +release bundles Tcl/Tk 9.0. +See the source code for the :mod:`_tkinter` module for more information about +supported versions. + +.. versionchanged:: 3.11 + Support for Tcl/Tk versions older than 8.5.12 was removed. Tkinter is not a thin wrapper, but adds a fair amount of its own logic to make the experience more pythonic. This documentation will concentrate on these @@ -30,7 +35,8 @@ details that are unchanged. .. note:: Tcl/Tk 8.5 (2007) introduced a modern set of themed user interface components - along with a new API to use them. Both old and new APIs are still available. + along with a new API to use them (see :mod:`tkinter.ttk`). + Both old and new APIs are still available. Most documentation you will find online still uses the old API and can be woefully outdated. @@ -47,7 +53,7 @@ details that are unchanged. Tcl/Tk Resources: - * `Tk commands `_ + * `Tk commands `_ Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. * `Tcl/Tk Home Page `_ @@ -103,16 +109,16 @@ Ttk bindings are provided in a separate module, :mod:`tkinter.ttk`. Internally, Tk and Ttk use facilities of the underlying operating system, -i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. +that is, Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. -When your Python application uses a class in Tkinter, e.g., to create a widget, +When your Python application uses a class in Tkinter, for example, to create a widget, the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. -Tkinter Modules +Tkinter modules --------------- Support for Tkinter is spread across several modules. Most applications will need the @@ -124,82 +130,6 @@ the modern themed widget set and API:: from tkinter import ttk -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) - - Construct a toplevel Tk widget, which is usually the main window of an - application, and initialize a Tcl interpreter for this widget. Each - instance has its own associated Tcl interpreter. - - The :class:`Tk` class is typically instantiated using all default values. - However, the following keyword arguments are currently recognized: - - *screenName* - When given (as a string), sets the :envvar:`DISPLAY` environment - variable. (X11 only) - *baseName* - Name of the profile file. By default, *baseName* is derived from the - program name (``sys.argv[0]``). - *className* - Name of the widget class. Used as a profile file and also as the name - with which Tcl is invoked (*argv0* in *interp*). - *useTk* - If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` - function sets this to ``False``. - *sync* - If ``True``, execute all X server commands synchronously, so that errors - are reported immediately. Can be used for debugging. (X11 only) - *use* - Specifies the *id* of the window in which to embed the application, - instead of it being created as an independent toplevel window. *id* must - be specified in the same way as the value for the -use option for - toplevel widgets (that is, it has a form like that returned by - :meth:`winfo_id`). - - Note that on some platforms this will only work correctly if *id* refers - to a Tk frame or toplevel that has its -container option enabled. - - :class:`Tk` reads and interprets profile files, named - :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl - interpreter and calls :func:`exec` on the contents of - :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the - profile files is the :envvar:`HOME` environment variable or, if that - isn't defined, then :data:`os.curdir`. - - .. attribute:: tk - - The Tk application object created by instantiating :class:`Tk`. This - provides access to the Tcl interpreter. Each widget that is attached - the same instance of :class:`Tk` has the same value for its :attr:`tk` - attribute. - - .. attribute:: master - - The widget object that contains this widget. For :class:`Tk`, the - :attr:`!master` is :const:`None` because it is the main window. The terms - *master* and *parent* are similar and sometimes used interchangeably - as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`!master` returns the object. - *parent*/*child* reflects the tree-like relationship while - *master* (or *container*)/*content* reflects the container structure. - - .. attribute:: children - - The immediate descendants of this widget as a :class:`dict` with the - child widget names as the keys and the child instance objects as the - values. - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) - - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. - - The modules that provide Tk support include: :mod:`!tkinter` @@ -217,6 +147,9 @@ The modules that provide Tk support include: :mod:`tkinter.font` Utilities to help work with fonts. +:mod:`tkinter.fontchooser` + Dialog to let the user choose a font. + :mod:`tkinter.messagebox` Access to standard Tk dialog boxes. @@ -226,6 +159,9 @@ The modules that provide Tk support include: :mod:`tkinter.simpledialog` Basic dialogs and convenience functions. +:mod:`tkinter.systray` + System tray icon and desktop notifications. + :mod:`tkinter.ttk` Themed widget set introduced in Tk 8.5, providing modern alternatives for many of the classic widgets in the main :mod:`!tkinter` module. @@ -246,7 +182,7 @@ Additional modules: Python's Integrated Development and Learning Environment (IDLE). Based on :mod:`!tkinter`. -:mod:`tkinter.constants` +:mod:`!tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the main :mod:`!tkinter` module. @@ -258,8 +194,10 @@ Additional modules: :mod:`turtle` Turtle graphics in a Tk window. +.. currentmodule:: tkinter -Tkinter Life Preserver + +Tkinter life preserver ---------------------- This section is not designed to be an exhaustive tutorial on either Tk or @@ -274,7 +212,7 @@ find more detailed documentation on them, including in the official Tcl/Tk reference manual. -A Hello World Program +A Hello World program ^^^^^^^^^^^^^^^^^^^^^ We'll start by walking through a "Hello World" application in Tkinter. This @@ -302,19 +240,20 @@ The following line creates a frame widget, which in this case will contain a label and a button we'll create next. The frame is fit inside the root window. -The next line creates a label widget holding a static text string. The -:meth:`grid` method is used to specify the relative layout (position) of the -label within its containing frame widget, similar to how tables in HTML work. +The next line creates a label widget holding a static text string. +The :meth:`~Grid.grid` method is used to specify the relative layout (position) +of the label within its containing frame widget, similar to how tables in HTML +work. -A button widget is then created, and placed to the right of the label. When -pressed, it will call the :meth:`destroy` method of the root window. +A button widget is then created, and placed to the right of the label. +When pressed, it will call the :meth:`~Misc.destroy` method of the root window. Finally, the :meth:`mainloop` method puts everything on the display, and responds to user input until the program terminates. -Important Tk Concepts +Important Tk concepts ^^^^^^^^^^^^^^^^^^^^^ Even this simple program illustrates the following key Tk concepts: @@ -346,7 +285,7 @@ event loop isn't running the event loop, your user interface won't update. -Understanding How Tkinter Wraps Tcl/Tk +Understanding how Tkinter wraps Tcl/Tk ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When your application uses Tkinter's classes and methods, internally Tkinter @@ -414,9 +353,9 @@ interactive Python shell or with :func:`print`, can help you identify what you need. To find out what configuration options are available on any widget, call its -:meth:`configure` method, which returns a dictionary containing a variety of -information about each object, including its default and current values. Use -:meth:`keys` to get just the names of each option. +:meth:`~Misc.configure` method, which returns a dictionary containing a variety +of information about each object, including its default and current values. +Use :meth:`~Misc.keys` to get just the names of each option. :: @@ -443,18 +382,20 @@ is helpful. print(set(dir(btn)) - set(dir(frm))) -Navigating the Tcl/Tk Reference Manual +Navigating the Tcl/Tk reference manual ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As noted, the official `Tk commands `_ -reference manual (man pages) is often the most accurate description of what -specific operations on widgets do. Even when you know the name of the option -or method that you need, you may still have a few places to look. +As noted, the official +`Tk commands `_ reference +manual (man pages) is often the most accurate description of what specific +operations on widgets do. +Even when you know the name of the option or method that you need, you may +still have a few places to look. While all operations in Tkinter are implemented as method calls on widget objects, you've seen that many Tcl/Tk operations appear as commands that take a widget pathname as its first parameter, followed by optional -parameters, e.g. +parameters, for example :: @@ -473,22 +414,23 @@ name of a method to call). In the official Tcl/Tk reference documentation, you'll find most operations -that look like method calls on the man page for a specific widget (e.g., -you'll find the :meth:`invoke` method on the -`ttk::button `_ +that look like method calls on the man page for a specific widget (for example, +you'll find the :meth:`~tkinter.ttk.Button.invoke` method on the +`ttk::button `_ man page), while functions that take a widget as a parameter often have -their own man page (e.g., -`grid `_). +their own man page (for example, +`grid `_). You'll find many common options and methods in the -`options `_ or -`ttk::widget `_ man +`options `_ or +`ttk::widget `_ man pages, while others are found in the man page for a specific widget class. -You'll also find that many Tkinter methods have compound names, e.g., -:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find -documentation for all of these in the -`winfo `_ man page. +You'll also find that many Tkinter methods have compound names, for example, +:meth:`~Misc.winfo_x`, :meth:`~Misc.winfo_height`, +:meth:`~Misc.winfo_viewable`. +You'd find documentation for all of these in the +`winfo `_ man page. .. note:: Somewhat confusingly, there are also methods on all Tkinter widgets @@ -517,14 +459,16 @@ from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. -Tcl/Tk applications are normally event-driven, meaning that after initialization, -the interpreter runs an event loop (i.e. :func:`Tk.mainloop`) and responds to events. -Because it is single-threaded, event handlers must respond quickly, otherwise they -will block other events from being processed. To avoid this, any long-running -computations should not run in an event handler, but are either broken into smaller -pieces using timers, or run in another thread. This is different from many GUI -toolkits where the GUI runs in a completely separate thread from all application -code including event handlers. +Tcl/Tk applications are normally event-driven, meaning that after +initialization, the interpreter runs an event loop (that is, +:meth:`Tk.mainloop `) and responds to events. +Because it is single-threaded, event handlers must respond quickly, otherwise +they will block other events from being processed. +To avoid this, any long-running computations should not run in an event +handler, but are either broken into smaller pieces using timers, or run in +another thread. +This is different from many GUI toolkits where the GUI runs in a completely +separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any :mod:`!tkinter` calls made from threads other than the one running the Tcl @@ -532,7 +476,9 @@ interpreter will fail. A number of special cases exist: -* Tcl/Tk libraries can be built so they are not thread-aware. In this case, +* Tcl/Tk libraries built without thread support are now rare: Tcl/Tk 9.0 (the + bundled version) is always thread-aware, so this case only arises with some + older 8.x builds. When the library is not thread-aware, :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. @@ -552,13 +498,13 @@ A number of special cases exist: called from the thread that created the Tcl interpreter. -Handy Reference +Handy reference --------------- .. _tkinter-setting-options: -Setting Options +Setting options ^^^^^^^^^^^^^^^ Options control things like the color and border width of a widget. Options can @@ -580,6 +526,17 @@ Use the config() method to update multiple attrs subsequent to object creation fred.config(fg="red", bg="blue") +.. note:: + + The ``fg`` and ``bg`` options used here, + and other options that control a widget's appearance, + belong to the classic :mod:`!tkinter` widgets. + The themed :mod:`tkinter.ttk` widgets recommended in the introduction + do not accept them; + style a themed widget through the :class:`ttk.Style ` + class instead. + The three ways of setting an option shown above apply to both widget sets. + For a complete explanation of a given option and its behavior, see the Tk man pages for the widget in question. @@ -594,17 +551,14 @@ document. Some options don't apply to some kinds of widgets. Whether a given widget responds to a particular option depends on the class of the widget; buttons have a ``command`` option, labels do not. -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. +The options supported by a given widget are listed in that widget's man page, +or can be queried at runtime by calling the :meth:`~Misc.config` method without +arguments, or by calling the :meth:`~Misc.keys` method on that widget. +The return value of these calls is a dictionary whose key is the name of the +option as a string (for example, ``'relief'``) and whose values are 5-tuples. -Some options, like ``bg`` are synonyms for common options with long names -(``bg`` is shorthand for "background"). Passing the ``config()`` method the name -of a shorthand option will return a 2-tuple, not 5-tuple. The 2-tuple passed -back will contain the name of the synonym and the "real" option (such as -``('bg', 'background')``). +Some options, like ``bg``, are synonyms for common options with long names +(``bg`` is shorthand for "background"). +-------+---------------------------------+--------------+ | Index | Meaning | Example | @@ -630,174 +584,206 @@ Of course, the dictionary printed will include all the options available and their values. This is meant only as an example. -The Packer -^^^^^^^^^^ +.. _pack-the-packer: +.. _tkinter-geometry-management: +.. _the-packer: +.. _packer-options: -.. index:: single: packing (widgets) +Geometry management +^^^^^^^^^^^^^^^^^^^ -The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of widgets within their container. -In contrast to the more cumbersome *placer* (which is -used less commonly, and we do not cover here), the packer takes qualitative -relationship specification - *above*, *to the left of*, *filling*, etc - and -works everything out to determine the exact placement coordinates for you. +.. index:: + single: geometry management (widgets) + single: packing (widgets) + +Creating a widget does not display it. +A widget appears only after it has been handed to a *geometry manager*, +which works out its size and position inside its container +and keeps the layout up to date as the container is resized or its content changes. +Forgetting to call a geometry manager is a common early mistake: +the widget is created, but nothing shows up. + +Tk provides three geometry managers. +Each is inherited by every widget, so any widget can be managed by any of them +(but see the warning below about the incompatibility of grid and pack). +The choice depends on the kind of layout you want. + +:meth:`grid ` + Arranges widgets in a two-dimensional table of rows and columns. + It is the most flexible manager and the one to reach for by default: + layouts that would otherwise need several nested frames can often be + expressed as a single grid, + and rows and columns can be told how to absorb extra space. -The size of any container widget is determined by the size of the "content widgets" -inside. The packer is used to control where content widgets appear inside the -container into which they are packed. You can pack widgets into frames, and frames -into other frames, in order to achieve the kind of layout you desire. -Additionally, the arrangement is dynamically adjusted to accommodate incremental -changes to the configuration, once it is packed. + :: -Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. + ttk.Label(frm, text="Name:").grid(column=0, row=0, sticky="w") + ttk.Entry(frm).grid(column=1, row=0) + ttk.Button(frm, text="OK").grid(column=1, row=1, sticky="e") -The pack() method can be called with keyword-option/value pairs that control -where the widget is to appear within its container, and how it is to behave when -the main application window is resized. Here are some examples:: +:meth:`pack ` + Stacks widgets against one side of their container + -- ``"top"`` (the default), ``"bottom"``, ``"left"`` or ``"right"`` -- + and can make them fill or expand into the space that is left. + It is convenient for simple arrangements, + such as a single row or column of widgets + or a content area framed by a toolbar and a status bar. - fred.pack() # defaults to side = "top" - fred.pack(side="left") - fred.pack(expand=1) + :: + toolbar.pack(side="top", fill="x") + status.pack(side="bottom", fill="x") + body.pack(side="left", expand=True, fill="both") -Packer Options -^^^^^^^^^^^^^^ +:meth:`place ` + Positions each widget at an explicit spot, + given either as absolute screen distances or as a fraction of the container's size. + It offers the most control but the least automatic behavior, and is used the least; + it suits special cases such as overlapping widgets or precise custom layouts. -For more extensive information on the packer and the options that it can take, -see the man pages and page 183 of John Ousterhout's book. + :: -anchor - Anchor type. Denotes where the packer is to place each content in its parcel. + background.place(x=0, y=0, relwidth=1.0, relheight=1.0) + badge.place(relx=1.0, rely=0.0, anchor="ne") -expand - Boolean, ``0`` or ``1``. +Layouts are built up by nesting: +grid or pack widgets, including frames, inside a frame or toplevel. +Toplevels are managed by the OS window manager. +Classic and themed :mod:`tkinter.ttk` widgets can be managed interchangeably. -fill - Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. +.. warning:: -ipadx and ipady - A distance - designating internal padding on each side of the content. + Do not apply :meth:`!pack` and :meth:`!grid` to two widgets that share the same container. + The two managers negotiate sizes in incompatible ways, + and the application can hang as they repeatedly resize the container against each other. + To combine them, keep each manager's widgets in a separate frame. -padx and pady - A distance - designating external padding on each side of the content. +The full set of options accepted by each manager, with their values and defaults, +is documented under :meth:`Grid.grid_configure`, :meth:`Pack.pack_configure` +and :meth:`Place.place_configure`; +see also the :manpage:`grid(3tk)`, :manpage:`pack(3tk)` and :manpage:`place(3tk)` +man pages. -side - Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. +.. _coupling-widget-variables: -Coupling Widget Variables +Coupling widget variables ^^^^^^^^^^^^^^^^^^^^^^^^^ -The current-value setting of some widgets (like text entry widgets) can be -connected directly to application variables by using special options. These -options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and -``value``. This connection works both ways: if the variable changes for any -reason, the widget it's connected to will be updated to reflect the new value. - -Unfortunately, in the current implementation of :mod:`!tkinter` it is not -possible to hand over an arbitrary Python variable to a widget through a -``variable`` or ``textvariable`` option. The only kinds of variables for which -this works are variables that are subclassed from a class called Variable, -defined in :mod:`!tkinter`. - -There are many useful subclasses of Variable already defined: -:class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. +Some widgets can tie their current value directly to a program variable, +so that the two stay in sync. +Options such as ``variable``, ``textvariable``, ``value``, ``onvalue`` and +``offvalue`` set up this connection: +when the user changes the widget the variable is updated, +and when the variable is set the widget redraws to match. + +A widget can be linked only to a :class:`Variable` object, +not to an ordinary Python variable. +This is not a limitation of :mod:`!tkinter` +but a consequence of how the two languages differ: +the link relies on Tcl being notified every time the value changes, +and Python offers no way to react when a plain variable is reassigned. +A :class:`Variable` sidesteps this by keeping its value inside the Tcl interpreter +and exposing it through explicit :meth:`~Variable.get` and :meth:`~Variable.set` methods. + +Ready-made subclasses cover the common types: +:class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` and :class:`BooleanVar`. +Pass one as a widget's ``textvariable`` (or ``variable``) option, +then read and update it with :meth:`~Variable.get` and :meth:`~Variable.set`; +the widget tracks it with no further work on your part. + +Keep a reference to the variable for as long as the widget uses it +-- for example by storing it as an attribute. +A :class:`Variable` that is garbage collected removes its underlying Tcl variable, +breaking the connection to the widget (see :class:`Variable`). For example:: import tkinter as tk + from tkinter import ttk - class App(tk.Frame): - def __init__(self, master): - super().__init__(master) - self.pack() + root = tk.Tk() - self.entrythingy = tk.Entry() - self.entrythingy.pack() + # Create the application variable and give it an initial value. + contents = tk.StringVar(value="this is a variable") - # Create the application variable. - self.contents = tk.StringVar() - # Set it to some value. - self.contents.set("this is a variable") - # Tell the entry widget to watch this variable. - self.entrythingy["textvariable"] = self.contents + # Tell the entry widget to track the variable. + entry = ttk.Entry(root, textvariable=contents) + entry.pack() - # Define a callback for when the user hits return. - # It prints the current value of the variable. - self.entrythingy.bind('', - self.print_contents) + # Print the current value whenever the user presses Return. + def print_contents(event): + print("The current entry content is:", contents.get()) - def print_contents(self, event): - print("Hi. The current entry content is:", - self.contents.get()) + entry.bind("", print_contents) - root = tk.Tk() - myapp = App(root) - myapp.mainloop() + # Setting the variable from the program updates the entry through the + # same link. + def clear(): + contents.set("") + + ttk.Button(root, text="Clear", command=clear).pack() + + root.mainloop() -The Window Manager +.. _tkinter-window-manager: + +The window manager ^^^^^^^^^^^^^^^^^^ .. index:: single: window manager (widgets) -In Tk, there is a utility command, ``wm``, for interacting with the window -manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`!tkinter`, these commands have -been implemented as methods on the :class:`Wm` class. Toplevel widgets are -subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods -directly. +The *window manager* is the part of the desktop responsible for the title bar, +border and controls drawn around each top-level window, +and for such things as its title, position, size and icon. +Tk gives access to these through the :class:`Wm` mixin, +which is inherited by the :class:`Tk` root window and by every :class:`Toplevel`. +You therefore call the window-manager methods directly on a top-level window. +Each has a short name and an equivalent ``wm_``-prefixed name, +for example :meth:`~Wm.title` and :meth:`~Wm.wm_title`. -To get at the toplevel window that contains a given widget, you can often just -refer to the widget's :attr:`master`. Of course if the widget has been packed inside of -a frame, the :attr:`!master` won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. +These methods act on the top-level window +whether its content is built from the classic widgets or the themed :mod:`tkinter.ttk` widgets. +To reach the top-level window containing an arbitrary widget, +call its :meth:`~Misc.winfo_toplevel` method. -Here are some examples of typical usage:: +For example:: import tkinter as tk + from tkinter import ttk + + root = tk.Tk() + root.title("My Application") + root.geometry("640x480") + root.minsize(320, 240) - class App(tk.Frame): - def __init__(self, master=None): - super().__init__(master) - self.pack() + ttk.Label(root, text="Hello").pack(padx=20, pady=20) - # create the application - myapp = App() + root.mainloop() - # - # here are method calls to the window manager class - # - myapp.master.title("My Do-Nothing Application") - myapp.master.maxsize(1000, 400) +See :class:`Wm` for the full set of window-manager methods. - # start the program - myapp.mainloop() +.. _Tk-option-data-types: -Tk Option Data Types +Tk option data types ^^^^^^^^^^^^^^^^^^^^ .. index:: single: Tk Option Data Types +Many widget options documented in the reference +accept values of a small number of common types, described here. + anchor Legal values are points of the compass: ``"n"``, ``"ne"``, ``"e"``, ``"se"``, ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. + There are ten built-in, named bitmaps: ``'error'``, ``'gray12'``, + ``'gray25'``, ``'gray50'``, ``'gray75'``, ``'hourglass'``, ``'info'``, + ``'questhead'``, ``'question'``, ``'warning'``. To specify an X bitmap + filename, give the full path to the file, preceded with an ``@``, as in + ``"@/usr/contrib/bitmap/gumby.bit"``. boolean You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. @@ -813,13 +799,20 @@ color Colors can be given as the names of X colors in the rgb.txt file, or as strings representing RGB values in 4 bit: ``"#RGB"``, 8 bit: ``"#RRGGBB"``, 12 bit: ``"#RRRGGGBBB"``, or 16 bit: ``"#RRRRGGGGBBBB"`` ranges, where R,G,B here - represent any legal hex digit. See page 160 of Ousterhout's book for details. + represent any legal hex digit. See the :manpage:`colors(3tk)` man page for + the list of named colors. cursor - The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the - string ``"hand2"``. You can also specify a bitmap and mask file of your own. - See page 179 of Ousterhout's book. + The name of the mouse cursor to display while the pointer is over the widget. + Tk provides a portable set of cursor names available on all platforms + (for example ``"arrow"``, ``"watch"``, ``"cross"``, or ``"hand2"``); + the standard X cursor names from :file:`cursorfont.h` may also be used, + without the ``XC_`` prefix (so ``XC_hand2`` becomes ``"hand2"``). + The full list of names, including the platform-specific ones, + is given in the :manpage:`cursors(3tk)` manual page. + You can also specify a bitmap and mask file of your own. + On Windows a cursor file (:file:`.cur` or :file:`.ani`) may be used directly, + giving its path preceded with an ``@``, as in ``"@C:/cursors/bart.ani"``. distance Screen distances can be specified in either pixels or absolute distances. @@ -828,10 +821,21 @@ distance millimetres, ``p`` for printer's points. For example, 3.5 inches is expressed as ``"3.5i"``. + When a screen distance option is read back (for example with :meth:`!cget`), + a distance with no unit suffix is returned as an :class:`int` or a :class:`float`. + A distance with a unit suffix depends on the screen resolution + and is returned as an opaque Tcl object that can be passed back to Tk. + + .. versionchanged:: next + Screen distances with no unit suffix are returned as an :class:`int` + or a :class:`float`. + font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. + Tk uses a font description such as ``{courier 10 bold}``; in + :mod:`!tkinter` this is most naturally passed as a tuple of + ``(family, size, *styles)`` (or as the equivalent string + ``"Courier 10 bold"``). Font sizes with positive numbers are measured in + points; sizes with negative numbers are measured in pixels. geometry This is a string of the form ``widthxheight``, where width and height are @@ -848,7 +852,8 @@ region relief Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. + ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, ``"ridge"``, and + ``"solid"``. scrollcommand This is almost always the :meth:`!set` method of some scrollbar widget, but can @@ -859,7 +864,7 @@ wrap .. _Bindings-and-Events: -Bindings and Events +Bindings and events ^^^^^^^^^^^^^^^^^^^ .. index:: @@ -875,9 +880,11 @@ of the bind method is:: where: sequence - is a string that denotes the target kind of event. (See the - :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, - :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). + is a string that denotes the target kind of event. Physical events use the + ```` form (for example ``""`` or + ``""``); application-defined virtual events use double angle + brackets, as in ``"<>"``. (See the + :manpage:`bind(3tk)` man page for details.) func is a Python function, taking one argument, to be invoked when the event occurs. @@ -921,9 +928,27 @@ they are denoted in Tk, which can be useful when referring to the Tk man pages. +----+---------------------+----+---------------------+ | %y | y | %Y | y_root | +----+---------------------+----+---------------------+ +| %# | serial | %b | num | ++----+---------------------+----+---------------------+ +| %d | detail | %D | delta | ++----+---------------------+----+---------------------+ + +The ``add`` parameter above only affects the bindings you make yourself. +Every widget also inherits *class bindings* +that implement its standard behavior -- +for example a :class:`Text` widget binds :kbd:`Control-t` +to transpose two characters. +These are described in the bindings section of the widget's Tk man page +(such as :manpage:`text(3tk)` or :manpage:`entry(3tk)`). + +Class bindings are processed separately from your own, +so binding an event yourself does not replace the default; both run. +To suppress an unwanted default binding, +bind the event on the widget +and return the string ``"break"`` from your callback. -The index Parameter +The index parameter ^^^^^^^^^^^^^^^^^^^ A number of widgets require "index" parameters to be passed. These are used to @@ -931,9 +956,32 @@ point at a specific place in a Text widget, or to particular characters in an Entry widget, or to particular menu items in a Menu widget. Entry widget indexes (index, view index, etc.) - Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`!tkinter` functions to access these special - points in text widgets: + Entry widgets have methods and options that refer to character positions + in the text being displayed. + Anytime an index is needed, you may pass in: + + * an integer which refers to the numeric position of a character, + counted from the beginning of the text, starting with 0; + + * the string ``"anchor"``, + which refers to the anchor point of the selection, + set with the widget's selection methods; + + * the string ``"end"``, + which refers to the position just after the last character; + + * the string ``"insert"``, + which refers to the character just after the insertion cursor; + + * the strings ``"sel.first"`` and ``"sel.last"``, + which refer to the first character in the selection + and the position just after the last + (it is an error to use these if there is no selection); + + * a string consisting of ``@`` followed by an integer, as in ``"@6"``, + where the integer is interpreted as an x pixel coordinate + in the entry's coordinate system, + selecting the character spanning that point. Text widget indexes The index notation for Text widgets is very rich and is best described in the Tk @@ -951,8 +999,9 @@ Menu indexes (menu.invoke(), menu.entryconfig(), etc.) * the string ``"last"`` which refers to the last menu item; - * An integer preceded by ``@``, as in ``@6``, where the integer is interpreted - as a y pixel coordinate in the menu's coordinate system; + * a string consisting of ``@`` followed by an integer, as in ``"@6"``, where + the integer is interpreted as a y pixel coordinate in the menu's coordinate + system; * the string ``"none"``, which indicates no menu entry at all, most often used with menu.activate() to deactivate all entries, and finally, @@ -988,7 +1037,7 @@ option (other options are available as well). :meth:`!copy`. The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a +some widget (for example, labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is deleted, the image data is deleted as well, and Tk will display an empty box wherever the image was used. @@ -998,51 +1047,5854 @@ wherever the image was used. The `Pillow `_ package adds support for formats such as BMP, JPEG, TIFF, and WebP, among others. -.. _tkinter-file-handlers: -File Handlers -------------- +Reference +--------- -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: +.. currentmodule:: tkinter - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) +This section documents the classes, methods, functions and constants of the +:mod:`!tkinter` module. +Most of them wrap Tcl/Tk commands; consult the official Tcl/Tk manual pages for +the full list of widget options and further details. -This feature is not available on Windows. +.. exception:: TclError -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. + The exception raised when a call into the Tcl interpreter fails, for example + when a widget is given an unknown option or an invalid value. +Base and mixin classes +^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Widget.tk.createfilehandler(file, mask, func) +.. class:: Misc() - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: + The :class:`!Misc` class is a mix-in inherited by :class:`Tk` and, through + :class:`BaseWidget`, by every widget. + It provides the large set of methods common to all Tk objects: querying + window information, managing event bindings and the event loop, controlling + the keyboard focus and pointer grabs, accessing the selection, clipboard and + option database, and assorted utility and introspection services. + Because they are inherited, these methods are available on every widget and + on the :class:`Tk` application object, and are documented here once rather + than repeated for each widget. - callback(file, mask) + .. method:: cget(key) + Return the current value of the configuration option named *key* for this + widget, as a string. + The expression ``widget[key]`` is equivalent and may be used instead. -.. method:: Widget.tk.deletefilehandler(file) + .. method:: config(cnf=None, **kw) + :no-typesetting: - Unregisters a file handler. + .. method:: configure(cnf=None, **kw) + Query or modify the configuration options of the widget. + With no arguments, return a dictionary mapping every available option + name to a tuple describing it (its name, X resource name, X resource + class, default value and current value). + If a single option name is given as a string, return the tuple for just + that option. + If one or more keyword arguments are given, or a dictionary is passed as + *cnf*, set each named option to the corresponding value; the expression + ``widget[key] = value`` sets a single option in the same way. -.. data:: READABLE - WRITABLE - EXCEPTION + :meth:`config` is an alias of :meth:`!configure`. - Constants used in the *mask* arguments. + .. method:: keys() + + Return a list of the names of all configuration options of this widget. + + .. method:: getboolean(s) + + Interpret the string *s* as a Tcl boolean and return the corresponding + :class:`bool`. + Tcl accepts values such as ``'1'``, ``'0'``, ``'yes'``, ``'no'``, + ``'true'`` and ``'false'``. + Raise :exc:`ValueError` if *s* is not a valid boolean. + + .. method:: getdouble(s) + + Interpret the string *s* as a Tcl floating-point number and return it as + a :class:`float`. + Raise :exc:`ValueError` if *s* is not a valid number. + + .. versionadded:: 3.5 + + + .. method:: getint(s) + + Interpret the string *s* as a Tcl integer and return it as an + :class:`int`. + Raise :exc:`ValueError` if *s* is not a valid integer. + + .. method:: getvar(name) + + Return the value of the Tcl global variable named *name*. + + .. method:: setvar(name, value) + + Set the Tcl global variable named *name* to *value*. + + The :meth:`!getvar` and :meth:`!setvar` methods give direct access to Tcl + variables. + In most code you will instead use a :class:`Variable` subclass such as + :class:`StringVar` or :class:`IntVar`, which wraps a Tcl variable and + converts its value to and from a Python type. + + .. method:: register(func, subst=None, needcleanup=1) + + Register the Python callable *func* as a Tcl command and return the name + of the new command as a string. + Whenever Tcl invokes that command, *func* is called; if *subst* is given, + it is applied to the command's arguments first. + This is the mechanism used internally to turn Python callbacks into the + command names passed to Tk options such as *command*. + Unless *needcleanup* is false, the command is deleted automatically when + the widget is destroyed. + + .. versionchanged:: 3.13 + The arguments passed to *func* are no longer converted to strings. + + .. method:: deletecommand(name) + + Delete the Tcl command named *name*, such as one previously returned by + :meth:`register`. + + .. method:: nametowidget(name) + + Return the widget instance corresponding to the Tk pathname *name*. + + .. method:: send(interp, cmd, *args) + + Send the Tcl command *cmd*, with the given *args*, to the Tcl interpreter + registered under the name *interp*, and return its result. + This is not available on all platforms. + + .. method:: destroy() + + Destroy this widget and all of its descendant widgets, and delete the Tcl + commands associated with them. + + .. method:: lift(aboveThis=None) + :no-typesetting: + + .. method:: tkraise(aboveThis=None) + + Raise this widget in the stacking order so that it is drawn on top of its + siblings. + If *aboveThis* is given, the widget is moved to be just above it in the + stacking order instead. + + :meth:`lift` is an alias of :meth:`!tkraise`. + + .. method:: lower(belowThis=None) + + Lower this widget in the stacking order so that it is drawn beneath its + siblings. + If *belowThis* is given, the widget is moved to be just below it in the + stacking order instead. + + :meth:`tkraise`/:meth:`lift` and :meth:`lower` are overridden by the + :class:`Canvas` widget, + where they restack canvas items instead. + + .. method:: image_names() + + Return the names of all images that currently exist in the Tcl + interpreter. + + This is overridden by the :class:`Text` widget, + where :meth:`!image_names` returns the names of its embedded images + instead. + + .. method:: image_types() + + Return the available image types, such as ``'photo'`` and ``'bitmap'``. + + .. method:: anchor(anchor=None) + :no-typesetting: + + .. method:: grid_anchor(anchor=None) + + Set the anchor that controls where the grid is placed inside this + container when the container is larger than the grid and no row or column + has a non-zero weight. + *anchor* is one of the usual anchor strings, such as ``'nw'`` (the + default) or ``'center'``. + Called with no argument, this method has no effect. + + :meth:`anchor` is an alias of :meth:`!grid_anchor`. + + .. versionadded:: 3.3 + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Return the bounding box, in pixels, of a region of the grid laid out in + this container, as a 4-tuple ``(xoffset, yoffset, width, height)``. + With no arguments the bounding box of the whole grid is returned. + If *column* and *row* are given, the box spans from the cell at row and + column 0 to that cell; if *col2* and *row2* are also given, it spans from + the cell (*column*, *row*) to the cell (*col2*, *row2*). + + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Query or set the properties of the column (or columns) *index* of the + grid managed by this container. + *index* may be a column number; when setting options it may also be a + list of column numbers, the string ``'all'`` to affect every column, or + a child widget whose occupied columns are affected. + The supported options are: + + *minsize* + The column's minimum size, in pixels. + + *weight* + An integer setting how much of any extra space is apportioned to the + column. + A weight of ``0`` keeps the column at its requested size, and a column + of weight two grows twice as fast as a column of weight one. + + *uniform* + The name of a uniform group. + Columns sharing a non-empty group name are kept in sizes that are + strictly proportional to their weights. + + *pad* + Extra space, in pixels, added to the largest widget in the column when + computing the column's size. + + With a single option name, return that option's value; with no options, + return a dictionary of all of them. + + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Query or set the properties of the row (or rows) *index* of the grid + managed by this container. + *index* is interpreted as for :meth:`grid_columnconfigure`, and the + supported options (*minsize*, *weight*, *uniform* and *pad*) are the + same, applied to a row instead of a column. + + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: grid_location(x, y) + + Return the ``(column, row)`` of the grid cell that contains the pixel at + position (*x*, *y*), given in pixels relative to this container. + For locations above or to the left of the grid, ``-1`` is returned for + the corresponding coordinate. + + .. method:: grid_propagate() + grid_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the grid geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Return the size of the grid managed by this container as a + ``(columns, rows)`` tuple. + + :meth:`size` is an alias of :meth:`!grid_size`, + except on :class:`Listbox` and + :class:`ttk.Treeview `, + which provide their own :meth:`!size` method. + + .. method:: grid_slaves(row=None, column=None) + + Return a list of the child widgets managed in this container's grid, most + recently managed first. + If *row* or *column* is given, only the children in that row or column + are returned. + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`grid_slaves`: return the child widgets managed in this + container's grid, optionally restricted to a given *row* or *column*. + + .. versionadded:: 3.15 + + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the pack geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Return a list of the child widgets managed by this container with the + pack geometry manager, in packing order. + + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`pack_slaves`: return the child widgets managed by this + container with the pack geometry manager, in packing order. + + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + + .. method:: place_slaves() + + Return a list of the child widgets managed by this container with the + place geometry manager. + + .. method:: place_content() + + Same as :meth:`place_slaves`: return the child widgets managed by this + container with the place geometry manager. + + .. versionadded:: 3.15 + + The methods with the ``bind`` and ``unbind`` prefixes associate event + patterns with callbacks and remove those associations. + + .. method:: bind(sequence=None, func=None, add=None) + + Bind the event pattern *sequence* on this widget to the callable *func*. + + *sequence* is an event pattern, such as ``''`` (a mouse click) + or ``''``, optionally a concatenation of several such + patterns that must occur shortly after one another. + When the event occurs, *func* is called with an :class:`Event` instance + describing it as its only argument; if *func* returns the string + ``'break'``, no further bindings for the event are invoked. + + If *add* is true, *func* is added to any functions already bound to + *sequence*; otherwise it replaces them. + The binding applies only to this widget. + + :meth:`!bind` returns a string identifier (a *funcid*) that can later be + passed to :meth:`unbind` to remove the binding without leaking the + associated Tcl command. + + If *func* is omitted, return the binding currently associated with + *sequence*; if *sequence* is also omitted, return a list of all the + sequences for which bindings exist on this widget. + + .. method:: bind_class(className, sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the binding tag *className* rather + than to a single widget, so that the binding applies to every widget + having that tag. + *className* is usually the name of a widget class, such as ``'Button'``, + in which case the binding affects all widgets of that class. + The set of binding tags for a widget can be inspected and changed with + :meth:`bindtags`. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: bind_all(sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the special binding tag ``'all'``, + so that the binding applies to every widget in the application. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: unbind(sequence, funcid=None) + + Remove bindings for the event pattern *sequence* on this widget. + + If *funcid* is given, only the function identified by it (a value + returned from a previous call to :meth:`bind`) is removed, and its + associated Tcl command is deleted. + Otherwise all bindings for *sequence* are destroyed, leaving it unbound. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound; other callbacks + bound to *sequence* are kept. + + + .. method:: unbind_class(className, sequence) + + Remove all bindings for the event pattern *sequence* from the binding tag + *className*. + See :meth:`bind_class`. + + .. method:: unbind_all(sequence) + + Remove all bindings for the event pattern *sequence* from the special + binding tag ``'all'``. + See :meth:`bind_all`. + + .. method:: bindtags(tagList=None) + + If *tagList* is omitted, return a tuple of the binding tags associated + with this widget. + When an event occurs in a widget, it is applied to each of the widget's + binding tags in order, and for each tag the most specific matching + binding is executed. + By default a widget has four binding tags: its own pathname, its widget + class, the pathname of its nearest toplevel ancestor, and ``'all'``, in + that order. + + If *tagList* is given, it must be a sequence of strings; the widget's + binding tags are set to its elements, which determines the order in which + bindings are evaluated. + + The methods with the ``event_`` prefix define virtual events and generate + events programmatically. + + .. method:: event_add(virtual, *sequences) + + Associate the virtual event *virtual*, whose name has the form + ``'<>'``, with each of the physical event patterns given by + *sequences*, so that the virtual event triggers whenever any of them + occurs. + If *virtual* is already defined, the new sequences are added to its + existing ones. + + .. method:: event_delete(virtual, *sequences) + + Remove each of *sequences* from those associated with the virtual event + *virtual*. + Sequences that are not currently associated with *virtual* are ignored. + If no *sequences* are given, all physical event sequences are removed, so + that *virtual* no longer triggers. + + .. method:: event_generate(sequence, **kw) + + Generate the event *sequence* on this widget and arrange for it to be + processed just as if it had come from the window system. + *sequence* must be a single event pattern, such as ``''`` or + ``'<>'``, not a concatenation of several. + Keyword arguments specify additional fields of the event, for example *x* + and *y* for the pointer position, or *when* to control when the event is + processed; refer to the Tk ``event`` manual page for the full list. + + .. method:: event_info(virtual=None) + + If *virtual* is omitted, return a tuple of all the virtual events that + are currently defined. + If *virtual* is given, return a tuple of the physical event sequences + currently associated with it, or an empty tuple if it is not defined. + + The methods with the ``after`` prefix schedule callbacks to run after a + delay or when the application is idle. + + .. method:: after(ms, func=None, *args, **kw) + + Schedule the callable *func* to be called after *ms* milliseconds, with + *args* and *kw* passed to it as positional and keyword arguments. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + If *func* is omitted, sleep for *ms* milliseconds instead, processing no + events during that time, and return ``None``. + + .. versionchanged:: 3.10 + *func* can now be any callable object, not only a function. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_cancel(id) + + Cancel a callback previously scheduled with :meth:`after` or + :meth:`after_idle`. + *id* must be an identifier returned by one of those methods; passing a + value that is not such an identifier raises :exc:`ValueError`. + If the callback has already run or been cancelled, this has no effect. + + .. versionchanged:: 3.7 + Passing ``None`` (or any false value) as *id* now raises + :exc:`ValueError`. + + + .. method:: after_idle(func, *args, **kw) + + Schedule the callable *func* to be called, with *args* and *kw* passed to + it, when the Tk main loop next becomes idle, that is, when it has no + other events to process. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_info(id=None) + + If *id* is omitted, return a tuple of the identifiers of all callbacks + currently scheduled with :meth:`after` and :meth:`after_idle` for this + interpreter. + + If *id* is given, it must identify a callback that has not yet run or + been cancelled, and the return value is a tuple ``(script, type)``, where + *script* refers to the function to be called and *type* is either + ``'idle'`` or ``'timer'``. + A :exc:`TclError` is raised if *id* does not exist. + + .. versionadded:: 3.13 + + + .. method:: mainloop(n=0) + + Enter the Tk event loop, which processes events until all windows are + destroyed. + This is normally called once, on the root window, to run the application. + + .. method:: quit() + + Quit the Tcl interpreter, causing :meth:`mainloop` to return. + + .. method:: update() + + Enter the event loop until all pending events, including idle callbacks, + have been processed. + This brings the display up to date and handles any events that are + already queued, then returns. + + .. method:: update_idletasks() + + Enter the event loop until all pending idle callbacks have been called. + This updates the display of windows, for example after geometry changes, + but does not process events caused by the user. + + .. method:: waitvar(name) + :no-typesetting: + + .. method:: wait_variable(name) + + Wait until the Tcl variable *name* is modified, continuing to process + events in the meantime so that the application stays responsive. + *name* is usually a :class:`Variable` instance, such as an + :class:`IntVar` or :class:`StringVar`. + + :meth:`waitvar` is an alias of :meth:`!wait_variable`. + + .. method:: wait_window(window=None) + + Wait until *window* is destroyed, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for the user to finish interacting with a + dialog box. + + .. method:: wait_visibility(window=None) + + Wait until the visibility state of *window* changes, for example when it + first appears on the screen, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for a newly created window to become + visible before acting on it. + + The methods with the ``focus_`` prefix manage the keyboard focus. + + .. method:: focus_set() + :no-typesetting: + + .. method:: focus() + + Direct the keyboard input focus for this widget's display to this widget. + If the application does not currently have the input focus on this + widget's display, the widget is remembered as the focus window for its + top level, and the focus will be redirected to it the next time the + window manager gives the focus to the top level. + :meth:`focus` is an alias of :meth:`!focus_set`, + except on the :class:`Canvas` and + :class:`ttk.Treeview ` widgets, + which provide their own :meth:`!focus` method. + + .. method:: focus_force() + + Direct the keyboard input focus to this widget even if the application + does not currently have the input focus for the widget's display. + This method should be used sparingly, if at all; normally an application + should wait for the window manager to give it the focus rather than + claiming it. + + .. method:: focus_get() + + Return the widget that currently has the keyboard focus in the + application, or ``None`` if no widget in the application has the focus. + Use :meth:`focus_displayof` to work correctly with several displays. + + .. method:: focus_displayof() + + Return the widget that currently has the keyboard focus on the display + where this widget is located, or ``None`` if no widget in the application + has the focus on that display. + + .. method:: focus_lastfor() + + Return the most recent widget to have had the keyboard focus among all + the widgets in the same top level as this widget; this is the widget that + will receive the focus the next time the window manager gives the focus + to the top level. + If no widget in that top level has ever had the focus, or if the most + recent focus widget has been deleted, the top level itself is returned. + + .. method:: tk_focusFollowsMouse() + + Reconfigure Tk to use an implicit focus model in which the focus is set + to a widget whenever the mouse pointer enters it. + This cannot easily be disabled once enabled. + + .. method:: tk_focusNext() + + Return the next widget after this one in the keyboard traversal order, or + ``None`` if there is none. + The traversal order goes first to the next child, then recursively to the + children of that child, and then to the next sibling higher in the + stacking order. + A widget is skipped if its ``takefocus`` option is set to ``0``. + This method is used in the default bindings for the :kbd:`Tab` key. + + .. method:: tk_focusPrev() + + Return the previous widget before this one in the keyboard traversal + order, or ``None`` if there is none. + See :meth:`tk_focusNext` for how the order is defined. + This method is used in the default bindings for the :kbd:`Shift-Tab` key. + + The methods with the ``grab_`` prefix set and query the input grab, which + directs all input events to a single widget. + + .. method:: grab_set() + + Set a local grab on this widget. + A grab confines pointer events to this widget and its descendants: while + the pointer is outside the widget's subtree, button presses and releases + and pointer motion are reported to the grab widget, and windows outside + the subtree become insensitive until the grab is released. + A local grab affects only the grabbing application. + Any grab previously set by this application on the widget's display is + automatically released. + Setting a grab is the usual way to make a dialog modal: while the grab is + in effect the user cannot interact with the other windows of the + application. + + .. method:: grab_set_global() + + Set a global grab on this widget. + A global grab is like the local grab set by :meth:`grab_set`, but it + locks out all other applications on the screen, so that only this + widget's subtree is sensitive to pointer events, and it also grabs the + keyboard. + Use with caution: it is easy to render a display unusable with a global + grab, since other applications stop receiving events until it is + released. + + .. method:: grab_release() + + Release the grab on this widget if there is one; otherwise do nothing. + + .. method:: grab_current() + + Return the widget that currently holds the grab in this application for + this widget's display, or ``None`` if there is no such widget. + + .. method:: grab_status() + + Return ``None`` if no grab is currently set on this widget, ``"local"`` + if a local grab is set, or ``"global"`` if a global grab is set. + + The methods with the ``selection_`` prefix retrieve and manage the X + selection. + + .. method:: selection_clear(**kw) + + Clear the X selection, so that no window owns it anymore. + The selection to clear is given by the keyword argument *selection*, an + atom name such as ``'PRIMARY'`` or ``'CLIPBOARD'``; it defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display on which to operate, and defaults to this widget. + + This is overridden by the :class:`Entry`, :class:`Listbox` and + :class:`Spinbox` widgets, + where :meth:`!selection_clear` clears the widget's own selection instead. + + .. method:: selection_get(**kw) + + Return the contents of the current X selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The keyword argument *type* specifies the form in which the data is to be + returned (the desired conversion target), an atom name such as + ``'STRING'`` or ``'FILE_NAME'``; it defaults to ``STRING``, except on + X11, where ``UTF8_STRING`` is tried first and ``STRING`` is used as a + fallback. + The *displayof* keyword argument names a widget that determines the + display from which to retrieve the selection, and defaults to this + widget. + + .. method:: selection_handle(command, **kw) + + Register *command* as a handler to supply the X selection owned by this + widget when another application requests it. + When the selection is retrieved, *command* is called with two arguments, + the starting character offset and the maximum number of characters to + return, and must return at most that many characters of the selection + starting at that offset; for very long selections it is called repeatedly + with increasing offsets. + The keyword argument *selection* names the selection (default + ``PRIMARY``) and the keyword argument *type* gives the form of the + selection that the handler supplies (such as ``'STRING'`` or + ``'FILE_NAME'``, default ``STRING``). + + .. method:: selection_own(**kw) + + Make this widget the owner of the X selection on its display. + The previous owner, if any, is notified that it has lost the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + + .. method:: selection_own_get(**kw) + + Return the widget in this application that owns the X selection on the + display containing this widget, or ``None`` if no widget in this + application owns the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display to query, and defaults to this widget. + + The methods with the ``clipboard_`` prefix manage the clipboard. + + .. method:: clipboard_append(string, **kw) + + Append *string* to the Tk clipboard and claim ownership of the clipboard + on this widget's display. + Before appending, the clipboard should be emptied with + :meth:`clipboard_clear`; all appends should be completed before returning + to the event loop so that the clipboard is updated atomically. + The keyword argument *type* specifies the form of the data, an atom name + such as ``'STRING'`` or ``'FILE_NAME'`` (default ``STRING``), and the + keyword argument *format* specifies the representation used to transmit + it (default ``STRING``). + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + The contents can be retrieved with :meth:`clipboard_get` or + :meth:`selection_get`. + + .. method:: clipboard_clear(**kw) + + Claim ownership of the clipboard on this widget's display and remove any + previous contents. + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + + .. method:: clipboard_get(**kw) + + Retrieve data from the clipboard on this widget's display. + The keyword argument *type* specifies the form in which the data is to be + returned, an atom name such as ``'STRING'`` or ``'FILE_NAME'``; it + defaults to ``STRING``, except on X11, where ``UTF8_STRING`` is tried + first and ``STRING`` is used as a fallback. + The *displayof* keyword argument names a widget that determines the + display, and defaults to the root window of the application. + This is equivalent to ``selection_get(selection='CLIPBOARD')``. + + The methods with the ``option_`` prefix query and modify the Tk option + database. + + .. method:: option_add(pattern, value, priority=None) + + Add an option to the Tk option database that associates *value* with + *pattern*. + *pattern* consists of names and/or classes separated by asterisks or + dots, in the usual X format. + *priority* is an integer between 0 and 100, or one of the symbolic names + ``'widgetDefault'`` (20), ``'startupFile'`` (40), ``'userDefault'`` (60), + or ``'interactive'`` (80); it defaults to ``interactive``. + + .. method:: option_clear() + + Clear the Tk option database. + Default options from the :envvar:`!RESOURCE_MANAGER` property or the + :file:`.Xdefaults` file are reloaded automatically the next time an + option is added to or removed from the database. + + .. method:: option_get(name, className) + + Return the value of the option matching this widget under *name* and + *className* from the Tk option database, or an empty string if there is + no matching entry. + When several entries match, the one with the highest priority is + returned, and among entries of equal priority the most recently added + one. + + .. method:: option_readfile(fileName, priority=None) + + Read the file named *fileName*, which should have the standard format for + an X resource database such as :file:`.Xdefaults`, and add all the + options it specifies to the Tk option database. + *priority* is interpreted as for :meth:`option_add` and defaults to + ``interactive``. + + .. method:: bell(displayof=0) + + Ring the bell on the display for this widget, using the display's current + bell-related settings, and reset the screen saver for the screen. + If *displayof* is given as a widget, the bell is rung on that widget's + display instead. + + .. method:: tk_setPalette(background, /) + tk_setPalette(*args, **kw) + + Set a new color scheme for all Tk widget elements. + Existing widgets are updated and the option database is changed so that + future widgets use the new colors. + A single color argument is taken as the normal background color, from + which a complete palette is computed. + Alternatively, the arguments may be given as keyword *name*/*value* pairs + naming individual options in the option database. + The recognized option names are ``activeBackground``, + ``activeForeground``, ``background``, ``disabledForeground``, + ``foreground``, ``highlightBackground``, ``highlightColor``, + ``insertBackground``, ``selectColor``, ``selectBackground``, + ``selectForeground``, and ``troughColor``; reasonable defaults are + computed for any that are not specified. + + .. method:: tk_bisque() + + Restore the application's colors to the light brown (bisque) color scheme + used in Tk 3.6 and earlier versions. + Provided for backward compatibility. + + .. method:: tk_strictMotif(boolean=None) + + Query or set whether Tk's look and feel should strictly adhere to Motif. + A true *boolean* value enables strict Motif compliance (for example, no + color change when the mouse passes over a slider). + Return the resulting setting. + + .. method:: tk_appname(name=None) + + Query or set the name used to communicate with this application through + the ``send`` command. + With no argument, return the current name; otherwise change it to *name* + and return the actual name set, which may have a suffix appended to keep + it unique among the applications on the display. + + .. versionadded:: next + + .. method:: tk_useinputmethods(boolean=None, *, displayof=0) + + Query or set whether Tk uses the X Input Methods (XIM) for filtering + events, and return the resulting state. + This is significant only on X11; if XIM support is not available it + always returns ``False``. + + .. versionadded:: next + + .. method:: tk_caret(*, x=None, y=None, height=None) + + Set or query the caret location for the widget's display. + The caret is the per-display insertion position used for global focus + indication (for accessibility) and for placing the input method + (XIM or IME) window. + With no argument, return the current location as a dictionary with the + keys ``'x'``, ``'y'`` and ``'height'``; otherwise update the given + coordinates. + + .. versionadded:: next + + .. method:: tk_scaling(number=None, *, displayof=0) + + Query or set the scaling factor used by Tk to convert between physical + units (such as points, inches or millimeters) and pixels, expressed as + the number of pixels per point (where a point is 1/72 inch). + With no argument, return the current factor; otherwise set it to the + floating-point *number*. + + .. versionadded:: next + + .. method:: tk_inactive(reset=False, *, displayof=0) + + Return the number of milliseconds since the last time the user interacted + with the system, or ``-1`` if the windowing system does not support this. + If *reset* is true, reset the inactivity timer to zero instead and return + ``None``. + + .. versionadded:: next + + The methods with the ``busy_`` prefix manage the busy state of a window, + which shows a busy cursor and ignores user input. + + .. method:: busy(**kw) + :no-typesetting: + + .. method:: busy_hold(**kw) + :no-typesetting: + + .. method:: tk_busy(**kw) + :no-typesetting: + + .. method:: tk_busy_hold(**kw) + + Make this widget appear busy. + A transparent window is placed in front of the widget, so that it and all + of its descendants in the widget hierarchy are blocked from pointer + events and display a busy cursor. + Normally :meth:`update` should be called immediately afterwards to ensure + that the hold operation is in effect before the application starts its + processing. + + The only supported configuration option is *cursor*, the cursor to be + displayed while the widget is busy; it may have any of the values + accepted by :meth:`!configure`. + + :meth:`busy_hold`, :meth:`busy` and :meth:`tk_busy` are aliases of + :meth:`!tk_busy_hold`. + + .. versionadded:: 3.13 + + + .. method:: busy_configure(cnf=None, **kw) + :no-typesetting: + + .. method:: busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_configure(cnf=None, **kw) + + Query or modify the configuration options of the busy window. + The widget must have been previously made busy by :meth:`tk_busy_hold`. + With no arguments, return a dictionary describing all of the available + options; if *cnf* is the name of an option, return a tuple describing + that one option. + Otherwise set the given options to the given values. + Options may have any of the values accepted by :meth:`tk_busy_hold`. + + The option database is referenced through the widget name or class. + For example, if a :class:`Frame` widget named ``frame`` is to be made + busy, the busy cursor can be specified for it by either of the calls:: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + + :meth:`busy_configure`, :meth:`busy_config` and :meth:`tk_busy_config` + are aliases of :meth:`!tk_busy_configure`. + + .. versionadded:: 3.13 + + + .. method:: busy_cget(option) + :no-typesetting: + + .. method:: tk_busy_cget(option) + + Return the current value of the busy configuration *option*. + The widget must have been previously made busy by :meth:`tk_busy_hold`, + and *option* may have any of the values accepted by that method. + + :meth:`busy_cget` is an alias of :meth:`!tk_busy_cget`. + + .. versionadded:: 3.13 + + + .. method:: busy_forget() + :no-typesetting: + + .. method:: tk_busy_forget() + + Make this widget no longer busy, releasing the resources (including the + transparent window) allocated when it was made busy. + User events will again be received by the widget. + These resources are also released when the widget is destroyed. + + :meth:`busy_forget` is an alias of :meth:`!tk_busy_forget`. + + .. versionadded:: 3.13 + + + .. method:: busy_status() + :no-typesetting: + + .. method:: tk_busy_status() + + Return ``True`` if the widget is currently busy, ``False`` otherwise. + + :meth:`busy_status` is an alias of :meth:`!tk_busy_status`. + + .. versionadded:: 3.13 + + + .. method:: busy_current(pattern=None) + :no-typesetting: + + .. method:: tk_busy_current(pattern=None) + + Return a list of widgets that are currently busy. + If *pattern* is given, only busy widgets whose path names match the + pattern are returned. + + :meth:`busy_current` is an alias of :meth:`!tk_busy_current`. + + .. versionadded:: 3.13 + + The methods with the ``winfo_`` prefix retrieve information about windows + managed by Tk. + + .. method:: winfo_atom(name, displayof=0) + + Return the integer identifier for the atom whose name is *name*, creating + a new atom if none exists. + If *displayof* is given, the atom is looked up on the display of that + window; otherwise it is looked up on the display of the application's + main window. + + .. method:: winfo_atomname(id, displayof=0) + + Return the textual name for the atom whose integer identifier is *id*. + This is the inverse of :meth:`winfo_atom`. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_cells() + + Return the number of cells in the colormap for the widget. + + .. method:: winfo_children() + + Return a list containing the widgets that are children of the widget, in + stacking order from lowest to highest. + Toplevel windows are returned as children of their logical parents. + + .. method:: winfo_class() + + Return the class name of the widget. + + .. method:: winfo_colormapfull() + + Return ``True`` if the colormap for the widget is known to be full, + ``False`` otherwise. + + .. method:: winfo_containing(rootX, rootY, displayof=0) + + Return the widget containing the point given by *rootX* and *rootY*, or + ``None`` if no window in this application contains the point. + The coordinates are in screen units in the coordinate system of the root + window. + If *displayof* is given, the coordinates refer to the screen containing + that window; otherwise they refer to the screen of the application's main + window. + + .. method:: winfo_depth() + + Return the depth of the widget, that is, the number of bits per pixel. + + .. method:: winfo_exists() + + Return ``True`` if the widget exists, ``False`` otherwise. + + .. method:: winfo_fpixels(number) + + Return a floating-point value giving the number of pixels in the widget + corresponding to the screen distance *number* (for example, + ``"2.0c"`` or ``"1i"``). + The result may be fractional; for a rounded integer value use + :meth:`winfo_pixels`. + + .. method:: winfo_geometry() + + Return the geometry of the widget, in the form ``widthxheight+x+y``. + All dimensions are in pixels. + An offset can be negative; see :meth:`~Wm.geometry`. + + .. method:: winfo_height() + + Return the height of the widget in pixels. + When a window is first created its height is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqheight`. + + .. method:: winfo_id() + + Return a low-level platform-specific identifier for the widget. + On Unix this is the X window identifier, and on Windows it is the window + handle. + + .. method:: winfo_interps(displayof=0) + + Return a tuple of the names of all Tcl interpreters currently registered + for a particular display. + If *displayof* is given, the return value refers to the display of that + window; otherwise it refers to the display of the application's main + window. + + .. method:: winfo_isdark() + + On macOS and Windows, return ``True`` if the widget is in "dark mode", + and ``False`` otherwise. + Always return ``False`` on X11. + + .. versionadded:: next + + Requires Tk 9.1 or newer. + + .. method:: winfo_ismapped() + + Return ``True`` if the widget is currently mapped, ``False`` otherwise. + + .. method:: winfo_manager() + + Return the name of the geometry manager currently responsible for the + widget, or an empty string if it is not managed by any geometry manager. + + .. method:: winfo_name() + + Return the widget's name within its parent, as opposed to its full path + name. + + .. method:: winfo_parent() + + Return the path name of the widget's parent, or an empty string if the + widget is the main window of the application. + + .. method:: winfo_pathname(id, displayof=0) + + Return the path name of the window whose identifier is *id*. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_pixels(number) + + Return the number of pixels in the widget corresponding to the screen + distance *number* (for example, ``"2.0c"`` or ``"1i"``). + The result is rounded to the nearest integer; for a fractional result use + :meth:`winfo_fpixels`. + + .. method:: winfo_pointerx() + + Return the pointer's *x* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_pointerxy() + + Return the pointer's coordinates as an ``(x, y)`` tuple, in pixels, + relative to the screen's root window (or virtual root, if one is in use). + Both coordinates are ``-1`` if the pointer is not on the same screen as + the widget. + + .. method:: winfo_pointery() + + Return the pointer's *y* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_reqheight() + + Return the widget's requested height in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_reqwidth() + + Return the widget's requested width in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_rgb(color) + + Return an ``(r, g, b)`` tuple of the red, green, and blue intensities, in + the range 0 to 65535, that correspond to *color* in the widget. + *color* may be specified in any of the forms acceptable for a color + option. + + .. method:: winfo_rootx() + + Return the *x* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_rooty() + + Return the *y* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_screen() + + Return the name of the screen associated with the widget, in the form + ``displayName.screenIndex``. + + .. method:: winfo_screencells() + + Return the number of cells in the default colormap for the widget's + screen. + + .. method:: winfo_screendepth() + + Return the depth of the root window of the widget's screen, that is, the + number of bits per pixel. + + .. method:: winfo_screenheight() + + Return the height of the widget's screen in pixels. + + .. method:: winfo_screenmmheight() + + Return the height of the widget's screen in millimeters. + + .. method:: winfo_screenmmwidth() + + Return the width of the widget's screen in millimeters. + + .. method:: winfo_screenvisual() + + Return the default visual class for the widget's screen, one of + ``"directcolor"``, ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, + ``"staticgray"``, or ``"truecolor"``. + + .. method:: winfo_screenwidth() + + Return the width of the widget's screen in pixels. + + .. method:: winfo_server() + + Return a string containing information about the server for the widget's + display. + The exact format of this string may vary from platform to platform. + + .. method:: winfo_toplevel() + + Return the top-of-hierarchy window containing the widget. + In standard Tk this is always a :class:`Toplevel` widget. + + .. method:: winfo_viewable() + + Return ``True`` if the widget and all of its ancestors up through the + nearest toplevel window are mapped, ``False`` otherwise. + + .. method:: winfo_visual() + + Return the visual class for the widget, one of ``"directcolor"``, + ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, ``"staticgray"``, + or ``"truecolor"``. + + .. method:: winfo_visualid() + + Return the X identifier for the visual for the widget. + + .. method:: winfo_visualsavailable(includeids=False) + + Return a list describing the visuals available for the widget's screen. + Each item consists of a visual class (see :meth:`winfo_visual`) followed + by an integer depth. + If *includeids* is true, the X identifier for the visual is also + included. + + .. method:: winfo_vrootheight() + + Return the height of the virtual root window associated with the widget + if there is one; otherwise return the height of the widget's screen. + + .. method:: winfo_vrootwidth() + + Return the width of the virtual root window associated with the widget if + there is one; otherwise return the width of the widget's screen. + + .. method:: winfo_vrootx() + + Return the *x* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_vrooty() + + Return the *y* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_width() + + Return the width of the widget in pixels. + When a window is first created its width is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqwidth`. + + .. method:: winfo_x() + + Return the *x* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: winfo_y() + + Return the *y* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: info_patchlevel() + + Return the Tcl/Tk patch level as a named tuple with the same five fields + as :data:`sys.version_info`: *major*, *minor*, *micro*, *releaselevel* + and *serial*. + *releaselevel* is ``'alpha'``, ``'beta'`` or ``'final'``. + Converting it to a string gives the version in the usual Tcl/Tk notation, + for example ``'9.0.3'`` for a final release or ``'9.1b2'`` for a + pre-release. + + .. versionadded:: 3.11 + + + +.. class:: Wm() + + The :class:`!Wm` mixin provides access to the window manager, allowing an + application to control such things as the title, geometry and icon of a + top-level window, the way it is resized, and how it responds to window + manager protocols. + It is mixed into :class:`Tk` and :class:`Toplevel`, so its methods are + available on every top-level window. + Each method has two equivalent spellings: a short name and a + ``wm_``-prefixed name (for example, :meth:`title` and :meth:`wm_title`). + See also :ref:`tkinter-window-manager`. + + .. method:: wm_aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + :no-typesetting: + + .. method:: aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + + Constrain the aspect ratio (the ratio of width to height) of the window. + If all four arguments are given, the window manager keeps the ratio + between ``minNumer/minDenom`` and ``maxNumer/maxDenom``; passing empty + strings removes any existing restriction. + With no arguments, return a tuple of the four current values, or ``None`` + if no aspect restriction is in effect. + :meth:`wm_aspect` is an alias of :meth:`!aspect`. + + .. method:: wm_attributes(*args, return_python_dict=False, **kwargs) + :no-typesetting: + + .. method:: attributes(*args, return_python_dict=False, **kwargs) + + Query or set platform-specific attributes of the window. + With no arguments, return the platform-specific flags and their values; + pass *return_python_dict* as true to get them as a dictionary. + A single option name such as ``'alpha'`` returns the value of that + option, and options are set using keyword arguments (``alpha=0.5``). + + The available attributes differ by platform. + All platforms support: + + *alpha* + The window's opacity, from ``0.0`` (fully transparent) to ``1.0`` + (opaque). + Where transparency is unsupported the value stays at ``1.0``. + + *appearance* + Whether the window is rendered in dark mode on Windows and macOS: + ``'auto'``, ``'light'`` or ``'dark'`` (this has no effect on X11). + + *fullscreen* + Whether the window takes up the entire screen and has no borders. + + *topmost* + Whether the window is displayed above all other windows. + + Windows additionally supports: + + *disabled* + Whether the window is in a disabled state. + + *toolwindow* + Whether the window uses the tool window style. + + *transparentcolor* + The color that is made fully transparent, or an empty string for none. + + macOS additionally supports: + + *class* + Whether the underlying Aqua window is an ``nswindow`` or an + ``nspanel``; this can only be set before the window is created. + + *modified* + The modification state shown by the window's close button and proxy + icon. + + *notify* + Whether the application's dock icon bounces to request attention. + + *stylemask* + The style mask of the underlying Aqua window, given as a list of bit + names such as ``titled`` or ``resizable``. + + *tabbingid* + The identifier of the tab group that the window belongs to. + + *tabbingmode* + Whether the window may be opened as a tab: ``'auto'``, ``'preferred'`` + or ``'disallowed'``. + + *titlepath* + The path of the file represented by the window's proxy icon. + + *transparent* + Whether the content area is transparent and the window shadow is + turned off. + + X11 additionally supports: + + *type* + The window type, or a list of types in order of preference, that the + window manager should use to interpret the window, such as + ``'dialog'`` or ``'splash'``. + + *zoomed* + Whether the window is maximized. + + .. note:: + + Tk 8.6 added the *type* attribute, and Tk 9.0 added the *appearance*, + *class*, *stylemask*, *tabbingid* and *tabbingmode* attributes. + + On X11 changes are applied asynchronously, so a queried value may not yet + reflect the most recent request. + :meth:`wm_attributes` is an alias of :meth:`!attributes`. + + .. versionchanged:: 3.13 + A single attribute may now be queried by name without the leading + ``-``, and attributes may be set using keyword arguments. + The *return_python_dict* parameter was added. + + .. deprecated:: 3.13 + Setting an attribute by passing the option name (with a leading + ``-``) and its value as two positional arguments, as in + ``w.attributes('-alpha', 0.5)``, is deprecated; use keyword arguments + instead. + + + .. method:: wm_client(name=None) + :no-typesetting: + + .. method:: client(name=None) + + Store *name*, which should be the name of the host on which the + application is running, in the window's ``WM_CLIENT_MACHINE`` property + for use by the window or session manager. + An empty string deletes the property. + With no argument, return the last name set, or an empty string. + :meth:`wm_client` is an alias of :meth:`!client`. + + .. method:: wm_colormapwindows(*wlist) + :no-typesetting: + + .. method:: colormapwindows(*wlist) + + Manipulate the ``WM_COLORMAP_WINDOWS`` property, which tells the window + manager about windows that have private colormaps. + If *wlist* is given, overwrite the property with those windows (their + order is a priority order for installing colormaps). + With no arguments, return the list of windows currently named in the + property. + :meth:`wm_colormapwindows` is an alias of :meth:`!colormapwindows`. + + .. method:: wm_command(value=None) + :no-typesetting: + + .. method:: command(value=None) + + Store *value* in the window's ``WM_COMMAND`` property for use by the + window or session manager; it should be a list giving the words of the + command used to invoke the application. + An empty string deletes the property. + With no argument, return the last value set, or an empty string. + :meth:`wm_command` is an alias of :meth:`!command`. + + .. method:: wm_deiconify() + :no-typesetting: + + .. method:: deiconify() + + Display the window in normal (non-iconified) form by mapping it. + If the window has never been mapped, this ensures it appears de-iconified + when it is first mapped. + On Windows the window is also raised and given the focus. + :meth:`wm_deiconify` is an alias of :meth:`!deiconify`. + + .. method:: wm_focusmodel(model=None) + :no-typesetting: + + .. method:: focusmodel(model=None) + + Set or query the focus model for the window. + *model* is either ``'active'`` (the window claims the input focus for + itself or its descendants, even when the focus is in another application) + or ``'passive'`` (the window relies on the window manager to give it the + focus). + With no argument, return the current model. + The default is ``'passive'``, which is what the :meth:`!focus` command + assumes. + :meth:`wm_focusmodel` is an alias of :meth:`!focusmodel`. + + .. method:: wm_forget(window) + :no-typesetting: + + .. method:: forget(window) + + Unmap *window* from the screen so that it is no longer managed by the + window manager. + A :class:`Toplevel` is then treated like a :class:`Frame`, although its + ``-menu`` configuration is remembered and the menu reappears if the + widget is managed again. + :meth:`wm_forget` is an alias of :meth:`!forget`. + + Not to be confused with :meth:`Pack.forget`. + + .. versionadded:: 3.3 + + .. method:: wm_frame() + :no-typesetting: + + .. method:: frame() + + Return the platform-specific window identifier for the outermost + decorative frame containing the window, if the window manager has + reparented it into such a frame; otherwise return the identifier of the + window itself. + :meth:`wm_frame` is an alias of :meth:`!frame`. + + .. method:: wm_geometry(newGeometry=None) + :no-typesetting: + + .. method:: geometry(newGeometry=None) + + Set or query the geometry of the window. + *newGeometry* has the form ``=widthxheight+x+y``, where any of ``=``, + ``widthxheight`` and the ``+x+y`` position may be omitted. + *width* and *height* are in pixels (or grid units for a gridded window); + a position preceded by ``+`` is measured from the left or top edge of the + screen and one preceded by ``-`` from the right or bottom edge. + An offset can be negative, as in ``'200x100+-9+-8'``, when the window + edge is positioned beyond the corresponding screen edge. + An empty string cancels any user-specified geometry, letting the window + revert to its natural size. + With no argument, return the current geometry as a string of the form + ``'200x200+10+10'``. + :meth:`wm_geometry` is an alias of :meth:`!geometry`. + + .. method:: wm_grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + :no-typesetting: + + .. method:: grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + + Manage the window as a gridded window and define the relationship between + grid units and pixels. + *baseWidth* and *baseHeight* are the numbers of grid units for the + window's internally requested size, and *widthInc* and *heightInc* are + the pixel sizes of a horizontal and vertical grid unit. + Empty strings turn off gridded management. + With no arguments, return a tuple of the four current values, or ``None`` + if the window is not gridded. + :meth:`wm_grid` is an alias of :meth:`!grid`. + + Not to be confused with the grid geometry manager :meth:`Grid.grid`. + + .. method:: wm_group(pathName=None) + :no-typesetting: + + .. method:: group(pathName=None) + + Set or query the leader of a group of related windows. + *pathName* gives the path name of the group leader; the window manager + may, for example, unmap all windows in the group when the leader is + iconified. + An empty string removes the window from any group. + With no argument, return the path name of the current group leader, or an + empty string. + :meth:`wm_group` is an alias of :meth:`!group`. + + .. method:: wm_iconbadge(badge) + :no-typesetting: + + .. method:: iconbadge(badge) + + Set a badge for the window's icon, intended for display in the Dock + (macOS), taskbar (Windows) or app panel (X11). + *badge* may be a positive integer (for example a count of unread + messages) or an exclamation point to denote that attention is needed; + an empty string removes the badge. + On X11 the variable ``::tk::icons::base_icon(window)`` must be set to the + window's icon image for the badge to appear. + :meth:`wm_iconbadge` is an alias of :meth:`!iconbadge`. + + .. versionadded:: next + + Requires Tk 9.0 or newer. + + .. method:: wm_iconbitmap(bitmap=None, default=None) + :no-typesetting: + + .. method:: iconbitmap(bitmap=None, default=None) + + Set or query the bitmap used by the window manager for the window's icon. + *bitmap* names a bitmap in one of the standard forms accepted by Tk; an + empty string cancels the current icon bitmap. + With no argument, return the name of the current icon bitmap, or an empty + string. + On Windows the *default* argument names an icon (for example an ``.ico`` + file) applied to all top-level windows that have no icon of their own. + :meth:`wm_iconbitmap` is an alias of :meth:`!iconbitmap`. + + .. method:: wm_iconify() + :no-typesetting: + + .. method:: iconify() + + Iconify the window. + If the window has not yet been mapped for the first time, arrange for it + to appear in the iconified state when it is eventually mapped. + :meth:`wm_iconify` is an alias of :meth:`!iconify`. + + .. method:: wm_iconmask(bitmap=None) + :no-typesetting: + + .. method:: iconmask(bitmap=None) + + Set or query the bitmap used as a mask for the icon (see + :meth:`iconbitmap`). + Where the mask is zero no icon is displayed; where it is one, the + corresponding bits of the icon bitmap are shown. + An empty string cancels the current mask. + With no argument, return the name of the current icon mask, or an empty + string. + :meth:`wm_iconmask` is an alias of :meth:`!iconmask`. + + .. method:: wm_iconname(newName=None) + :no-typesetting: + + .. method:: iconname(newName=None) + + Set or query the name displayed by the window manager inside the window's + icon. + With no argument, return the current icon name, or an empty string if + none has been set (in which case the window manager normally displays the + window's title). + :meth:`wm_iconname` is an alias of :meth:`!iconname`. + + .. method:: wm_iconphoto(default=False, *images) + :no-typesetting: + + .. method:: iconphoto(default=False, *images) + + Set the titlebar icon for the window from one or more :class:`PhotoImage` + objects given in *images*. + Several images of different sizes (for example 16x16 and 32x32) may be + supplied so that the window manager can choose an appropriate one. + The image data is taken as a snapshot at the time of the call; later + changes to the images are not reflected. + If *default* is true, the icon is also applied to all top-level windows + created in the future. + On macOS only the first image is used. + :meth:`wm_iconphoto` is an alias of :meth:`!iconphoto`. + + .. versionadded:: 3.3 + + .. method:: wm_iconposition(x=None, y=None) + :no-typesetting: + + .. method:: iconposition(x=None, y=None) + + Set or query a hint to the window manager about where the window's icon + should be positioned. + Empty strings cancel an existing hint. + With no arguments, return a tuple of the two current values, or ``None`` + if no hint is in effect. + :meth:`wm_iconposition` is an alias of :meth:`!iconposition`. + + .. method:: wm_iconwindow(pathName=None) + :no-typesetting: + + .. method:: iconwindow(pathName=None) + + Set or query the window used as the icon for the window. + When the window is iconified, *pathName* is mapped to serve as its icon + and unmapped again when it is de-iconified. + An empty string cancels the association. + With no argument, return the path name of the current icon window, or an + empty string. + Not all window managers support icon windows, and the concept is + meaningless on non-X11 platforms. + :meth:`wm_iconwindow` is an alias of :meth:`!iconwindow`. + + .. method:: wm_manage(widget) + :no-typesetting: + + .. method:: manage(widget) + + Make *widget* a stand-alone top-level window, decorated by the window + manager with a title bar and so on. + Only :class:`Frame`, :class:`LabelFrame` and :class:`Toplevel` widgets + may be used (the :mod:`tkinter.ttk` versions are **not** accepted); + passing any other widget type raises an error. + :meth:`wm_manage` is an alias of :meth:`!manage`. + + .. versionadded:: 3.3 + + .. method:: wm_maxsize(width=None, height=None) + :no-typesetting: + + .. method:: maxsize(width=None, height=None) + + Set or query the maximum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no larger than *width* and + *height*. + With no arguments, return a tuple of the current maximum width and + height. + The maximum size defaults to the size of the screen. + :meth:`wm_maxsize` is an alias of :meth:`!maxsize`. + + .. method:: wm_minsize(width=None, height=None) + :no-typesetting: + + .. method:: minsize(width=None, height=None) + + Set or query the minimum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no smaller than *width* and + *height*. + With no arguments, return a tuple of the current minimum width and + height. + The minimum size defaults to one pixel in each dimension. + :meth:`wm_minsize` is an alias of :meth:`!minsize`. + + .. method:: wm_overrideredirect(boolean=None) + :no-typesetting: + + .. method:: overrideredirect(boolean=None) + + Set or query the override-redirect flag for the window. + When this flag is set, the window is ignored by the window manager: it is + not reparented into a decorative frame and the user cannot manipulate it + through the usual window manager controls. + With no argument, return a boolean indicating whether the flag is set, + or ``None`` if it has not been set. + The flag is reliably honored only when the window is first mapped or + remapped from the withdrawn state. + :meth:`wm_overrideredirect` is an alias of :meth:`!overrideredirect`. + + .. method:: wm_positionfrom(who=None) + :no-typesetting: + + .. method:: positionfrom(who=None) + + Set or query the source of the window's current position. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + position was requested by the program or by the user; an empty string + cancels the current source. + With no argument, return the current source, or an empty string if none + has been set. + Tk automatically sets the source to ``'user'`` when :meth:`geometry` is + called, unless it has been set explicitly to ``'program'``. + :meth:`wm_positionfrom` is an alias of :meth:`!positionfrom`. + + .. method:: wm_protocol(name=None, func=None) + :no-typesetting: + + .. method:: protocol(name=None, func=None) + + Register *func* as the handler for the window manager protocol *name*, an + atom such as ``'WM_DELETE_WINDOW'``, ``'WM_SAVE_YOURSELF'`` or + ``'WM_TAKE_FOCUS'``; *func* is then called whenever the window manager + sends a message of that protocol. + Tk installs a default ``WM_DELETE_WINDOW`` handler that destroys the + window, which this method can replace. + If *func* is an empty string, the handler is removed. + With only *name*, return the name of its registered handler command, or + an empty string if none is set (the default ``WM_DELETE_WINDOW`` handler + is not reported); with no arguments, return a tuple of the protocols that + currently have handlers. + :meth:`wm_protocol` is an alias of :meth:`!protocol`. + + .. method:: wm_resizable(width=None, height=None) + :no-typesetting: + + .. method:: resizable(width=None, height=None) + + Control whether the user may interactively resize the window. + *width* and *height* are boolean values that determine whether the + window's width and height may be changed. + With no arguments, return a tuple of two ``0``/``1`` values indicating + whether each dimension is currently resizable. + By default a window is resizable in both dimensions. + :meth:`wm_resizable` is an alias of :meth:`!resizable`. + + .. method:: wm_sizefrom(who=None) + :no-typesetting: + + .. method:: sizefrom(who=None) + + Set or query the source of the window's current size. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + size was requested by the program or by the user; an empty string cancels + the current source. + With no argument, return the current source, or an empty string if none + has been set. + :meth:`wm_sizefrom` is an alias of :meth:`!sizefrom`. + + .. method:: wm_stackorder(relation=None, window=None) + :no-typesetting: + + .. method:: stackorder(relation=None, window=None) + + Query the stacking order of top-level windows. + With no arguments, return a list of the mapped top-level widgets in + stacking order, from lowest to highest, recursively including this + window's top-level children. + If *relation* is ``'isabove'`` or ``'isbelow'`` and *window* is another + top-level, return ``True`` if this window is respectively above or below + *window* in the stacking order, and ``False`` otherwise. + :meth:`wm_stackorder` is an alias of :meth:`!stackorder`. + + .. versionadded:: next + + .. method:: wm_state(newstate=None) + :no-typesetting: + + .. method:: state(newstate=None) + + Set or query the state of the window. + With no argument, return the current state: one of ``'normal'``, + ``'iconic'``, ``'withdrawn'``, ``'icon'`` or, on Windows and macOS only, + ``'zoomed'``. + ``'iconic'`` refers to a window that has been iconified, while ``'icon'`` + refers to a window serving as the icon for another window (see + :meth:`iconwindow`); the ``'icon'`` state cannot be set. + :meth:`wm_state` is an alias of :meth:`!state`. + + Not to be confused with :meth:`ttk.Widget.state + `. + + .. method:: wm_title(string=None) + :no-typesetting: + + .. method:: title(string=None) + + Set or query the title for the window, which the window manager should + display in the window's title bar. + With no argument, return the current title. + The title defaults to the window's name. + :meth:`wm_title` is an alias of :meth:`!title`. + + .. method:: wm_transient(master=None) + :no-typesetting: + + .. method:: transient(master=None) + + Mark the window as a transient window (such as a pull-down menu or + dialog) working on behalf of *master*, the path name of another top-level + window. + An empty string clears the transient status. + With no argument, return the path name of the current master, or an empty + string. + A transient window mirrors state changes in its master and may be + decorated differently by the window manager; it is an error to make a + window a transient of itself. + :meth:`wm_transient` is an alias of :meth:`!transient`. + + .. method:: wm_withdraw() + :no-typesetting: + + .. method:: withdraw() + + Withdraw the window from the screen, unmapping it and causing the window + manager to forget about it. + If the window has never been mapped, it is instead mapped in the + withdrawn state. + It is sometimes necessary to withdraw a window and then re-map it (for + example with :meth:`deiconify`) to make some window managers notice + changes to window attributes. + :meth:`wm_withdraw` is an alias of :meth:`!withdraw`. + + +.. class:: Pack() + + Geometry manager that arranges widgets by packing them against the sides of + their container. + The :class:`!Pack` mix-in is inherited by all widgets (through + :class:`Widget`) and provides the methods for managing a widget with the + *pack* geometry manager. + See also :ref:`tkinter-geometry-management`. + + .. note:: + + :class:`Pack`, :class:`Place` and :class:`Grid` all define the short + method names :meth:`!forget`, :meth:`!info`, :meth:`!slaves`, + :meth:`!content` and :meth:`!propagate`. + On a widget the bare names resolve to the *pack* manager's versions, + since :class:`Pack` and :class:`Misc` precede :class:`Place` and + :class:`Grid` in the method resolution order, + whatever manager actually manages the widget; + and :meth:`!configure`/:meth:`!config` configure the widget's options, + not its geometry. + Use the explicit ``pack_*``, ``grid_*`` and ``place_*`` methods + (and ``pack``, ``grid``, ``place`` for geometry configuration) + to act on a specific geometry manager. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: pack_configure(cnf={}, **kw) + pack(cnf={}, **kw) + + Pack the widget inside its container, positioning it relative to the + siblings already packed there. + The supported options are: + + *side* + Which side of the container to pack the widget against: ``'top'`` (the + default), ``'bottom'``, ``'left'`` or ``'right'``. + + *fill* + Whether to stretch the widget to fill its parcel: ``'none'`` (the + default), ``'x'``, ``'y'`` or ``'both'``. + + *expand* + Whether the widget should expand to consume any extra space in its + container (a boolean, default false). + + *anchor* + Where to position the widget in its parcel when the parcel is larger + than the widget: an anchor such as ``'n'`` or ``'sw'`` (default + ``'center'``). + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *after* + Pack the widget after the given widget in the packing order, using the + same container. + + *before* + Pack the widget before the given widget in the packing order, using + the same container. + + *in_* + The container in which to pack the widget; it defaults to the parent + widget. + + :meth:`pack`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!pack_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: pack_forget() + + Unmap the widget and remove it from the packing order, forgetting its + packing options. + It can be packed again later with :meth:`pack_configure`. + :meth:`forget` is an alias of :meth:`!pack_forget`, + except on :class:`PanedWindow`, + :class:`ttk.Notebook ` and + :class:`ttk.PanedWindow `, + which provide their own :meth:`!forget` method. + + Not to be confused with :meth:`Wm.forget`. + + .. method:: info() + :no-typesetting: + + .. method:: pack_info() + + Return a dictionary of the widget's current packing options. + :meth:`info` is an alias of :meth:`!pack_info`. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Same as :meth:`Misc.pack_propagate`, treating this widget as a container: + enable or disable geometry propagation. + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Same as :meth:`Misc.pack_slaves`: return the list of widgets packed in + this widget. + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: content() + :no-typesetting: + + .. method:: pack_content() + + Same as :meth:`Misc.pack_content`. + :meth:`content` is an alias of :meth:`!pack_content`. + + .. versionadded:: 3.15 + + +.. class:: Place() + + Geometry manager that places widgets at explicit positions and sizes within + their container. + The :class:`!Place` mix-in is inherited by all widgets (through + :class:`Widget`). + See also :ref:`tkinter-geometry-management`. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: place_configure(cnf={}, **kw) + place(cnf={}, **kw) + + Place the widget inside its container at an absolute or relative + position. + The supported options are: + + *x*, *y* + The absolute horizontal and vertical position of the widget's anchor + point, as a screen distance (default ``0``). + + *relx*, *rely* + The horizontal and vertical position of the widget's anchor point as a + fraction of the container's width and height, where ``0.0`` is the + left or top edge and ``1.0`` is the right or bottom edge. + If both the absolute and the relative option are given, their values + are summed. + + *anchor* + Which point of the widget is placed at the given position: an anchor + such as ``'n'`` or ``'se'`` (default ``'nw'``). + + *width*, *height* + The absolute width and height of the widget, as a screen distance. + By default the widget's requested size is used. + + *relwidth*, *relheight* + The width and height of the widget as a fraction of the container's + width and height. + If both the absolute and the relative option are given, their values + are summed. + + *bordermode* + How the container's border affects placement: ``'inside'`` (the + default) measures the area inside the border, ``'outside'`` measures + the area including the border, and ``'ignore'`` uses the official X + area. + + *in_* + The container relative to which the widget is placed; it must be the + widget's parent or a descendant of the parent, and defaults to the + parent. + + :meth:`place`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!place_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: place_forget() + + Unmap the widget and remove it from the placement, forgetting its place + options. + + .. method:: info() + :no-typesetting: + + .. method:: place_info() + + Return a dictionary of the widget's current place options. + + .. method:: slaves() + :no-typesetting: + + .. method:: place_slaves() + + Same as :meth:`Misc.place_slaves`: return the list of widgets placed in + this widget. + + .. method:: content() + :no-typesetting: + + .. method:: place_content() + + Same as :meth:`Misc.place_content`. + + .. versionadded:: 3.15 + + +.. class:: Grid() + + Geometry manager that arranges widgets in a two-dimensional grid of rows and + columns within their container. + The :class:`!Grid` mix-in is inherited by all widgets (through + :class:`Widget`). + See also :ref:`tkinter-geometry-management`. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: grid_configure(cnf={}, **kw) + grid(cnf={}, **kw) + + Position the widget in a cell of its container's grid. + + Not to be confused with :meth:`Wm.grid`. + + The supported options are: + + *row*, *column* + The row and column of the cell to place the widget in, counting from + ``0``. + *column* defaults to the column after the previous widget placed in + the same :meth:`!grid_configure` call (or ``0``), and *row* defaults + to the next empty row. + + *rowspan*, *columnspan* + The number of rows and columns the widget should span (default ``1``). + + *sticky* + How to position or stretch the widget when its cell is larger than the + widget: a string containing zero or more of the characters ``'n'``, + ``'s'``, ``'e'`` and ``'w'``, naming the cell sides the widget sticks + to. + Specifying both ``'n'`` and ``'s'`` (or ``'e'`` and ``'w'``) stretches + the widget to fill the height (or width) of the cell. + The default is ``''``, which centers the widget at its requested size. + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *in_* + The container in whose grid to place the widget; it defaults to the + parent widget. + + :meth:`grid`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!grid_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: grid_forget() + + Unmap the widget and remove it from the grid, forgetting its grid + options. + + .. method:: grid_remove() + + Unmap the widget and remove it from the grid, but remember its grid + options so that it is restored to the same cell if it is gridded again. + + .. method:: info() + :no-typesetting: + + .. method:: grid_info() + + Return a dictionary of the widget's current grid options. + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Same as :meth:`Misc.grid_bbox`. + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_columnconfigure`: query or set the options (such + as *weight*, *minsize*, *pad* and *uniform*) of a grid column. + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_rowconfigure`: query or set the options of a + grid row. + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: location(x, y) + :no-typesetting: + + .. method:: grid_location(x, y) + + Same as :meth:`Misc.grid_location`: return the ``(column, row)`` of the + cell that covers the pixel at *x*, *y*. + :meth:`location` is an alias of :meth:`!grid_location`. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Same as :meth:`Misc.grid_size`: return a ``(columns, rows)`` tuple giving + the size of the grid. + :meth:`size` is an alias of :meth:`!grid_size`, + except on :class:`Listbox` and + :class:`ttk.Treeview `, + which provide their own :meth:`!size` method. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: grid_propagate() + grid_propagate(flag) + + Same as :meth:`Misc.grid_propagate`. + + .. method:: slaves(row=None, column=None) + :no-typesetting: + + .. method:: grid_slaves(row=None, column=None) + + Same as :meth:`Misc.grid_slaves`: return the widgets managed in the grid, + optionally restricted to a *row* and/or *column*. + + .. method:: content(row=None, column=None) + :no-typesetting: + + .. method:: grid_content(row=None, column=None) + + Same as :meth:`Misc.grid_content`. + + .. versionadded:: 3.15 + + +.. class:: XView() + + Mix-in providing the horizontal-scrolling interface shared by widgets such + as :class:`Entry`, :class:`Canvas`, :class:`Listbox`, :class:`Text` and + :class:`Spinbox`. + A widget's :meth:`xview` method is registered as the *command* of a + horizontal :class:`Scrollbar`. + + .. method:: xview(*args) + + Query or change the horizontal position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``xview`` widget command and + are usually generated by a scrollbar; :meth:`xview_moveto` and + :meth:`xview_scroll` provide a more convenient interface. + + .. method:: xview_moveto(fraction) + + Adjust the view so that *fraction* of the total width of the document is + off-screen to the left. + *fraction* is a number between 0 and 1. + + .. method:: xview_scroll(number, what) + + Shift the view left or right by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + left and a positive one scrolls right. + + +.. class:: YView() + + Mix-in providing the vertical-scrolling interface shared by widgets such as + :class:`Canvas`, :class:`Listbox` and :class:`Text`. + A widget's :meth:`yview` method is registered as the *command* of a vertical + :class:`Scrollbar`. + + .. method:: yview(*args) + + Query or change the vertical position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``yview`` widget command, + usually generated by a scrollbar; :meth:`yview_moveto` and + :meth:`yview_scroll` provide a more convenient interface. + + .. method:: yview_moveto(fraction) + + Adjust the view so that *fraction* of the total height of the document is + off-screen above the top. + *fraction* is a number between 0 and 1. + + .. method:: yview_scroll(number, what) + + Shift the view up or down by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + up and a positive one scrolls down. + + +.. class:: BaseWidget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for all widgets. + It inherits from :class:`Misc` and adds the machinery that creates the + underlying Tk widget; application code normally uses :class:`Widget` or a + concrete widget class rather than instantiating :class:`!BaseWidget` + directly. + + .. method:: destroy() + + Destroy this widget and all of its children, removing the corresponding + Tk widgets and deleting the associated Tcl commands. + + +.. class:: Widget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for the standard widgets. + It combines :class:`BaseWidget` with the geometry-manager mix-ins + :class:`Pack`, :class:`Place` and :class:`Grid`, so that every widget can be + managed by any of the three geometry managers. + The concrete widget classes (:class:`Button`, :class:`Label`, and so on) + derive from :class:`!Widget`. + + +Toplevel widgets +^^^^^^^^^^^^^^^^ + +.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) + + Construct a toplevel Tk widget, which is usually the main window of an + application, and initialize a Tcl interpreter for this widget. Each + instance has its own associated Tcl interpreter. + Inherits from :class:`Misc` and :class:`Wm`. + + To create a Tcl interpreter without initializing the Tk subsystem, use the + :func:`Tcl` factory function instead. + + The :class:`Tk` class is typically instantiated using all default values. + However, the following keyword arguments are currently recognized: + + *screenName* + When given (as a string), sets the :envvar:`DISPLAY` environment + variable. (X11 only) + *baseName* + Name of the profile file. By default, *baseName* is derived from the + program name (``sys.argv[0]``). + *className* + Name of the widget class. Used as a profile file and also as the name + with which Tcl is invoked (*argv0* in *interp*). + *useTk* + If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` + function sets this to ``False``. + *sync* + If ``True``, execute all X server commands synchronously, so that errors + are reported immediately. Can be used for debugging. (X11 only) + *use* + Specifies the *id* of the window in which to embed the application, + instead of it being created as an independent toplevel window. *id* must + be specified in the same way as the value for the -use option for + toplevel widgets (that is, it has a form like that returned by + :meth:`~Misc.winfo_id`). + + Note that on some platforms this will only work correctly if *id* refers + to a Tk frame or toplevel that has its -container option enabled. + + :class:`Tk` reads and interprets profile files, named + :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl + interpreter and calls :func:`exec` on the contents of + :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the + profile files is the :envvar:`HOME` environment variable or, if that + isn't defined, then :data:`os.curdir`. + + .. note:: + + On Windows, creating a Tcl interpreter (by instantiating :class:`Tk` or + calling :func:`Tcl`) sets the :envvar:`HOME` environment variable for + the process, if it is not already set, to ``%HOMEDRIVE%%HOMEPATH%`` (or + :envvar:`USERPROFILE`, or ``c:\``). This is done by Tcl and can affect + other code that reads :envvar:`HOME`. + + .. attribute:: tk + + The Tk application object created by instantiating :class:`Tk`. This + provides access to the Tcl interpreter. Each widget that is attached + the same instance of :class:`Tk` has the same value for its :attr:`tk` + attribute. + + .. attribute:: master + + The widget object that contains this widget. + For :class:`Tk`, the :attr:`!master` is :const:`None` because it is the + main window. + The terms *master* and *parent* are similar and sometimes used + interchangeably as argument names; however, calling + :meth:`~Misc.winfo_parent` returns a string of the widget name whereas + :attr:`!master` returns the object. + *parent*/*child* reflects the tree-like relationship while *master* (or + *container*)/*content* reflects the container structure. + + .. attribute:: children + + The immediate descendants of this widget as a :class:`dict` with the + child widget names as the keys and the child instance objects as the + values. + + .. method:: destroy() + + Destroy this and all descendant widgets and, for the main window, end the + connection to the underlying Tcl interpreter. + + .. method:: loadtk() + + Finish loading and initializing the Tk subsystem. + This is needed only when the interpreter was created without Tk (for + example through :func:`Tcl`); it is called automatically when *useTk* is + true. + + .. method:: readprofile(baseName, className) + + Read and source the user's profile files :file:`.{className}.tcl` and + :file:`.{baseName}.tcl` into the Tcl interpreter, and execute the + corresponding :file:`.{className}.py` and :file:`.{baseName}.py` files. + This is called during initialization; see the description of the + constructor above. + + .. method:: report_callback_exception(exc, val, tb) + + Report a callback exception. + This is called when an exception propagates out of a Tkinter callback; + *exc*, *val* and *tb* are the exception type, value and traceback as + returned by :func:`sys.exc_info`. + The default implementation prints a traceback to :data:`sys.stderr`. + It can be overridden to customize error handling, for example to display + the traceback in a dialog. + + +.. class:: Toplevel(master=None, cnf={}, **kw) + + A :class:`!Toplevel` widget is a top-level window, similar to a + :class:`Frame` except that its X parent is the root window of a screen + rather than its logical parent. + Its primary purpose is to serve as a container for dialog boxes and other + collections of widgets; its only visible features are its background and an + optional 3-D border. + Notable options include *menu*, which installs a :class:`Menu` as the + window's menubar. + Inherits from :class:`BaseWidget` and :class:`Wm`, so a toplevel is managed + by the window manager. + Refer to the Tk ``toplevel`` manual page for the full list of options. + + +Widget classes +^^^^^^^^^^^^^^ + +.. class:: Button(master=None, cnf={}, **kw) + + A :class:`!Button` widget displays a textual string, bitmap or image and + invokes a command when the user presses it (by clicking mouse button 1 over + the button or, when the button has focus, by pressing the space key). + Inherits from :class:`Widget`. + In addition to the standard widget options, a button accepts the options + documented in the Tk ``button`` manual page, such as *command* (the callback + invoked when the button is pressed), *textvariable*, *state* and *default*. + + .. method:: invoke() + + Invoke the command associated with the button, if there is one, and + return its result, or an empty string if no command is associated with + the button. + This is ignored if the button's state is ``disabled``. + + .. method:: flash() + + Flash the button by redisplaying it several times, alternating between + the active and normal colors. + At the end of the flash the button is left in the same normal or active + state as when the method was called. + This is ignored if the button's state is ``disabled``. + + +.. class:: Canvas(master=None, cnf={}, **kw) + + A :class:`!Canvas` widget implements structured graphics. + It displays any number of *items*, such as arcs, lines, ovals, polygons, + rectangles, text, bitmaps, images and embedded windows, which may be drawn, + moved, re-colored and bound to events. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``canvas`` manual page for the full list of widget and item + options. + + Each item has a unique integer *id*, assigned when it is created, and zero + or more string *tags*. + A tag is an arbitrary string that does not have the form of an integer; the + same tag may be shared by many items, which makes tags convenient for + grouping items. + The special tag ``'all'`` matches every item in the canvas, and + ``'current'`` matches the topmost item under the mouse pointer. + Most methods take a *tagOrId* argument that may be an integer id naming a + single item, or a tag naming zero or more items; as described in the Tk + ``canvas`` manual page, a tag may also be a logical expression of tags + combined with the operators ``&&``, ``||``, ``^``, ``!`` and parentheses. + When a method that operates on a single item is given a *tagOrId* matching + several items, it normally uses the lowest matching item in the display + list. + + The items are kept in a *display list* that determines drawing order: items + later in the list are drawn on top of earlier ones. + A newly created item is placed at the top of the list; the order can be + changed with :meth:`tag_raise` and :meth:`tag_lower`. + + .. method:: tk_print() + + Print the contents of the canvas using the native print dialog. + Requires Tk 8.7/9.0 or newer. + + .. versionadded:: next + + .. method:: create_arc(*args, **kw) + create_bitmap(*args, **kw) + create_image(*args, **kw) + create_line(*args, **kw) + create_oval(*args, **kw) + create_polygon(*args, **kw) + create_rectangle(*args, **kw) + create_text(*args, **kw) + create_window(*args, **kw) + + Create a new item of the corresponding type and return its integer id. + Each method is called as ``create_TYPE(coord..., **options)``: the + leading positional arguments give the coordinates that define the item + (as separate numbers, as a single sequence of numbers, or as coordinate + pairs), and the keyword arguments set item-specific options. + Coordinates and screen distances may be given as numbers (interpreted as + pixels) or as strings with a unit suffix (``'m'``, ``'c'``, ``'i'`` or + ``'p'`` for millimetres, centimetres, inches or printer's points), but + are always stored and returned in pixels. + + The item types are: ``arc`` (an arc-shaped region that is a section of an + oval, defined by two diagonally opposite corners ``x1, y1, x2, y2`` of + the enclosing rectangle); ``bitmap`` (a two-color bitmap positioned at a + point ``x, y``); ``image`` (a Tk image positioned at a point ``x, y``); + ``line`` (a line or curve through the points ``x1, y1, ..., xn, yn``); + ``oval`` (a circle or ellipse inscribed in the rectangle + ``x1, y1, x2, y2``); ``polygon`` (a closed polygon through the points + ``x1, y1, ..., xn, yn``); ``rectangle`` (a rectangle with corners + ``x1, y1, x2, y2``); ``text`` (a string of text positioned at a point + ``x, y``); and ``window`` (a child widget embedded in the canvas at a + point ``x, y``, specified with the *window* option). + + Most item types accept a common set of *standard item options*, plus a + few options specific to each type. + Option names are passed as keyword arguments, without the leading + hyphen. + + The standard item options are: + + *fill* + The color used to fill the item's interior, or to draw a *line* + item or the characters of a *text* item. + An empty string (the default for all types except *line* and *text*) + leaves the item unfilled. + + *outline* + The color used to draw the item's outline. + An empty string draws no outline. + + *width* + The width of the outline, defaulting to ``1.0``. + Has no effect if *outline* is empty. + + *dash* + A dash pattern for the outline, given either as a sequence of + segment lengths in pixels or as a string of the characters + ``'.'``, ``','``, ``'-'``, ``'_'`` and space. + An empty pattern (the default) draws a solid outline. + + *dashoffset* + The starting offset in pixels into the *dash* pattern. + Ignored if there is no *dash* pattern. + + *stipple* + A bitmap used as a stipple pattern when filling the item. + Only well supported on X11. + + *outlinestipple* + A bitmap used as a stipple pattern when drawing the outline. + Has no effect if *outline* is empty. + + *offset*, *outlineoffset* + The offset of the fill and outline stipple patterns, given as + ``'x,y'`` or as a side such as ``'n'``, ``'se'`` or ``'center'``. + Stipple offsets are only supported on X11. + + *state* + Overrides the canvas state for this item; one of ``'normal'``, + ``'disabled'`` or ``'hidden'``. + + *tags* + A single tag or a sequence of tags to associate with the item, + replacing any existing tags. + + Many of these options have *active...* and *disabled...* variants + (such as *activefill*, *disabledfill*, *activewidth*, *disableddash*, + *activeoutline*, *disabledstipple*) that override the base option when + the item is the active item (under the mouse pointer) or is in the + disabled state. + + The following item types support additional options. + + For ``arc`` items: + + *start* + The start of the arc's angular range, in degrees measured + counter-clockwise from the 3-o'clock position. + + *extent* + The size of the angular range, in degrees counter-clockwise from + *start*. + + *style* + How the arc is drawn: ``'pieslice'`` (the default), ``'chord'`` or + ``'arc'``. + + For ``line`` items: + + *arrow* + Where to draw arrowheads: ``'none'`` (the default), ``'first'``, + ``'last'`` or ``'both'``. + + *arrowshape* + A sequence of three distances describing the shape of the + arrowheads. + + *capstyle* + How line ends are drawn: ``'butt'`` (the default), ``'projecting'`` + or ``'round'``. + + *joinstyle* + How line vertices are drawn: ``'round'`` (the default), ``'bevel'`` + or ``'miter'``. + + *smooth* + The smoothing method: a false value (the default) for no smoothing, + or ``'true'``/``'bezier'`` or ``'raw'`` to draw the line as a curve. + + *splinesteps* + The number of line segments approximating each spline when + *smooth* is enabled. + + For ``polygon`` items: + + *joinstyle*, *smooth*, *splinesteps* + As for ``line`` items, applied to the polygon's outline. + + For ``text`` items: + + *text* + The string to display; newline characters start new lines. + + *font* + The font used for the text. + + *justify* + How lines are justified: ``'left'`` (the default), ``'right'`` or + ``'center'``. + + *anchor* + How the text is positioned relative to its point, defaulting to + ``'center'``. + + *width* + The maximum line length; if non-zero, lines are wrapped at spaces. + + *angle* + How many degrees to rotate the text counter-clockwise about its + positioning point, from ``0.0`` to ``360.0`` (default ``0.0``). + + *underline* + The index of a character to underline, or ``-1`` for none. + + For ``bitmap`` items: + + *bitmap* + The bitmap to display. + + *anchor* + How the bitmap is positioned relative to its point. + + *background*, *foreground* + The colors used for the bitmap's ``0`` and ``1`` pixels; an empty + *background* makes the ``0`` pixels transparent. + Both have *active...* and *disabled...* variants, and *bitmap* has + *activebitmap* and *disabledbitmap* variants. + + For ``image`` items: + + *image* + The Tk image to display, previously created with the image + protocols. + + *anchor* + How the image is positioned relative to its point. + + Both options have *active...* and *disabled...* variants + (*activeimage*, *disabledimage*) used in the active and disabled + states. + + For ``window`` items: + + *window* + The widget to embed; it must be a child of the canvas or of one of + its ancestors, and may not be a top-level window. + + *anchor* + How the window is positioned relative to its point. + + *width*, *height* + The size to assign to the window; if zero (the default), the window + is given its requested size. + + ``oval`` and ``rectangle`` items have no type-specific options; they + use only the standard item options. + + .. note:: + + Tk 8.6 added the *angle* option and Tk 9.0 added the *underline* + option for ``text`` items. + + .. method:: coords(tagOrId) + coords(tagOrId, coordList, /) + coords(tagOrId, /, *coordList) + + Query or modify the coordinates of an item. + With only *tagOrId*, return a list of the floating-point coordinates of + the item given by *tagOrId* (the first matching item if it matches + several). + Given new coordinates, replace the coordinates of that item with them; + like the ``create_*`` methods, the coordinates may be given as separate + numbers, as a single sequence, or as coordinate pairs. + The returned coordinates are always in pixels, regardless of the units + used to specify them; for rectangles, ovals and arcs they are ordered + left, top, right, bottom. + + .. versionchanged:: 3.12 + The arguments are now flattened: the coordinates may be given as + separate arguments, as a single sequence, or grouped in pairs, like + the ``create_*`` methods. + + + .. method:: move(tagOrId, xAmount, yAmount, /) + + Move each of the items given by *tagOrId* in the canvas coordinate space + by adding *xAmount* to every x-coordinate and *yAmount* to every + y-coordinate of the item. + + .. method:: moveto(tagOrId, x='', y='') + + Move the items given by *tagOrId* so that the first coordinate pair (the + upper-left corner of the bounding box) of the lowest matching item is at + position (*x*, *y*). + *x* or *y* may be an empty string, in which case the corresponding + coordinate is unchanged. + All matching items keep their positions relative to each other. + + .. versionadded:: 3.8 + + + .. method:: scale(tagOrId, xOrigin, yOrigin, xScale, yScale, /) + + Rescale the coordinates of all items given by *tagOrId* in canvas + coordinate space. + Each x-coordinate is adjusted so that its distance from *xOrigin* changes + by a factor of *xScale*, and each y-coordinate so that its distance from + *yOrigin* changes by a factor of *yScale* (a factor of ``1.0`` leaves the + coordinate unchanged). + + .. method:: rotate(tagOrId, xOrigin, yOrigin, angle, /) + + Rotate the coordinates of all items given by *tagOrId* in canvas + coordinate space about the origin (*xOrigin*, *yOrigin*) by *angle* + degrees anticlockwise. + Negative values of *angle* rotate clockwise. + + .. versionadded:: next + + .. method:: delete(*tagOrIds) + + Delete each of the items given by the *tagOrIds* arguments. + + .. method:: dchars(tagOrId, first, /) + dchars(tagOrId, first, last, /) + + Delete from each of the items given by *tagOrId* the characters (for text + items) or coordinates (for line and polygon items) in the range from + *first* to *last* inclusive; *last* defaults to *first*. + Items that do not support indexing ignore this operation. + + .. method:: insert(tagOrId, beforeThis, string, /) + + Insert *string* into each of the items given by *tagOrId* just before the + character or coordinate whose index is *beforeThis*. + For line and polygon items *string* must be a valid sequence of + coordinates. + + .. method:: rchars(tagOrId, first, last, string, /) + + Replace the characters (for text items) or coordinates (for line and + polygon items) in the range from *first* to *last* inclusive of each of + the items given by *tagOrId* with *string*. + For line and polygon items *string* must be a valid sequence of + coordinates. + Items that do not support indexing ignore this operation. + + .. versionadded:: next + + .. method:: itemcget(tagOrId, option) + + Return the current value of the configuration option *option* for the + item given by *tagOrId* (the lowest matching item if it matches several). + This is like :meth:`~Misc.cget` but applies to an individual item. + + .. method:: itemconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(tagOrId, cnf=None, **kw) + + Query or modify the configuration options of the items given by + *tagOrId*. + This mirrors :meth:`~Misc.configure`, except that it applies to + individual items rather than to the canvas as a whole. + With no options, it returns a dictionary describing the current options + of the first matching item; otherwise it sets the given options on every + matching item. + The legal options are those accepted by the corresponding ``create_*`` + method. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: type(tagOrId) + + Return the type of the item given by *tagOrId* (the first matching item + if it matches several), such as ``'rectangle'`` or ``'text'``, or + ``None`` if *tagOrId* does not match any item. + + .. method:: gettags(tagOrId, /) + + Return a tuple of the tags associated with the item given by *tagOrId* + (the first matching item in display-list order if it matches several). + Return an empty tuple if no item matches or the item has no tags. + + .. method:: dtag(tagOrId, /) + dtag(tagOrId, tagToDelete, /) + + Remove the tag *tagToDelete* (which defaults to *tagOrId*) from each of + the items given by *tagOrId*. + Items that do not have that tag are unaffected. + + .. method:: addtag(newtag, searchSpec, /, *args) + + Add the tag *newtag* to each item selected by the search specification + *searchSpec* (and any further *args*). + *searchSpec* is one of ``'above'``, ``'all'``, ``'below'``, + ``'closest'``, ``'enclosed'``, ``'overlapping'`` or ``'withtag'``; the + ``addtag_*`` methods below are convenient wrappers that supply each of + these forms. + + .. method:: addtag_above(newtag, tagOrId) + + Add the tag *newtag* to the item just above (after) *tagOrId* in the + display list. + + .. method:: addtag_all(newtag) + + Add the tag *newtag* to all items in the canvas. + + .. method:: addtag_below(newtag, tagOrId) + + Add the tag *newtag* to the item just below (before) *tagOrId* in the + display list. + + .. method:: addtag_closest(newtag, x, y, halo=None, start=None) + + Add the tag *newtag* to the item closest to the point (*x*, *y*). + If *halo* is given, any item within that distance of the point is treated + as overlapping it. + If *start* is given (a tag or id), select the topmost closest item that + lies below *start* in the display list, which can be used to step through + all the closest items. + + .. method:: addtag_enclosed(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= *y2*. + + .. method:: addtag_overlapping(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item that overlaps or is enclosed within + the rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= + *y2*. + + .. method:: addtag_withtag(newtag, tagOrId) + + Add the tag *newtag* to every item given by *tagOrId*. + + .. method:: find(searchSpec, /, *args) + + Return a tuple of the ids of all items selected by the search + specification *searchSpec* (and any further *args*), in stacking order + with the lowest item first. + The search specification has any of the forms accepted by :meth:`addtag`. + The ``find_*`` methods below are more convenient wrappers around it. + + .. method:: find_above(tagOrId) + + Return a tuple containing the id of the item just above *tagOrId* in the + display list. + + .. method:: find_all() + + Return a tuple of the ids of all items in the canvas, in stacking order. + + .. method:: find_below(tagOrId) + + Return a tuple containing the id of the item just below *tagOrId* in the + display list. + + .. method:: find_closest(x, y, halo=None, start=None) + + Return a tuple containing the id of the item closest to the point (*x*, + *y*). + *halo* and *start* are interpreted as for :meth:`addtag_closest`. + + .. method:: find_enclosed(x1, y1, x2, y2) + + Return a tuple of the ids of all items completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_overlapping(x1, y1, x2, y2) + + Return a tuple of the ids of all items that overlap or are enclosed + within the rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_withtag(tagOrId) + + Return a tuple of the ids of all items given by *tagOrId*. + + .. method:: lift(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tkraise(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tag_raise(tagOrId, aboveThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just above the item given by *aboveThis*, or to the top of the display + list if *aboveThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items, whose stacking order is + controlled by :meth:`Misc.tkraise` and :meth:`Misc.lower` instead. + :meth:`lift` and :meth:`tkraise` are aliases of :meth:`!tag_raise`. + + .. method:: lower(tagOrId, belowThis=None, /) + :no-typesetting: + + .. method:: tag_lower(tagOrId, belowThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just below the item given by *belowThis*, or to the bottom of the display + list if *belowThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items. + :meth:`lower` is an alias of :meth:`!tag_lower`. + + .. note:: + + On a :class:`Canvas`, :meth:`tkraise`/:meth:`lift` and :meth:`lower` + restack canvas items, + shadowing the inherited :meth:`Misc.tkraise`/:meth:`Misc.lift` and + :meth:`Misc.lower` methods that restack the widget itself, + which are therefore not available. + + .. method:: tag_bind(tagOrId, sequence=None, func=None, add=None) + + Bind the callback *func* to the event *sequence* for all items given by + *tagOrId*, so that *func* is invoked whenever that event occurs for one + of the items. + This is like :meth:`Widget.bind ` but operates on canvas items + rather than on whole widgets; only mouse, keyboard and virtual events may + be bound. + Mouse events are directed to the current item and keyboard events to the + focus item (see :meth:`focus`). + If *add* is true the new binding is added to any existing bindings for + the same sequence, rather than replacing them. + Return the identifier of the bound function, which can be passed to + :meth:`tag_unbind`. + + .. method:: tag_unbind(tagOrId, sequence, funcid=None) + + Remove for all items given by *tagOrId* the binding for the event + *sequence*. + If *funcid* is given, only that callback (as returned by + :meth:`tag_bind`) is unbound and deregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: bbox(tagOrId, /, *tagOrIds) + + Return a 4-tuple ``(x1, y1, x2, y2)`` giving an approximate bounding box, + in pixels, that encloses all the items given by *tagOrId* and any further + *tagOrIds*. + The result may overestimate the true bounding box by a few pixels. + Return ``None`` if no item matches or the matching items have nothing to + display. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: canvasx(screenx, gridspacing=None) + + Given a window x-coordinate *screenx*, return the canvas x-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: canvasy(screeny, gridspacing=None) + + Given a window y-coordinate *screeny*, return the canvas y-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: focus() + focus(tagOrId, /) + + With *tagOrId*, set the keyboard focus for the canvas to the first item + given by *tagOrId* that supports the insertion cursor; the focus is left + unchanged if no such item exists. + If *tagOrId* is an empty string, reset the focus so that no item has it. + With no argument, return the id of the item that currently has the focus, + or an empty string if none does. + An item only displays the insertion cursor when both it is the focus item + and its canvas has the input focus. + + This shadows the inherited :meth:`!Misc.focus`; + use :meth:`~Misc.focus_set` to focus the widget itself. + + .. method:: icursor(tagOrId, index, /) + + Set the insertion cursor of the items given by *tagOrId* to just before + the character given by *index*. + Items that do not support an insertion cursor are unaffected. + The cursor is only displayed when the item has the focus, but its + position may be set at any time. + + .. method:: index(tagOrId, index, /) + + Return as an integer the numerical index within *tagOrId* corresponding + to *index*, which is a textual description of a position (for text items + an index into the characters, for line and polygon items an index into + the coordinates). + If *tagOrId* matches several items, the first one that supports indexing + is used. + + .. method:: select_adjust(tagOrId, index) + + Adjust the end of the selection in *tagOrId* nearest to *index* so that + it is at *index*, and make the other end the anchor point for future + :meth:`select_to` calls. + If the selection is not currently in *tagOrId*, this behaves like + :meth:`select_to`. + + .. method:: select_clear() + + Clear the selection if it is in this canvas; otherwise do nothing. + + .. method:: select_from(tagOrId, index) + + Set the selection anchor point to just before the character given by + *index* in the item given by *tagOrId*. + This does not change the selection itself; it sets the fixed end for + future :meth:`select_to` calls. + + .. method:: select_item() + + Return the id of the item that holds the selection, or ``None`` if the + selection is not in this canvas. + Unlike :meth:`find` and the ``find_*`` methods, this returns the id as a + string rather than an integer. + + .. method:: select_to(tagOrId, index) + + Set the selection to the characters of *tagOrId* between the selection + anchor point and *index*, inclusive of *index*. + The anchor point is the one set by the most recent :meth:`select_adjust` + or :meth:`select_from` call. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y, gain=10) + + Scroll the canvas by *gain* times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the canvas at high speed through its window. + + .. method:: postscript(cnf={}, **kw) + + Generate a PostScript (Encapsulated PostScript, version 3.0) + representation of part or all of the canvas. + If the *file* or *channel* option is given, the PostScript is written + there and an empty string is returned; otherwise it is returned as a + string. + By default only the area currently visible in the window is generated, so + it is usually necessary either to call :meth:`~Misc.update` first or to + use the *width* and *height* options. + Supported options include *colormap*, *colormode*, *file*, *fontmap*, + *height*, *pageanchor*, *pageheight*, *pagewidth*, *pagex*, *pagey*, + *rotate*, *width*, *x* and *y*. + + +.. class:: Checkbutton(master=None, cnf={}, **kw) + + A :class:`!Checkbutton` widget displays a textual string, bitmap or image + together with a square indicator, and toggles a boolean selection when + pressed. + It has all the behavior of a simple button and, in addition, can be + selected: when selected the indicator is drawn with a check mark and the + associated variable is set to the ``onvalue``, and when deselected the + indicator is drawn empty and the variable is set to the ``offvalue``. + Inherits from :class:`Widget`. + In addition to the standard widget options, a checkbutton accepts the + options documented in the Tk ``checkbutton`` manual page, such as + *variable*, *onvalue*, *offvalue* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the checkbutton with the + mouse: toggle the selection state of the button and invoke the associated + command, if there is one. + Return the result of the command, or an empty string if no command is + associated with the checkbutton. + This is ignored if the checkbutton's state is ``disabled``. + + .. method:: select() + + Select the checkbutton and set the associated variable to its + ``onvalue``. + + .. method:: deselect() + + Deselect the checkbutton and set the associated variable to its + ``offvalue``. + + .. method:: toggle() + + Toggle the selection state of the button, redisplaying it and modifying + its associated variable to reflect the new state. + + .. method:: flash() + + Flash the checkbutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the checkbutton is left in the same normal or + active state as when the method was called. + This is ignored if the checkbutton's state is ``disabled``. + + +.. class:: Entry(master=None, cnf={}, **kw) + + An :class:`!Entry` widget displays a single line of text and lets the user + edit it. + Inherits from :class:`Widget` and :class:`XView`; since entries can hold + strings too long to fit in the window, they support horizontal scrolling + through :meth:`~XView.xview`. + + In addition to the standard widget options, an entry accepts the options + documented in the Tk ``entry`` manual page. + Notable ones are *textvariable* (the name of a variable kept in sync with + the entry's contents), *show* (if set, each character is displayed as the + given character rather than its true value, useful for password entry), + *validate* and *validatecommand* (which together let a callback accept or + reject edits), and *state* (one of ``'normal'``, ``'disabled'`` or + ``'readonly'``). + + Many of the methods below take an *index* argument that selects a character + in the entry's string. + As described in the Tk ``entry`` manual page, *index* may be a number + (counting from 0), ``'insert'`` (the character just after the insertion + cursor), ``'end'`` (just after the last character), ``'anchor'`` (the + selection anchor point), ``'sel.first'`` and ``'sel.last'`` (the ends of the + selection), or ``@x`` (the character covering pixel x-coordinate *x* in the + window). + Out-of-range indices are rounded to the nearest legal value. + + .. method:: delete(first, last=None) + + Delete the characters from index *first* up to but not including index + *last*. + If *last* is omitted, only the single character at *first* is deleted. + + .. method:: get() + + Return the entry's current string. + + .. method:: insert(index, string) + + Insert *string* just before the character given by *index*. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*. + + .. method:: select_adjust(index) + :no-typesetting: + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by + *index*, and adjust that end to be at *index* (including but not going + beyond it); the other end becomes the anchor point for future + :meth:`selection_to` calls. + If there is no selection in the entry, a new one is created between + *index* and the most recent anchor point, inclusive. + :meth:`select_adjust` is an alias of :meth:`!selection_adjust`. + + .. method:: select_clear() + :no-typesetting: + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget the method has no effect. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on an :class:`Entry`. + + .. method:: select_from(index) + :no-typesetting: + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection. + :meth:`select_from` is an alias of :meth:`!selection_from`. + + .. method:: select_present() + :no-typesetting: + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the entry, ``False`` + otherwise. + :meth:`select_present` is an alias of :meth:`!selection_present`. + + .. method:: select_range(start, end) + :no-typesetting: + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + :meth:`select_range` is an alias of :meth:`!selection_range`. + + .. method:: select_to(index) + :no-typesetting: + + .. method:: selection_to(index) + + Set the selection between the anchor point and *index*: if *index* is + before the anchor point, the selection runs from *index* up to but not + including the anchor; if *index* is after it, from the anchor up to but + not including *index*; if they coincide, nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If there is no selection in the entry, a new one is created using the + most recent anchor point. + :meth:`select_to` is an alias of :meth:`!selection_to`. + + .. method:: scan_mark(x) + + Record *x* and the current view in the entry window, for use with later + :meth:`scan_dragto` calls. + Typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Compute the difference between *x* and the *x* given to the last + :meth:`scan_mark` call, and adjust the view left or right by 10 times + that difference. + Typically associated with mouse motion events, to produce the effect of + dragging the entry at high speed through the window. + + .. method:: validate() + + Force an evaluation of the command given by the *validatecommand* option, + independently of the conditions specified by the *validate* option, and + return whether the value is considered valid. + + .. versionadded:: next + + +.. class:: Frame(master=None, cnf={}, **kw) + + A :class:`!Frame` widget is a simple container. + Its primary purpose is to act as a spacer or container for complex window + layouts; its only features are its background and an optional 3-D border to + make the frame appear raised or sunken. + Inherits from :class:`Widget`. + Refer to the Tk ``frame`` manual page for the full list of options. + + +.. class:: Label(master=None, cnf={}, **kw) + + A :class:`!Label` widget displays a non-interactive textual string, bitmap + or image. + The displayed text is set with the *text* option or linked to a variable + through *textvariable*, and an image can be shown using the *image* option. + Text must all be in a single font but may occupy multiple lines, and one + character may be underlined with the *underline* option. + Inherits from :class:`Widget`. + Refer to the Tk ``label`` manual page for the full list of options. + + +.. class:: LabelFrame(master=None, cnf={}, **kw) + + A :class:`!LabelFrame` widget is a container that has the features of a + :class:`Frame` plus the ability to display a label. + The label text is set with the *text* option and positioned with + *labelanchor*, or an arbitrary widget may be used as the label by giving it + as the *labelwidget* option. + Inherits from :class:`Widget`. + Refer to the Tk ``labelframe`` manual page for the full list of options. + + +.. class:: Listbox(master=None, cnf={}, **kw) + + A :class:`!Listbox` widget displays a list of single-line text items, one + per line, of which the user can select one or more. + The way the selection behaves is governed by the *selectmode* option, which + is one of ``browse`` (the default; at most one item, which may be dragged + with the mouse), ``single`` (at most one item), ``multiple`` (any number of + items, toggled individually), or ``extended`` (any number of items, + including discontiguous ranges, selected by clicking and dragging). + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``listbox`` manual page for the full list of options. + + Many of the methods take an *index* argument identifying a particular item. + As described in the Tk ``listbox`` manual page, *index* may be a numeric + index (counting from 0 at the top), ``'active'`` (the item with the location + cursor, set with :meth:`activate`), ``'anchor'`` (the selection anchor, set + with :meth:`selection_anchor`), ``'end'`` (the last item, or for + :meth:`index` and :meth:`insert` the position just after it), or ``@x,y`` + (the item covering pixel coordinates *x*, *y* in the listbox window). + Arguments named *first* and *last* are indices of the same forms. + + .. method:: insert(index, *elements) + + Insert the given *elements* as new items just before the item given by + *index*. + If *index* is ``'end'``, the new items are appended to the end of the + list. + + .. method:: delete(first, last=None) + + Delete the items in the range from *first* to *last* inclusive. + If *last* is omitted, it defaults to *first*, so that a single item is + deleted. + + .. method:: get(first, last=None) + + If *last* is omitted, return the contents of the item given by *first*, + or an empty string if *first* refers to a non-existent item. + If *last* is given, return a tuple of all the items in the range from + *first* to *last* inclusive. + + .. method:: size() + + Return the total number of items in the listbox. + + This shadows the inherited :meth:`!Misc.size`; + use :meth:`~Misc.grid_size` for the grid size. + + .. method:: index(index) + + Return the integer index value corresponding to *index*, or ``None`` if + *index* is out of range. + If *index* is ``'end'``, the result is a count of the number of items in + the listbox (not the index of the last item). + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` describing the bounding box, in + pixels relative to the widget, of the text of the item given by *index*. + Return ``None`` if no part of that item is visible on the screen, or if + *index* refers to a non-existent item; if the item is only partly + visible, the result still gives the full area of the item, including the + parts that are not visible. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: nearest(y) + + Given a y-coordinate within the listbox window, return the index of the + visible item nearest to that y-coordinate. + + .. method:: see(index) + + Adjust the view so that the item given by *index* is visible. + If the item is already visible the method has no effect; if it is near an + edge of the window the listbox scrolls just enough to bring it into view + at that edge, otherwise the listbox scrolls to center the item. + + .. method:: activate(index) + + Set the active item to the one given by *index*. + If *index* is outside the range of items, the closest item is activated + instead. + The active item is drawn as specified by the *activestyle* option when + the widget has the input focus, and its index may be retrieved with the + ``'active'`` index. + + .. method:: curselection() + + Return a tuple containing the numerical indices of all of the items that + are currently selected, or an empty tuple if no items are selected. + + .. method:: select_anchor(index) + :no-typesetting: + + .. method:: selection_anchor(index) + + Set the selection anchor to the item given by *index*. + If *index* refers to a non-existent item, the closest item is used. + The selection anchor is the end of the selection that is fixed while + dragging out a selection with the mouse, and may afterwards be referred + to with the ``'anchor'`` index. + :meth:`select_anchor` is an alias of :meth:`!selection_anchor`. + + .. method:: select_clear(first, last=None) + :no-typesetting: + + .. method:: selection_clear(first, last=None) + + Deselect any of the items in the range from *first* to *last* inclusive + that are selected. + The selection state of items outside this range is not changed. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Listbox`. + + .. method:: select_includes(index) + :no-typesetting: + + .. method:: selection_includes(index) + + Return ``True`` if the item given by *index* is currently selected, + ``False`` otherwise. + :meth:`select_includes` is an alias of :meth:`!selection_includes`. + + .. method:: select_set(first, last=None) + :no-typesetting: + + .. method:: selection_set(first, last=None) + + Select all of the items in the range from *first* to *last* inclusive, + without affecting the selection state of items outside that range. + :meth:`select_set` is an alias of :meth:`!selection_set`. + + .. method:: itemcget(index, option) + + Return the current value of the configuration option *option* for the + item given by *index*. + + .. method:: itemconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the item given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual item rather than to the listbox as a whole. + With no options, it returns a dictionary describing the current options + of the item; otherwise it sets the given options. + The supported item options are *background*, *foreground*, + *selectbackground* and *selectforeground*. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the listbox by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the list at high speed through the window. + + +.. class:: Menu(master=None, cnf={}, **kw) + + A :class:`!Menu` widget displays a column of entries, each of which may be a + command, a checkbutton, a radiobutton, a cascade (which posts an associated + submenu) or a separator. + Menus are used as the menubar of a toplevel window, as pulldown menus posted + from a cascade entry or menubutton, and as popup menus. + Inherits from :class:`Widget`. + + Many of the entry methods take an *index* argument that selects which entry + to operate on. + As described in the Tk ``menu`` manual page, *index* may be a numeric index + (counting from 0 at the top), ``'active'`` (the currently active entry), + ``'end'`` or ``'last'`` (the bottommost entry), ``'none'`` (no entry at all, + written ``{}`` in Tcl), ``@y`` (the entry covering pixel y-coordinate *y* in + the menu window), or a pattern matched against the labels of the entries + from the top down. + + .. method:: add(itemType, cnf={}, **kw) + + Add a new entry to the bottom of the menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'`` and determines the type of the new + entry; the remaining options configure it. + The :meth:`!add_command`, :meth:`!add_cascade`, :meth:`!add_checkbutton`, + :meth:`!add_radiobutton` and :meth:`!add_separator` convenience methods + call this method with the corresponding *itemType*. + + The entry is configured by the following options, although not every + option applies to every entry type (a separator accepts none of them): + + *label* + The text to display in the entry. + + *command* + The function to call when the entry is invoked (command, checkbutton + and radiobutton entries). + + *accelerator* + A string displayed at the right of the entry to advertise an + accelerator keystroke; it does not itself create the binding. + + *underline* + The index of a character in the label to underline for keyboard + traversal. + + *state* + One of ``'normal'``, ``'active'`` or ``'disabled'``. + + *image* + An image to display instead of, or together with, the text label. + + *compound* + Where to show the image relative to the text: ``'none'`` (the + default), ``'text'``, ``'image'``, ``'top'``, ``'bottom'``, ``'left'`` + or ``'right'``. + + *bitmap* + A bitmap to display instead of the text label. + + *font* + The font to use for the text. + + *background*, *foreground* + The entry's background and foreground colors in its normal state + (ignored on macOS). + + *activebackground*, *activeforeground* + The background and foreground colors used when the entry is active + (ignored on macOS). + + *columnbreak* + If true, the entry starts a new column instead of being placed below + the previous entry. + + *hidemargin* + If true, the standard margin around the entry is omitted, which is + useful when a menu is used as a palette. + + *menu* + The submenu posted by a cascade entry; it must be a child of this + menu. + + *variable* + The variable associated with a checkbutton or radiobutton entry. + + *onvalue*, *offvalue* + The values stored in *variable* when a checkbutton entry is selected + or cleared. + + *value* + The value stored in *variable* when a radiobutton entry is selected. + + *indicatoron* + Whether to display the indicator of a checkbutton or radiobutton + entry. + + *selectcolor* + The color of the indicator of a checkbutton or radiobutton entry when + it is selected. + + *selectimage* + The image displayed when a checkbutton or radiobutton entry is + selected and *image* is also given. + + .. method:: add_cascade(cnf={}, **kw) + + Add a new cascade entry to the bottom of the menu. + A cascade entry has an associated submenu, given by its *menu* option, + which must be a child of this menu; posting the entry posts the submenu + next to it. + + .. method:: add_checkbutton(cnf={}, **kw) + + Add a new checkbutton entry to the bottom of the menu. + When invoked, a checkbutton entry toggles between its *onvalue* and + *offvalue*, storing the result in its associated *variable*, and displays + an indicator showing whether it is selected. + + .. method:: add_command(cnf={}, **kw) + + Add a new command entry to the bottom of the menu. + A command entry behaves much like a button: when it is invoked, the + callback given by its *command* option is called. + + .. method:: add_radiobutton(cnf={}, **kw) + + Add a new radiobutton entry to the bottom of the menu. + Radiobutton entries sharing the same *variable* form a group of which + only one may be selected at a time; selecting an entry stores its *value* + in the variable. + + .. method:: add_separator(cnf={}, **kw) + + Add a separator to the bottom of the menu. + A separator is displayed as a horizontal dividing line and cannot be + activated or invoked. + + .. method:: insert(index, itemType, cnf={}, **kw) + + Same as :meth:`add`, except that the new entry is inserted just before + the entry given by *index* instead of being appended to the end of the + menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'``. + The :meth:`!insert_command`, :meth:`!insert_cascade`, + :meth:`!insert_checkbutton`, :meth:`!insert_radiobutton` and + :meth:`!insert_separator` convenience methods call this method with the + corresponding *itemType*. + + .. method:: insert_cascade(index, cnf={}, **kw) + + Insert a new cascade entry before the entry given by *index* (see + :meth:`add_cascade`). + + .. method:: insert_checkbutton(index, cnf={}, **kw) + + Insert a new checkbutton entry before the entry given by *index* (see + :meth:`add_checkbutton`). + + .. method:: insert_command(index, cnf={}, **kw) + + Insert a new command entry before the entry given by *index* (see + :meth:`add_command`). + + .. method:: insert_radiobutton(index, cnf={}, **kw) + + Insert a new radiobutton entry before the entry given by *index* (see + :meth:`add_radiobutton`). + + .. method:: insert_separator(index, cnf={}, **kw) + + Insert a separator before the entry given by *index* (see + :meth:`add_separator`). + + .. method:: delete(index1, index2=None) + + Delete all of the menu entries between *index1* and *index2* inclusive. + If *index2* is omitted, it defaults to *index1*, so that a single entry + is deleted. + Attempts to delete a tear-off entry are ignored; remove it by changing + the *tearoff* option instead. + + .. method:: entrycget(index, option) + + Return the current value of the configuration option *option* for the + entry given by *index*. + + .. method:: entryconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: entryconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the entry given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual entry rather than to the menu as a whole. + With no options, it returns a dictionary describing the current options + of the entry; otherwise it sets the given options. + The supported options are those accepted by :meth:`add` for the entry's + type. + :meth:`entryconfig` is an alias of :meth:`!entryconfigure`. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, or ``None`` if + *index* selects no entry. + + .. method:: type(index) + + Return the type of the entry given by *index*: one of ``'command'``, + ``'cascade'``, ``'checkbutton'``, ``'radiobutton'``, ``'separator'`` or + ``'tearoff'`` (for the tear-off entry). + + .. method:: activate(index) + + Make the entry given by *index* the active entry, redisplaying it with + its active colors, and deactivate any previously active entry. + If *index* selects no entry, or the selected entry is disabled, the menu + ends up with no active entry. + + .. method:: invoke(index) + + Invoke the action of the entry given by *index*, as if it had been + clicked. + Nothing happens if the entry is disabled. + If the entry has a *command* associated with it, the result of that + command is returned; otherwise the result is an empty string. + + .. method:: post(x, y) + + Display the menu on the screen at the root-window coordinates *x* and + *y*, adjusting them if necessary so that the whole menu is visible. + If the *postcommand* option has been specified, it is evaluated before + the menu is posted. + + .. method:: postcascade(index) + + Post the submenu associated with the cascade entry given by *index*, + unposting any previously posted submenu. + This has no effect if *index* does not name a cascade entry or if the + menu itself is not posted. + + .. versionadded:: next + + .. method:: tk_popup(x, y, entry='') + + Post the menu as a popup at the root-window coordinates *x* and *y*. + If *entry* is given, the menu is positioned so that this entry is + displayed under the pointer. + + .. method:: unpost() + + Unmap the menu so that it is no longer displayed, also unposting any + posted lower-level cascaded submenu. + This has no effect on Windows and macOS, which manage the unposting of + menus themselves. + + .. method:: xposition(index) + + Return the x-coordinate, within the menu window, of the leftmost pixel of + the entry given by *index*. + + .. versionadded:: 3.3 + + .. method:: yposition(index) + + Return the y-coordinate, within the menu window, of the topmost pixel of + the entry given by *index*. + + +.. class:: Menubutton(master=None, cnf={}, **kw) + + A :class:`!Menubutton` widget displays a textual string, bitmap or image and + posts an associated :class:`Menu`, given by its *menu* option, when the user + presses it. + Like a :class:`Label` it can show *text*, a *textvariable*, or an *image*, + and the *direction* option controls where the menu appears relative to the + button. + Inherits from :class:`Widget`. + Refer to the Tk ``menubutton`` manual page for the full list of options. + + +.. class:: Message(master=None, cnf={}, **kw) + + A :class:`!Message` widget displays a non-interactive textual string, given + by the *text* option or linked to a variable through *textvariable*. + Unlike a :class:`Label`, it breaks the string into multiple lines in order + to produce a given aspect ratio, choosing line breaks at word boundaries, + and it can justify the text left, centered or right. + Inherits from :class:`Widget`. + Refer to the Tk ``message`` manual page for the full list of options. + + +.. class:: OptionMenu(master, variable, value, *values, **kwargs) + + A helper subclass of :class:`Menubutton` that displays a pop-up menu of + mutually exclusive choices. + *variable* is a :class:`Variable` kept in sync with the selection, *value* + is the initial choice, and *values* are the remaining menu entries. + The keyword argument *command* may be given a callback that is invoked with + the selected value, and the keyword argument *name* sets the Tk widget name. + Other keyword arguments are passed to the underlying :class:`Menubutton` + and may override its default appearance. + + .. method:: destroy() + + Destroy the widget, also cleaning up the associated pop-up menu. + + .. versionchanged:: 3.14 + Added support for the *name* keyword argument. + + .. versionchanged:: next + Other :class:`Menubutton` options can now be passed as keyword arguments. + + + +.. class:: PanedWindow(master=None, cnf={}, **kw) + + A :class:`!PanedWindow` is a geometry-manager widget that arranges any + number of child *panes* in a row (when *orient* is ``'horizontal'``) or a + column (when *orient* is ``'vertical'``). + Each pane holds one widget, and each pair of adjacent panes is separated by + a movable *sash* that the user can drag with the mouse to resize the widgets + on either side of it. + Inherits from :class:`Widget`. + + The *orient* option selects the layout direction, *sashwidth* sets the width + of each sash and *sashrelief* its relief. + When *showhandle* is true a small handle is drawn on each sash that the user + can grab to drag it. + Refer to the Tk ``panedwindow`` manual page for the full list of options. + + .. method:: add(child, **kw) + + Add *child* to the panedwindow as a new pane, placed after any existing + panes. + The keyword arguments specify per-pane management options for *child*; + they may be any of the options accepted by :meth:`paneconfigure`. + + .. method:: forget(child) + :no-typesetting: + + .. method:: remove(child) + + Remove the pane containing *child* from the panedwindow. + All geometry management options for *child* are forgotten. + :meth:`forget` is an alias of :meth:`!remove`. + This shadows the inherited geometry-manager :meth:`!forget`; + use :meth:`~Pack.pack_forget`, :meth:`~Grid.grid_forget` or + :meth:`~Place.place_forget` to remove the widget itself from its manager. + + .. method:: panes() + + Return a tuple of the widgets managed by the panedwindow, one per pane, + in order. + + .. method:: panecget(child, option) + + Return the current value of the management option *option* for the pane + containing *child*. + *option* may be any value allowed by :meth:`paneconfigure`. + + .. method:: paneconfig(child, cnf=None, **kw) + :no-typesetting: + + .. method:: paneconfigure(child, cnf=None, **kw) + + Query or modify the management options of the pane containing the widget + *child*. + With no options, it returns a dictionary describing all of the available + options for the pane; given a single option name as a string, it returns + a description of that one option; otherwise it sets the given options. + The supported options include *after* and *before* (insert the pane after + or before another managed window), *height* and *width* (the outer + dimensions of the window, including any border), *minsize* (the minimum + size in the paned dimension), *padx* and *pady* (extra space to leave on + each side of the window), *sticky* (position or stretch the window within + an oversized pane, using a string of the characters ``n``, ``s``, ``e`` + and ``w``), *hide* (hide the pane while keeping it in the list of panes) + and *stretch* (how extra space is allocated to the pane: one of + ``'always'``, ``'first'``, ``'last'``, ``'middle'`` or ``'never'``). + :meth:`paneconfig` is an alias of :meth:`!paneconfigure`. + + .. deprecated-removed:: next 3.18 + The first parameter was renamed from *tagOrId* to *child*. + The old name is still accepted as a keyword argument. + + .. method:: identify(x, y) + + Identify the panedwindow component underneath the point given by *x* and + *y*, in window coordinates. + If the point is over a sash or a sash handle, the result is a two-element + tuple containing the index of the sash or handle and a word indicating + whether it is over a sash or a handle, such as ``(0, 'sash')`` or + ``(2, 'handle')``. + If the point is over any other part of the panedwindow, the result is an + empty string. + + .. method:: sash(*args) + + Query or change the position of the sashes in the panedwindow. + This is a thin wrapper around the Tk ``sash`` subcommand; the convenience + methods :meth:`sash_coord`, :meth:`sash_mark` and :meth:`sash_place` + should normally be used instead. + + .. method:: sash_coord(index) + + Return the current x and y coordinate pair for the sash given by *index*, + which must be an integer between 0 and one less than the number of panes + in the panedwindow. + The coordinates returned are those of the top left corner of the region + containing the sash. + + .. method:: sash_mark(index) + + Record the current mouse position for the sash given by *index*, for use + together with later sash-drag operations to move the sash. + + .. method:: sash_place(index, x, y) + + Place the sash given by *index* at the coordinates *x* and *y*. + + .. method:: proxy(*args) + + Query or change the position of the sash proxy, the "ghost" sash shown + while a sash is being dragged with non-opaque resizing. + This is a thin wrapper around the Tk ``proxy`` subcommand; the + convenience methods :meth:`proxy_coord`, :meth:`proxy_forget` and + :meth:`proxy_place` should normally be used instead. + + .. method:: proxy_coord() + + Return a tuple containing the x and y coordinates of the most recent + proxy location. + + .. method:: proxy_forget() + + Remove the proxy from the display. + + .. method:: proxy_place(x, y) + + Place the proxy at the coordinates *x* and *y*. + + +.. class:: Radiobutton(master=None, cnf={}, **kw) + + A :class:`!Radiobutton` widget displays a textual string, bitmap or image + together with a diamond or circular indicator, and selects one choice out of + several. + It has all the behavior of a simple button and, in addition, can be + selected: typically several radiobuttons share a single *variable*, and + selecting one sets that variable to the radiobutton's *value*; each + radiobutton also monitors the variable and automatically selects or + deselects itself when the variable changes. + Inherits from :class:`Widget`. + In addition to the standard widget options, a radiobutton accepts the + options documented in the Tk ``radiobutton`` manual page, such as + *variable*, *value* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the radiobutton with the + mouse: select the button and invoke the associated command, if there is + one. + Return the result of the command, or an empty string if no command is + associated with the radiobutton. + This is ignored if the radiobutton's state is ``disabled``. + + .. method:: select() + + Select the radiobutton and set the associated variable to the value + corresponding to this widget. + + .. method:: deselect() + + Deselect the radiobutton and set the associated variable to an empty + string. + If this radiobutton was not currently selected, this has no effect. + + .. method:: flash() + + Flash the radiobutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the radiobutton is left in the same normal or + active state as when the method was called. + This is ignored if the radiobutton's state is ``disabled``. + + +.. class:: Scale(master=None, cnf={}, **kw) + + A :class:`!Scale` widget lets the user select a numerical value by moving a + slider along a trough. + It can be oriented vertically or horizontally and can optionally display a + label and the current value. + Inherits from :class:`Widget`. + + In addition to the standard widget options, a scale accepts the options + documented in the Tk ``scale`` manual page, such as *from_*, *to*, + *resolution*, *orient*, *tickinterval*, *variable* and *command*. + As elsewhere in :mod:`!tkinter`, the leading ``-`` of the Tk option name is + dropped; *from* is spelled ``from_`` because :keyword:`from` is a Python + keyword. + + With a non-integer *resolution*, see :ref:`numeric values and the locale + `. + + .. method:: get() + + Return the current value of the scale. + The result is an integer if the scale's *resolution* yields whole + numbers, and a float otherwise. + + .. method:: set(value) + + Set the scale to *value*, moving the slider accordingly. + This has no effect if the scale is disabled. + + .. method:: coords(value=None) + + Return a tuple ``(x, y)`` giving the pixel coordinates, relative to the + widget, of the point on the centerline of the trough that corresponds to + *value*. + If *value* is omitted, the scale's current value is used. + + .. method:: identify(x, y) + + Return a string describing the part of the scale at the pixel coordinates + *x*, *y*: ``'slider'``, ``'trough1'`` (the part of the trough above or to + the left of the slider), ``'trough2'`` (below or to the right of the + slider), or an empty string if the point is not over any of these + elements. + + +.. class:: Scrollbar(master=None, cnf={}, **kw) + + A :class:`!Scrollbar` widget displays a slider and two arrows that let the + user scroll an associated widget, such as a :class:`Listbox`, :class:`Text`, + :class:`Canvas` or :class:`Entry`. + It is connected to the scrolled widget by setting that widget's + *xscrollcommand* or *yscrollcommand* option to the scrollbar's :meth:`set` + method, and the scrollbar's *command* option to the scrolled widget's + :meth:`~XView.xview` or :meth:`~YView.yview` method. + Inherits from :class:`Widget`. + + .. method:: get() + + Return the current scrollbar settings as a tuple ``(first, last)`` of two + fractions between 0 and 1, describing the portion of the document that is + currently visible, as last passed to :meth:`set`. + + .. method:: set(first, last) + + Set the scrollbar. + *first* and *last* are fractions between 0 and 1 giving the positions of + the start and end of the visible portion of the associated document. + This method is normally registered as the scrolled widget's + *xscrollcommand* or *yscrollcommand* and called by that widget. + + .. method:: activate(index=None) + + Mark the element *index* (one of ``'arrow1'``, ``'slider'`` or + ``'arrow2'``) as active, displaying it according to the + *activebackground* and *activerelief* options. + If *index* is omitted, return the name of the currently active element, + or ``None`` if no element is active. + + .. versionchanged:: 3.5 + The *index* argument is now optional. + + .. method:: delta(deltax, deltay) + + Return a float indicating the fractional change in the scrollbar setting + that corresponds to moving the slider by *deltax* pixels horizontally + (for horizontal scrollbars) or *deltay* pixels vertically (for vertical + scrollbars). + + .. method:: fraction(x, y) + + Return a float between 0 and 1 indicating where the point at pixel + coordinates *x*, *y* lies in the trough: 0 corresponds to the top or left + of the trough and 1 to the bottom or right. + + .. method:: identify(x, y) + + Return the name of the element under the pixel coordinates *x*, *y* (such + as ``'arrow1'``), or an empty string if the point does not lie in any + element of the scrollbar. + + +.. class:: Spinbox(master=None, cnf={}, **kw) + + A :class:`!Spinbox` widget is an :class:`Entry`-like widget with a pair of + up/down arrow buttons that let the user step through a range of values in + addition to editing the value directly. + The set of values may be a numeric range given by the *from_*, *to* and + *increment* options, or an explicit list of strings given by the *values* + option (which takes precedence over the range). + Each time an arrow is invoked the *command* callback, if any, is called; the + *wrap* option controls whether stepping past either end of the range wraps + around to the other end; the *format* option specifies how numeric values + are formatted; and the *validate* option enables validation of the entered + text. + Inherits from :class:`Widget` and :class:`XView`. + + With a non-integer *increment*, see :ref:`numeric values and the locale + `. + + Many of the methods take an *index* argument identifying a character in the + spinbox's string. + As described in the Tk ``spinbox`` manual page, *index* may be a numeric + index (counting from 0), ``'anchor'`` (the selection anchor point), + ``'end'`` (just after the last character), ``'insert'`` (the character just + after the insertion cursor), ``'sel.first'`` or ``'sel.last'`` (the ends of + the selection), or ``@x`` (the character covering pixel x-coordinate *x* in + the window). + + .. method:: get() + + Return the spinbox's string. + + .. method:: insert(index, s) + + Insert the characters of the string *s* just before the character given + by *index*. + + .. method:: delete(first, last=None) + + Delete one or more characters of the spinbox. + *first* is the index of the first character to delete, and *last* is the + index of the character just after the last one to delete. + If *last* is omitted, a single character at *first* is deleted. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, as a string. + + .. method:: bbox(index) + + Return a tuple of four integers ``(x, y, width, height)`` describing the + bounding box of the character given by *index*. + *x* and *y* are the pixel coordinates of the upper-left corner of the + character relative to the widget, and *width* and *height* are its size + in pixels. + The bounding box may refer to a region outside the visible area of the + window. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: identify(x, y) + + Return the name of the window element at the pixel coordinates *x*, *y*: + one of ``'buttondown'``, ``'buttonup'``, ``'entry'`` or ``'none'``. + + .. method:: invoke(element) + + Invoke the spin button given by *element*, either ``'buttonup'`` or + ``'buttondown'``, triggering the action associated with it. + + .. method:: scan(*args) + + A thin wrapper around the Tk ``scan`` widget subcommand, used to + implement fast dragging of the view: ``scan('mark', x)`` records *x* and + the current view, and ``scan('dragto', x)`` adjusts the view relative to + that mark. + The :meth:`scan_mark` and :meth:`scan_dragto` methods wrap the two forms. + + .. method:: scan_mark(x) + + Record *x* and the current view in the spinbox window, for use with a + later :meth:`scan_dragto` call. + This is typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Adjust the view by 10 times the difference between *x* and the *x* passed + to the last :meth:`scan_mark` call. + This is typically associated with mouse motion events, producing the + effect of dragging the spinbox at high speed through the window. + + .. method:: selection(*args) + + A thin wrapper around the Tk ``selection`` widget subcommand, used to + adjust the selection within the spinbox. + It has several forms depending on the first argument, such as + ``selection('adjust', index)``, ``selection('clear')``, + ``selection('element', ?elem?)``, ``selection('from', index)``, + ``selection('present')``, ``selection('range', start, end)`` and + ``selection('to', index)``. + The :meth:`selection_adjust`, :meth:`selection_clear`, + :meth:`selection_element`, :meth:`selection_from`, + :meth:`selection_present`, :meth:`selection_range` and + :meth:`selection_to` methods wrap these forms. + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by *index* + and adjust that end of the selection to be at *index* (including but not + going beyond *index*). + The other end becomes the anchor point for future :meth:`selection_to` + calls. + If the selection is not currently in the spinbox, a new selection is + created to include the characters between *index* and the most recent + anchor point, inclusive. + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget, the method has no effect. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Spinbox`. + + .. method:: selection_element(element=None) + + Set or get the currently selected element. + If *element* (one of ``'buttonup'``, ``'buttondown'`` or ``'none'``) is + given, that spin button is selected and displayed depressed; otherwise + the name of the currently selected element is returned. + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection itself. + + .. versionadded:: 3.8 + + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the spinbox, + ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + + .. versionadded:: 3.8 + + + .. method:: selection_to(index) + + Set the selection between *index* and the anchor point. + If *index* is before the anchor point, the selection runs from *index* up + to but not including the anchor point; if it is after, the selection runs + from the anchor point up to but not including *index*; if it is the same, + nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If the selection is not in this widget, a new selection is created using + the most recent anchor point. + + .. versionadded:: 3.8 + + .. method:: validate() + + Force an evaluation of the command given by the *validatecommand* option, + independently of the conditions specified by the *validate* option, and + return whether the value is considered valid. + + .. versionadded:: next + + + +.. class:: Text(master=None, cnf={}, **kw) + + A :class:`!Text` widget displays and edits multi-line text. + Portions of the text may be styled with **tags**, particular positions may + be annotated with floating **marks**, and arbitrary images and other widgets + may be embedded in the text. + The widget also provides an unlimited undo/redo mechanism and supports peer + widgets that share the same underlying data. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``text`` manual page for the full list of options. + + Most of the methods take one or more *index* arguments that identify a + position within the text. + As described in the Tk ``text`` manual page, an index is a string consisting + of a base, optionally followed by one or more modifiers. + The base may be ``'line.char'`` (line *line*, character *char*, where lines + are counted from 1 and characters within a line from 0; ``'line.end'`` + refers to the newline ending the line), ``'end'`` (the position just after + the last newline), the name of a mark, ``'tag.first'`` or ``'tag.last'`` + (the first character tagged with *tag*, or the position just after the last + such character), the name of an embedded image or window, or ``@x,y`` (the + character covering pixel coordinates *x*, *y* in the widget). + A modifier such as ``'+5 chars'``, ``'-3 lines'``, ``'linestart'``, + ``'lineend'``, ``'wordstart'`` or ``'wordend'`` adjusts the index relative + to its base; several modifiers may be combined and are applied from left to + right, for example ``'insert wordstart - 1 c'``. + + .. method:: tk_print() + + Print the contents of the text widget using the native print dialog. + Requires Tk 8.7/9.0 or newer. + + .. versionadded:: next + + .. method:: insert(index, chars, *args) + + Insert the string *chars* just before the character at *index* (if + *index* is ``'end'``, just before the final newline). + By default the new text inherits any tags present on both sides of the + insertion point. + If *args* is given, it consists of alternating *tagList*, *chars* values: + the preceding *chars* receives exactly the tags listed (a tag list may be + a single tag name or a sequence of names), overriding the surrounding + tags. + + .. method:: delete(index1, index2=None) + + Delete the range of characters from *index1* up to but not including + *index2*. + If *index2* is omitted, the single character at *index1* is deleted. + The widget always keeps a newline as its last character, so a deletion + that would remove it is adjusted accordingly. + + .. method:: replace(index1, index2, chars, *args) + + Replace the range of characters from *index1* up to but not including + *index2* with *chars*. + This is equivalent to a :meth:`delete` followed by an :meth:`insert` at + *index1*; *args* is interpreted as for :meth:`insert`. + + .. versionadded:: 3.3 + + .. method:: get(index1, index2=None) + + Return the text from *index1* up to but not including *index2* as a + string. + If *index2* is omitted, return the single character at *index1*. + Embedded images and windows are omitted from the result. + + .. method:: index(index) + + Return the position corresponding to *index* in the canonical + ``'line.char'`` form. + + .. method:: compare(index1, op, index2) + + Compare the positions of *index1* and *index2* using the relational + operator *op*, which must be one of ``'<'``, ``'<='``, ``'=='``, + ``'>='``, ``'>'`` or ``'!='``, and return the boolean result. + + .. method:: count(index1, index2, *options, return_ints=False) + + Count the number of items of the requested kinds between *index1* and + *index2*; the count is negative if *index1* is after *index2*. + Each of *options* names a kind of item to count: ``'chars'``, + ``'displaychars'``, ``'displayindices'``, ``'displaylines'``, + ``'indices'``, ``'lines'``, ``'xpixels'`` or ``'ypixels'`` (the default, + used when no option is given, is ``'indices'``). + The pseudo-option ``'update'`` forces any out-of-date layout information + to be recalculated before the following options are evaluated. + When *return_ints* is true and a single counting option is given, return + a plain integer; otherwise return a tuple with one integer per counting + option (or ``None`` if the result is empty). + + .. versionadded:: 3.3 + + .. versionchanged:: 3.13 + Added the *return_ints* parameter. + + + .. method:: see(index) + + Adjust the view so that the character given by *index* is visible. + If it is already visible the method has no effect; if it is a short + distance out of view the widget scrolls just enough to bring it to the + nearest edge, otherwise it scrolls to center *index* in the window. + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` giving the bounding box, in + pixels, of the visible part of the character at *index*, or ``None`` if + that character is not visible on the screen. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: dlineinfo(index) + + Return a tuple ``(x, y, width, height, baseline)`` describing the display + line that contains *index*: the first four values give the bounding box + of the line in pixels and *baseline* gives the offset of the baseline + measured down from the top of the area. + Return ``None`` if that display line is not visible on the screen. + + .. method:: mark_set(markName, index) + + Set the mark named *markName* to the position just before the character + at *index*, creating the mark if it does not already exist. + A mark created this way has right gravity by default. + + .. method:: mark_unset(*markNames) + + Remove each of the marks named in *markNames*. + The special ``insert`` and ``current`` marks may not be removed. + + .. method:: mark_names() + + Return a tuple of the names of all marks currently set in the widget. + + .. method:: mark_gravity(markName, direction=None) + + If *direction* is omitted, return the gravity of mark *markName*, either + ``'left'`` or ``'right'``. + Otherwise set its gravity to *direction*. + The gravity determines on which side of the mark text inserted at the + mark's position appears: a mark with right gravity (the default) stays to + the right of such text. + + .. method:: mark_next(index) + + Return the name of the first mark at or after *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just after that + mark. + + .. method:: mark_previous(index) + + Return the name of the last mark at or before *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just before that + mark. + + .. method:: tag_add(tagName, index1, *args) + + Add the tag *tagName* to the range of characters from *index1* up to but + not including the next index in *args*. + Further pairs of indices may follow in *args* to tag additional ranges; a + trailing single index tags just the character at that index. + + .. method:: tag_remove(tagName, index1, index2=None) + + Remove the tag *tagName* from the characters from *index1* up to but not + including *index2* (or from the single character at *index1* if *index2* + is omitted). + The tag itself continues to exist even if no characters carry it. + + .. method:: tag_delete(*tagNames) + + Delete each of the tags named in *tagNames*, removing them from all + characters and discarding their options and bindings. + + .. method:: tag_config(tagName, cnf=None, **kw) + :no-typesetting: + + .. method:: tag_configure(tagName, cnf=None, **kw) + + Query or modify the configuration options of the tag *tagName*. + This mirrors :meth:`~Misc.configure`, except that it applies to a tag + rather than to the widget as a whole: with no options it returns a + dictionary describing the current options, otherwise it sets the given + options. + Defining a tag this way also gives it a priority higher than any existing + tag. + + The supported tag options, all controlling the appearance of the tagged + text, are: + + *font* + The font to use for the text. + + *foreground* + The color to use for the text. + + *background* + The color to use for the area behind the text. + + *fgstipple*, *bgstipple* + Bitmaps used to stipple the foreground (text) and the background; + only well supported on X11. + + *borderwidth* + The width of the border drawn around the text according to *relief* + (default ``0``). + + *relief* + The 3-D appearance of the text's border: ``'flat'`` (the default), + ``'raised'``, ``'sunken'``, ``'ridge'``, ``'groove'`` or ``'solid'``. + + *offset* + How far the text is raised above (or, if negative, lowered below) the + baseline, for superscripts and subscripts. + + *underline* + Whether to underline the text. + + *underlinefg* + The color of the underline; it defaults to the text color. + + *overstrike* + Whether to draw a line through the middle of the text. + + *overstrikefg* + The color of the overstrike line; it defaults to the text color. + + *elide* + Whether the text is elided (hidden). + + *justify* + How to justify the first character of a display line: ``'left'`` (the + default), ``'right'`` or ``'center'``. + + *wrap* + How to wrap lines that are too long: ``'char'``, ``'word'`` or + ``'none'``. + + *lmargin1*, *lmargin2* + The indentation, in pixels, of the first display line of a logical + line and of the remaining display lines. + + *lmargincolor* + The color of the left margin area. + + *rmargin* + The right-hand margin, in pixels. + + *rmargincolor* + The color of the right margin area. + + *spacing1*, *spacing2*, *spacing3* + Extra space, in pixels, above the first display line of a logical + line, between its display lines, and below its last display line. + + *tabs* + The set of tab stops, in the same form as the widget's *tabs* option. + + *tabstyle* + How tab stops are interpreted: ``'tabular'`` or ``'wordprocessor'``. + + *selectbackground*, *selectforeground* + The background and foreground colors used for the text while it is + selected. + + .. note:: + + Tk 8.6 added the *lmargincolor*, *overstrikefg*, *rmargincolor*, + *selectbackground*, *selectforeground* and *underlinefg* options. + + :meth:`tag_config` is an alias of :meth:`!tag_configure`. + + .. method:: tag_cget(tagName, option) + + Return the current value of the configuration option *option* for the tag + *tagName*. + + .. method:: tag_names(index=None) + + If *index* is omitted, return a tuple of the names of all tags defined in + the widget; otherwise return only the names of the tags applied to the + character at *index*. + The names are ordered from lowest to highest priority. + + .. method:: tag_ranges(tagName) + + Return a tuple of indices describing all ranges of text tagged with + *tagName*. + The result alternates start and end indices, so that elements ``2*i`` and + ``2*i+1`` bound the *i*-th range. + + .. method:: tag_nextrange(tagName, index1, index2=None) + + Search forward from *index1* (up to *index2* if given) for the first + range of characters tagged with *tagName*, and return a two-element tuple + of its start and end indices, or an empty tuple if there is no such + range. + + .. method:: tag_prevrange(tagName, index1, index2=None) + + Search backward from *index1* (down to *index2* if given) for the nearest + preceding range of characters tagged with *tagName*, and return a + two-element tuple of its start and end indices, or an empty tuple if + there is no such range. + + .. method:: tag_raise(tagName, aboveThis=None) + + Raise the priority of tag *tagName* so that it is just above the priority + of *aboveThis*, or to the highest priority of all tags if *aboveThis* is + omitted. + When the display options of overlapping tags conflict, the + higher-priority tag wins. + + .. method:: tag_lower(tagName, belowThis=None) + + Lower the priority of tag *tagName* so that it is just below the priority + of *belowThis*, or to the lowest priority of all tags if *belowThis* is + omitted. + + .. method:: tag_bind(tagName, sequence, func, add=None) + + Bind the event *sequence* on characters tagged with *tagName* to the + callback *func*, so that *func* is invoked when that event occurs over + such a character. + If *add* is true the binding is added alongside any existing bindings for + *sequence*, otherwise it replaces them. + Works like :meth:`~Misc.bind` and returns the identifier of the new + binding. + + .. method:: tag_unbind(tagName, sequence, funcid=None) + + Remove the bindings of the event *sequence* on characters tagged with + *tagName*. + If *funcid* is given, only that binding (as returned by :meth:`tag_bind`) + is removed and its callback is unregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: image_create(index, cnf={}, **kw) + + Embed an image at *index* and return the name assigned to this image + instance, which may then be used as an index or passed to the other + ``image_*`` methods. + The options, given in *cnf* and *kw*, include *image* (the Tk image to + display), *name* (a base name for the instance), *align*, *padx* and + *pady*. + + .. method:: image_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded image at *index*. + + .. method:: image_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded image at + *index*, like :meth:`~Misc.configure` but applied to that image. + + .. method:: image_names() + + Return a tuple of the names of all images embedded in the widget. + + .. note:: + + This shadows the inherited :meth:`Misc.image_names`, + which returns the names of all images in the Tcl interpreter; + that method is not available on a :class:`Text`. + + .. method:: window_create(index, cnf={}, **kw) + + Embed a window (any widget) at *index*. + The options, given in *cnf* and *kw*, include *window* (the widget to + embed), *create* (a callback that creates the widget on demand), *align*, + *stretch*, *padx* and *pady*. + The embedded widget must be a descendant of the text widget's parent. + + .. method:: window_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded window at *index*. + + .. method:: window_config(index, cnf=None, **kw) + :no-typesetting: + + .. method:: window_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded window at + *index*, like :meth:`~Misc.configure` but applied to that window. + + :meth:`window_config` is an alias of :meth:`!window_configure`. + + .. method:: window_names() + + Return a tuple of the names of all windows embedded in the widget. + + .. method:: edit(*args) + + Low-level wrapper around the Tk ``edit`` widget command that controls the + undo/redo mechanism and the modified flag; *args* is the ``edit`` + subcommand and its arguments. + The :meth:`!edit_\*` methods below are thin wrappers around it and are + usually more convenient. + + .. method:: edit_modified(arg=None) + + If *arg* is omitted, return the current state of the modified flag as + a :class:`bool`; the flag is set automatically whenever the text is + inserted or deleted. + Otherwise set the flag to the boolean *arg*. + + .. method:: edit_canundo() + + Return ``True`` if there is an edit action on the undo stack that can be + undone, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_canredo() + + Return ``True`` if there is an edit action on the redo stack that can be + reapplied, and ``False`` otherwise. + + .. versionadded:: next + + .. method:: edit_undo() + + Undo the most recent edit action, that is, all the inserts and deletes + recorded on the undo stack since the previous separator, and move it to + the redo stack. + Raises :exc:`TclError` if the undo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_redo() + + Reapply the most recently undone edit action, provided no further edits + have been made since, and move it back to the undo stack. + Raises :exc:`TclError` if the redo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_reset() + + Clear the undo and redo stacks. + + .. method:: edit_separator() + + Push a separator onto the undo stack, marking a boundary between edit + actions for undo and redo. + Has no effect unless the *undo* option is true. + Separators are inserted automatically when the *autoseparators* option is + true. + + .. method:: search(pattern, index, stopindex=None, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, *, nolinestop=None, strictlimits=None) + + Search for *pattern* starting at *index* and return the index of the + first character of the first match, or an empty string if there is no + match. + Searching stops at *stopindex* if given; otherwise it wraps around the + ends of the text until the starting position is reached again. + The following boolean keyword flags control the search: *forwards* or + *backwards* select the direction (forward is the default); *exact* (the + default) or *regexp* select literal or regular-expression matching; + *nocase* makes the match case-insensitive; *elide* causes hidden text to + be searched as well; *nolinestop* (regexp only) lets ``.`` and ``[^`` + match newlines; and *strictlimits* requires the whole match to lie within + *index* and *stopindex*. + If *count* is a :class:`Variable`, the number of index positions in the + match is stored in it. + + .. versionchanged:: 3.15 + Added the *nolinestop* and *strictlimits* parameters. + + + .. method:: search_all(pattern, index, stopindex=None, *, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None, nolinestop=None, overlap=None, strictlimits=None) + + Like :meth:`search`, but find every match in the searched range and + return a tuple of the starting indices of all matches (empty if there are + none). + By default overlapping matches are not reported; passing a true *overlap* + returns every match that is not wholly contained in another. + If *count* is a :class:`Variable`, it receives a list with one element + per match. + + .. versionadded:: 3.15 + + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the widget by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events, producing the effect of + dragging the text at high speed through the window. + + .. method:: debug(boolean=None) + + If *boolean* is omitted, return whether internal consistency checks of + the B-tree data structure are enabled. + Otherwise enable or disable them. + The setting is shared by all text widgets and may noticeably slow down + widgets holding large amounts of text. + + .. method:: dump(index1, index2=None, command=None, **kw) + + Return the contents of the widget from *index1* up to but not including + *index2* (or just the segment at *index1* if *index2* is omitted), + including text and information about marks, tags, images and windows. + The result is a list of ``(key, value, index)`` triples, where *key* is + one of ``'text'``, ``'mark'``, ``'tagon'``, ``'tagoff'``, ``'image'`` or + ``'window'``. + By default all kinds are reported; passing any of the keyword arguments + *all*, *text*, *mark*, *tag*, *image* or *window* as true restricts the + dump to the selected kinds. + If *command* is given, it is called once per triple with the three values + as arguments and nothing is returned. + + .. method:: peer_create(newPathName, cnf={}, **kw) + + Create a peer text widget with the path name *newPathName* that shares + this widget's underlying data (text, marks, tags, images and the undo + stack). + Changes made through any peer are reflected in all of them. + By default the peer covers the same lines as this widget; standard text + options, including *startline* and *endline*, may be given to override + this. + + .. versionadded:: 3.3 + + .. method:: peer_names() + + Return a tuple of the path names of this widget's peers, not including + the widget itself. + + .. versionadded:: 3.3 + + .. method:: sync(command=None) + + Control the synchronization of the displayed view with the underlying + text, which may lag behind when line heights have not yet been computed + (for example, for lines that have never been displayed). + If *command* is omitted, bring the line metrics up to date immediately by + forcing computation of any outdated line heights, and return once they + are current. + Otherwise schedule *command* to be called, with no arguments, exactly + once as soon as all line heights are up to date; if there are no pending + calculations, it is called immediately. + + .. versionadded:: next + + .. method:: pendingsync() + + Return ``True`` if the line height calculations are not up to date, and + ``False`` otherwise. + The ``<>`` virtual event fires whenever this state + changes, with the *detail* field set to the new value. + + .. versionadded:: next + + .. method:: yview_pickplace(*what) + + Adjust the view so that the location given by *what* is visible. + This is an obsolete equivalent of :meth:`see`, which should be used + instead. + + +Variable classes +^^^^^^^^^^^^^^^^ + +.. class:: Variable(master=None, value=None, name=None) + + The base class for the Tk variable wrappers. + A Tk variable is a value stored in the Tcl interpreter that can be linked to + widgets through their *variable* or *textvariable* options (see + :ref:`coupling-widget-variables`), so that changes propagate both ways: + updating the variable updates every widget bound to it, and a user editing + such a widget updates the variable. + + *master* is the widget whose Tcl interpreter owns the variable; if omitted, + the default root window is used. + *value* is the initial value; if omitted, a type-specific default is used. + *name* is the name of the variable in the Tcl interpreter; if omitted, a + unique name of the form ``'PY_VARnum'`` is generated. + If *name* matches an existing variable and *value* is omitted, the existing + value is retained. + + In most cases you should use one of the typed subclasses below -- + :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` or + :class:`BooleanVar` -- rather than :class:`!Variable` directly. + + .. note:: + + When a :class:`!Variable` is garbage collected, its Tcl variable is unset. + Keep a reference to it for as long as a widget is linked to it, for example + by storing it as an attribute rather than in a local variable. + Otherwise Tk recreates the Tcl variable to keep the widget working, but it + is never unset again, leaking one Tcl variable per dropped wrapper. + + .. versionchanged:: 3.10 + Two variables now compare equal (``==``) only when they have the same + name, are of the same class, and belong to the same Tcl interpreter. + + .. method:: get() + + Return the current value of the variable. + For the base class the value is returned as a string; the typed + subclasses convert it to the appropriate Python type. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + .. method:: trace_add(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'array'``, ``'read'``, ``'write'`` or + ``'unset'``, or a list or tuple of such strings. + + When triggered, *callback* is called with three arguments: the name of + the Tcl variable, an index (or an empty string if the variable is not an + element of an array), and the *mode* that triggered the call. + + Return the internal name of the registered callback, which can be passed + to :meth:`trace_remove`. + + .. versionadded:: 3.6 + + .. method:: trace_remove(mode, cbname) + + Remove a trace callback from the variable. + *mode* must match the *mode* that was passed to :meth:`trace_add`, and + *cbname* is the callback name returned by :meth:`trace_add`. + + .. versionadded:: 3.6 + + .. method:: trace_info() + + Return a list of ``(modes, cbname)`` pairs describing all traces + currently set on the variable, where *modes* is a tuple of mode strings + and *cbname* is the internal callback name. + + .. versionadded:: 3.6 + + .. method:: trace(mode, callback) + :no-typesetting: + + .. method:: trace_variable(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'r'``, ``'w'`` or ``'u'``, for read, write + or unset. + Return the internal name of the registered callback. + :meth:`trace` is an alias of :meth:`!trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_add` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + .. method:: trace_vdelete(mode, cbname) + + Remove the trace callback named *cbname* registered for *mode* with + :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_remove` instead. This method wraps a Tcl feature + that was removed in Tcl 9.0. + + .. method:: trace_vinfo() + + Return a list of ``(mode, cbname)`` pairs for all traces set on the + variable with :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_info` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + +.. class:: StringVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a string. + The default value is ``''``. + + .. method:: get() + + Return the value of the variable as a :class:`str`. + + +.. class:: IntVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds an integer. + The default value is ``0``. + + .. method:: get() + + Return the value of the variable as an :class:`int`. + + +.. class:: DoubleVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a float. + The default value is ``0.0``. + + .. method:: get() + + Return the value of the variable as a :class:`float`. + + .. _tkinter-numeric-locale: + + .. note:: + + A floating-point value is always parsed with a period (``.``) as the + decimal separator, but :class:`Spinbox`, :class:`Scale` and + :class:`ttk.Spinbox ` format it according to the + ``LC_NUMERIC`` locale. Under a locale that uses a comma they produce a + value that :meth:`get` cannot read, raising :exc:`TclError`. Set + ``LC_NUMERIC`` to a locale that uses a period (such as ``'C'``) to avoid + this. + + +.. class:: BooleanVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a boolean. + The default value is ``False``. + + .. method:: get() + + Return the value of the variable as a :class:`bool`. + Raise a :exc:`ValueError` if the value cannot be interpreted as a + boolean. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*, converting it to a boolean. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + +Image classes +^^^^^^^^^^^^^ + +.. class:: Image(imgtype, name=None, cnf={}, master=None, **kw) + + Base class for Tk images. + *imgtype* is the Tk image type, one of ``'photo'`` or ``'bitmap'``. + An image is a named object that can be displayed by widgets through their + *image* option; deleting all references to the :class:`!Image` object + deletes the underlying Tk image. + Usually you create a :class:`PhotoImage` or :class:`BitmapImage` rather than + an :class:`!Image` directly. + + The image's configuration options are given by *cnf* and *kw* and may be + queried and changed later with the mapping protocol (using ``image[key]``) + or with the :meth:`configure` method. + + .. method:: config(**kw) + :no-typesetting: + + .. method:: configure(**kw) + + Modify one or more configuration options of the image. + The valid options depend on the image type; see :class:`PhotoImage` and + :class:`BitmapImage`. + :meth:`config` is an alias of :meth:`!configure`. + + .. method:: height() + + Return the height of the image, in pixels. + + .. method:: width() + + Return the width of the image, in pixels. + + .. method:: type() + + Return the type of the image, that is the value of *imgtype* with which + it was created (for example ``'photo'`` or ``'bitmap'``). + + +.. class:: PhotoImage(name=None, cnf={}, master=None, **kw) + + A full-color image (the Tk ``photo`` image type), stored internally with a + varying degree of transparency per pixel. + It can read and write GIF, PPM/PGM and (in Tk 8.6 and later) PNG files, read + SVG files (in Tk 9.0 and later), and be drawn in widgets. + Inherits from :class:`Image`. + + The configuration options include *data* (the image contents as a string), + *file* (the name of a file to read the contents from), *format* (the name of + the file format handler), *width* and *height* (the size of the image, used + when building it up piece by piece), *gamma* and *palette*. + + .. method:: blank() + + Blank the image; that is, set the entire image to have no data, so that + it is displayed as transparent and the background of whatever window it + is displayed in shows through. + + .. method:: redither() + + Recalculate the dithered image in each window where it is displayed. + This is useful when the image data was supplied in pieces, in which case + the dithered image may not be exactly correct. + + .. versionadded:: next + + .. method:: cget(option) + + Return the current value of the configuration option *option*. + + .. method:: copy(*, from_coords=None, zoom=None, subsample=None) + + Return a new :class:`PhotoImage` with a copy of this image. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + ``(x1, y1)`` and ``(x2, y2)`` specify diagonally opposite corners of the + rectangle. + If *x2* and *y2* are not specified, they default to the bottom-right + corner of the source image. + The pixels copied include the left and top edges of the rectangle but not + the bottom or right edges. + If *from_coords* is not given, the whole source image is copied. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + .. versionchanged:: 3.13 + Added the *from_coords*, *zoom* and *subsample* parameters. + + + .. method:: copy_replace(sourceImage, *, from_coords=None, to=None, \ + shrink=False, zoom=None, subsample=None, \ + compositingrule=None) + + Copy a region from *sourceImage* (which must be a :class:`PhotoImage`) + into this image, possibly with pixel zooming and/or subsampling. + If no options are specified, the whole of *sourceImage* is copied into + this image, starting at coordinates ``(0, 0)``. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + *to* specifies a rectangular sub-region of the destination image to be + affected. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If *x2* and *y2* are not specified, they default to ``(x1, y1)`` plus the + size of the source region (after subsampling and zooming, if specified). + If *x2* and *y2* are specified, the source region is replicated if + necessary to fill the destination region in a tiled fashion. + + If *shrink* is true, the size of the destination image is reduced, if + necessary, so that the region being copied into is at the bottom-right + corner of the image. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + *compositingrule* specifies how transparent pixels in the source image + are combined with the destination image. + With ``'overlay'`` (the default), the old contents of the destination + image remain visible, as if the source image were printed on a piece of + transparent film and placed over the top of the destination. + With ``'set'``, the old contents of the destination image are discarded + and the source image is used as-is. + + .. versionadded:: 3.13 + + + .. method:: data(format=None, *, from_coords=None, background=None, \ + grayscale=False, metadata=None) + + Return the image data. + + *format* specifies the name of the image file format handler to use. + If it is not given, the data is returned as a tuple (one element per row) + of strings containing space-separated (one element per pixel/column) + colors in ``#RRGGBB`` format. + + *from_coords* specifies a rectangular region of the image to be returned. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region, including ``(x1, y1)`` and excluding ``(x2, y2)``. + If *from_coords* is not given, the whole image is returned. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionadded:: 3.13 + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: get(x, y, *, withalpha=False) + + Return the color of the pixel at coordinates (*x*, *y*) as an + ``(r, g, b)`` tuple of three integers between 0 and 255, representing the + red, green and blue components respectively. + If *withalpha* is true, the returned tuple has a fourth element giving + the alpha (opacity) value of the pixel. + + .. versionchanged:: next + Added the *withalpha* parameter, which requires Tcl/Tk 9.0 or newer. + + .. method:: put(data, to=None, *, format=None, metadata=None) + + Set pixels of the image to the colors given in *data*, which must be a + string or a nested sequence of horizontal rows of pixel colors (for + example ``"{red green} {blue yellow}"``). + + *to* specifies the coordinates of the region of the image into which the + data are copied. + It must be a tuple or a list of 2 or 4 integers ``(x1, y1)`` or + ``(x1, y1, x2, y2)`` giving the top-left corner, and optionally the + bottom-right corner, of the region. + The default position is ``(0, 0)``. + + *format* specifies the format of the image *data*, so that only image + file format handlers whose names begin with it are tried. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionchanged:: next + Added the *format* and *metadata* parameters. + + .. method:: read(filename, format=None, *, from_coords=None, to=None, \ + shrink=False, metadata=None) + + Read image data from the file named *filename* into the image. + + *format* specifies the format of the image data in the file. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + *from_coords* specifies a rectangular sub-region of the image file data + to be copied to the destination image. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image in the file. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole of the image in the file is + read. + + *to* specifies the coordinates of the top-left corner of the region of + the image into which the data are read. + The default is ``(0, 0)``. + + If *shrink* is true, the size of the image is reduced, if necessary, so + that the region into which the file data are read is at the bottom-right + corner of the image. + + .. versionadded:: 3.13 + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: subsample(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` based on this image but using only every + *x*-th pixel in the X direction and every *y*-th pixel in the Y + direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + .. method:: transparency_get(x, y) + + Return ``True`` if the pixel at coordinates (*x*, *y*) is fully + transparent, ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: transparency_set(x, y, boolean) + + Make the pixel at coordinates (*x*, *y*) fully transparent if *boolean* + is true, fully opaque otherwise. + + .. versionadded:: 3.8 + + + .. method:: write(filename, format=None, from_coords=None, *, \ + background=None, grayscale=False, metadata=None) + + Write image data from the image to the file named *filename*. + + *format* specifies the name of the image file format handler to use. + If it is not given, the format is guessed from the file extension. + + *from_coords* specifies a rectangular region of the image to be written. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole image is written. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + *metadata* is a dictionary passed to the image format driver. + It requires Tcl/Tk 9.0 or newer. + + .. versionchanged:: 3.13 + Added the *background* and *grayscale* parameters. + + .. versionchanged:: next + Added the *metadata* parameter. + + + .. method:: zoom(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` with this image magnified by a factor of + *x* in the X direction and *y* in the Y direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + +.. class:: BitmapImage(name=None, cnf={}, master=None, **kw) + + A two-color image (the Tk ``bitmap`` image type) created from an X11 bitmap. + Each pixel displays a foreground color, a background color, or nothing + (producing a transparent effect). + Inherits from :class:`Image`. + + The configuration options are *data* or *file* (the source bitmap, given as + a string in X11 bitmap format or as the name of a file in that format), + *maskdata* or *maskfile* (the mask bitmap, in the same forms), and + *foreground* and *background* (the two colors). + For pixels where the mask is zero the image displays nothing; for other + pixels it displays the foreground color where the source is one and the + background color where the source is zero. + If *background* is set to an empty string, the background pixels are + transparent. + + :class:`!BitmapImage` has no methods of its own beyond those inherited from + :class:`Image`. + + +Other classes +^^^^^^^^^^^^^ + +.. class:: Event() + + A container for the attributes of an event passed to a callback bound with + :meth:`Misc.bind`. + An :class:`!Event` instance has the following attributes, each corresponding + to a field of the underlying Tk event; depending on the event type, some + attributes may be set to the string ``'??'`` to indicate that they are not + meaningful. + See :ref:`bindings-and-events`. + + .. attribute:: serial + + The serial number of the event. + + .. attribute:: num + + The mouse button that was pressed or released (for button events). + + .. attribute:: focus + + Whether the window has the focus (for ``Enter`` and ``Leave`` events). + + .. attribute:: height + width + + The new height and width of the window (for ``Configure`` and ``Expose`` + events). + + .. attribute:: keycode + + The keycode of the key that was pressed or released. + + .. attribute:: state + + The state of the event, as a number (for most events) or a string (for + ``Visibility`` events). + + .. attribute:: time + + The timestamp of the event, in milliseconds. + + .. attribute:: x + y + + The pointer position relative to the widget, in pixels. + + .. attribute:: x_root + y_root + + The pointer position relative to the top-left corner of the screen, in + pixels. + + .. attribute:: char + + The character typed, as a string (for key events). + + .. attribute:: send_event + + ``True`` if the event was sent by another application. + + .. attribute:: keysym + + The symbolic name of the key that was pressed or released. + + .. attribute:: keysym_num + + The numeric value of :attr:`keysym`. + + .. attribute:: type + + The :class:`EventType` of the event. + + .. attribute:: widget + + The widget on which the event occurred. + + .. attribute:: delta + + The amount the mouse wheel was rotated (for ``MouseWheel`` events). + + .. attribute:: user_data + + The data string of a virtual event, as passed to the *data* option of + :meth:`Misc.event_generate`. + It is ``'??'`` for non-virtual events. + + .. versionadded:: 3.15 + + .. attribute:: detail + + A fixed detail string for ``Enter``, ``Leave``, ``FocusIn``, ``FocusOut`` + and ``ConfigureRequest`` events (see the Tcl/Tk documentation). + It is ``'??'`` for other events. + + .. versionadded:: 3.15 + + +.. class:: EventType(*values) + + An :class:`enum.StrEnum` enumerating the Tk event types, used as the value + of :attr:`Event.type`. + Its members include, among others, ``KeyPress``, ``KeyRelease``, + ``ButtonPress``, ``ButtonRelease``, ``Motion``, ``Enter``, ``Leave``, + ``FocusIn``, ``FocusOut``, ``Configure``, ``Map``, ``Unmap``, ``Expose``, + ``Destroy`` and ``MouseWheel``. + + .. versionadded:: 3.6 + + + +.. class:: CallWrapper(func, subst, widget) + + Internal helper that wraps a Python callback so that it can be invoked from + Tcl. + *func* is the Python function, *subst* is an optional function that + pre-processes the Tcl arguments, and *widget* is the widget used for error + reporting. + Instances are created automatically by :meth:`Misc.register`; this class is + not normally used directly. + + +Module-level functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) + + The :func:`Tcl` function is a factory function which creates an object much + like that created by the :class:`Tk` class, except that it does not + initialize the Tk subsystem. + This is most often useful when driving the Tcl interpreter in an environment + where one doesn't want to create extraneous toplevel windows, or where one + cannot (such as Unix/Linux systems without an X server). + An object created by the :func:`Tcl` object can have a Toplevel window + created (and the Tk subsystem initialized) by calling its :meth:`~Tk.loadtk` + method. + +.. function:: NoDefaultRoot() + + Inhibit the creation of an implicit default root window. + Afterwards :mod:`!tkinter` no longer creates a shared default root + automatically, and operations that rely on one --- such as constructing a + widget without an explicit *master* --- raise a :exc:`RuntimeError`. + Call this early in larger applications to make the root window explicit. + +.. function:: mainloop(n=0) + + Run the Tk main event loop on the default root window until all windows are + destroyed. + Equivalent to calling :meth:`Misc.mainloop` on the default root. + +.. function:: getboolean(s) + + Convert the Tcl boolean string *s* (one of ``'1'``, ``'true'``, ``'yes'``, + ``'on'`` and similar, or their false counterparts) to a Python + :class:`bool`. + Raise :exc:`TclError` for an invalid value. + +.. function:: getdouble(s) + + Convert *s* to a floating-point number. + This is the built-in :class:`float`. + +.. function:: getint(s) + + Convert *s* to an integer. + This is the built-in :class:`int`. + +.. function:: image_names() + + Return the names of all existing images in the default root's interpreter. + +.. function:: image_types() + + Return the available image types (such as ``'photo'`` and ``'bitmap'``) in + the default root's interpreter. + + +.. _tkinter-file-handlers: + +File handlers +^^^^^^^^^^^^^ + +Tk allows you to register and unregister a callback function which will be +called from the Tk mainloop when I/O is possible on a file descriptor. +Only one handler may be registered per file descriptor. Example code:: + + import tkinter + widget = tkinter.Tk() + mask = tkinter.READABLE | tkinter.WRITABLE + widget.tk.createfilehandler(file, mask, callback) + ... + widget.tk.deletefilehandler(file) + +This feature is not available on Windows. + +Since you don't know how many bytes are available for reading, you may not +want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` +:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, +since these will insist on reading a predefined number of bytes. +For sockets, the :meth:`~socket.socket.recv` or +:meth:`~socket.socket.recvfrom` methods will work fine; for other files, +use raw reads or ``os.read(file.fileno(), maxbytecount)``. + + +.. method:: Widget.tk.createfilehandler(file, mask, func) + + Registers the file handler callback function *func*. The *file* argument + may either be an object with a :meth:`~io.IOBase.fileno` method (such as + a file or socket object), or an integer file descriptor. The *mask* + argument is an ORed combination of any of the three constants below. + The callback is called as follows:: + + callback(file, mask) + + +.. method:: Widget.tk.deletefilehandler(file) + + Unregisters a file handler. + + +.. data:: READABLE + WRITABLE + EXCEPTION + + Constants used in the *mask* arguments. + + +Constants +^^^^^^^^^ + +The following symbolic constants are available in both the :mod:`!tkinter` +and :mod:`!tkinter.constants` namespaces. + +.. data:: TRUE + YES + ON + + Truthy values, all equal to the integer ``1``. + +.. data:: FALSE + NO + OFF + + Falsy values, all equal to the integer ``0``. + +.. data:: N + S + E + W + NE + NW + SE + SW + NS + EW + NSEW + CENTER + + Compass directions (``'n'``, ``'s'``, ``'e'``, ``'w'`` and the diagonals and + edges) plus ``CENTER`` (``'center'``), used as values for the *anchor* and + *sticky* options and by methods such as :meth:`Misc.grid_anchor`. + +.. data:: LEFT + RIGHT + TOP + BOTTOM + + Sides for the *side* option of the packer (see :meth:`Pack.pack_configure`). + +.. data:: X + Y + BOTH + NONE + + Values for the *fill* option of the packer: ``'x'``, ``'y'``, ``'both'`` or + ``'none'``. + +.. data:: RAISED + SUNKEN + FLAT + RIDGE + GROOVE + SOLID + + Values for the *relief* option, which controls a widget's 3-D border. + +.. data:: HORIZONTAL + VERTICAL + + Values for the *orient* option of widgets such as :class:`Scale`, + :class:`Scrollbar` and :class:`PanedWindow`. + +.. data:: CHAR + WORD + + Values for the *wrap* option of the :class:`Text` widget, selecting line + wrapping on character or word boundaries. + +.. data:: BASELINE + + The text-alignment value ``'baseline'``. + +.. data:: INSIDE + OUTSIDE + + Values for the *bordermode* option of the placer (see + :meth:`Place.place_configure`). + +.. data:: INSERT + CURRENT + END + ANCHOR + SEL + SEL_FIRST + SEL_LAST + + Symbolic indices used by the :class:`Text`, :class:`Entry`, :class:`Listbox` + and :class:`Canvas` widgets, such as ``'insert'`` (the insertion cursor), + ``'current'``, ``'end'``, ``'anchor'`` and the bounds of the selection + (``'sel.first'`` and ``'sel.last'``). + +.. data:: ALL + + The special tag ``'all'``, which matches every item of a :class:`Canvas` or + every character of a :class:`Text` (for example ``canvas.delete(ALL)``). + +.. data:: NORMAL + DISABLED + ACTIVE + HIDDEN + + Values for the *state* option of various widgets and items. + +.. data:: CASCADE + CHECKBUTTON + COMMAND + RADIOBUTTON + SEPARATOR + + Menu entry types, used as the *itemType* argument of :meth:`Menu.add` and + :meth:`Menu.insert`. + +.. data:: SINGLE + BROWSE + MULTIPLE + EXTENDED + + Values for the *selectmode* option of the :class:`Listbox` widget. + +.. data:: PIESLICE + CHORD + ARC + + Values for the *style* option of :class:`Canvas` arc items. + +.. data:: BUTT + PROJECTING + ROUND + BEVEL + MITER + + Values for the *capstyle* (``'butt'``, ``'projecting'``, ``'round'``) and + *joinstyle* (``'round'``, ``'bevel'``, ``'miter'``) options of + :class:`Canvas` line items. + +.. data:: FIRST + LAST + + Values for the *arrow* option of :class:`Canvas` line items, indicating + which ends have arrowheads. + +.. data:: MOVETO + SCROLL + + The first argument passed by a :class:`Scrollbar` to the :meth:`XView.xview` + or :meth:`YView.yview` method of the scrolled widget. + +.. data:: UNITS + PAGES + + Values for the *what* argument of :meth:`XView.xview_scroll` and + :meth:`YView.yview_scroll`. + +.. data:: UNDERLINE + NUMERIC + DOTBOX + + Other option values: ``'underline'``, ``'numeric'`` and ``'dotbox'``. diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index eb30b9c3eacc1b6..2b147873cc23493 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,4 +1,4 @@ -:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +:mod:`!tkinter.scrolledtext` --- Scrolled text widget ===================================================== .. module:: tkinter.scrolledtext @@ -13,15 +13,26 @@ implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. +The text widget and scrollbar are packed together in a :class:`~tkinter.Frame`, +and the methods of the :class:`~tkinter.Pack`, :class:`~tkinter.Grid` and +:class:`~tkinter.Place` geometry managers are acquired from the +:class:`~tkinter.Frame` object. +This allows the :class:`ScrolledText` widget to be used directly to achieve +most normal geometry management behavior. Should more specific control be necessary, the following attributes are available: -.. class:: ScrolledText(master=None, **kw) +.. class:: ScrolledText(master=None, *, use_ttk=False, **kw) + + The keyword arguments are passed to the :class:`~tkinter.Text` widget. + + When *use_ttk* is true, the surrounding frame and the scroll bar are the + themed :mod:`tkinter.ttk` widgets; + the default is the classic :mod:`tkinter` widgets. + + .. versionchanged:: next + Added the *use_ttk* parameter. .. attribute:: frame diff --git a/Doc/library/tkinter.systray.rst b/Doc/library/tkinter.systray.rst new file mode 100644 index 000000000000000..22ecbf5b71d6aae --- /dev/null +++ b/Doc/library/tkinter.systray.rst @@ -0,0 +1,72 @@ +:mod:`!tkinter.systray` --- System tray icon and notifications +============================================================== + +.. module:: tkinter.systray + :synopsis: System tray icon and desktop notifications + +**Source code:** :source:`Lib/tkinter/systray.py` + +.. versionadded:: next + +-------------- + +The :mod:`!tkinter.systray` module provides the :class:`SysTrayIcon` class +as an interface to the system tray (or taskbar) icon, +and the :func:`notify` function which sends a desktop notification. +They require Tk 8.7/9.0 or newer. + +Only one system tray icon is supported per Tcl interpreter. + +.. class:: SysTrayIcon(master=None, *, exists=False, **options) + + The class implementing the system tray icon. + + With *exists* false (the default), a new icon is created; + creating a second one raises :exc:`~tkinter.TclError`. + With *exists* true, the instance refers to the already-existing icon + instead of creating one, reconfiguring it with any given options. + + The supported configuration options are: + + * *image* --- the image displayed in the system tray + (required when creating an icon). + On Windows, it must be a :class:`!PhotoImage`. + * *text* --- the text displayed in the tooltip of the icon. + * *button1* --- a callback that is called without arguments + when the icon is clicked with the left mouse button. + * *button3* --- a callback that is called without arguments + when the icon is clicked with the right mouse button. + + .. method:: configure(**options) + config(**options) + + Query or modify the options of the system tray icon. + With no arguments, return a dict of all option values. + With a string argument, return the value of that option. + Otherwise, set the given options. + + .. method:: cget(option) + + Return the value of the given option of the system tray icon. + + .. method:: exists() + + Return whether the system tray icon exists. + + .. method:: destroy() + + Destroy the system tray icon. + A new icon can be created afterwards. + + .. method:: notify(title, message) + + Send a desktop notification with the given title and message. + + +.. function:: notify(title, message, *, master=None) + + Send a desktop notification with the given title and message + without creating a system tray icon first. + On Windows, sending a notification requires an existing system + tray icon, which is also displayed in the notification; + use the :meth:`SysTrayIcon.notify` method instead. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index e1383e189a31a2b..b7ab8d25001d1de 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -11,19 +11,28 @@ -------------- The :mod:`!tkinter.ttk` module provides access to the Tk themed widget set, -introduced in Tk 8.5. It provides additional benefits including anti-aliased font -rendering under X11 and window transparency (requiring a composition -window manager on X11). +introduced in Tk 8.5. +Its widgets adapt their appearance to the platform's native theme, +giving an application a better and more consistent look and feel +than the classic :mod:`tkinter` widgets, whose appearance is fixed. The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. +Ttk widgets are used just like the classic :mod:`tkinter` widgets +and share the same machinery: +the widget hierarchy, the geometry managers, variable coupling and event binding. +Those foundational concepts are covered in the :mod:`tkinter` documentation +and are not repeated here. + +.. versionadded:: 3.1 + .. seealso:: - `Tk Widget Styling Support `_ - A document introducing theming support for Tk + `Tk Widget Styling Support (TIP #48) `_ + The Tcl Improvement Proposal that introduced the themed widget styling engine. Using Ttk @@ -40,26 +49,28 @@ To override the basic Tk widgets, the import should follow the Tk import:: That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, -:class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to +:class:`LabelFrame`, :class:`Menubutton`, :class:`OptionMenu`, +:class:`PanedWindow`, :class:`Radiobutton`, :class:`Scale`, +:class:`Scrollbar` and :class:`Spinbox`) to automatically replace the Tk widgets. -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. +.. note:: + Overriding the classic widgets with ``from tkinter.ttk import *`` + is convenient for adapting existing code, + but new code is usually clearer if it imports the module + as ``from tkinter import ttk`` and refers to the themed widgets explicitly, + such as ``ttk.Button``. -.. seealso:: +This has the direct benefit of using the new widgets which gives a better look +and feel across platforms; however, the replacement widgets are not completely +compatible. +The main difference is that widget options such as ``fg``, ``bg`` and others +related to widget styling are no longer present in Ttk widgets. +Instead, use the :class:`ttk.Style {%- endblock -%} {% block body %}

{{ docstitle|e }}

@@ -21,63 +50,82 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

{% trans %}Documentation sections:{% endtrans %}

- - -
-

{% trans %}What's new in Python {{ version }}?{% endtrans %}
- {% trans whatsnew_index=pathto("whatsnew/index") %}Or all "What's new" documents since Python 2.0{% endtrans %}

-

{% trans %}Tutorial{% endtrans %}
- {% trans %}Start here: a tour of Python's syntax and features{% endtrans %}

-

{% trans %}Library reference{% endtrans %}
- {% trans %}Standard library and builtins{% endtrans %}

-

{% trans %}Language reference{% endtrans %}
- {% trans %}Syntax and language elements{% endtrans %}

-

{% trans %}Python setup and usage{% endtrans %}
- {% trans %}How to install, configure, and use Python{% endtrans %}

-

{% trans %}Python HOWTOs{% endtrans %}
- {% trans %}In-depth topic manuals{% endtrans %}

- 		-

{% trans %}Installing Python modules{% endtrans %}
- {% trans %}Third-party modules and PyPI.org{% endtrans %}

-

{% trans %}Distributing Python modules{% endtrans %}
- {% trans %}Publishing modules for use by other people{% endtrans %}

-

{% trans %}Extending and embedding{% endtrans %}
- {% trans %}For C/C++ programmers{% endtrans %}

-

{% trans %}Python's C API{% endtrans %}
- {% trans %}C API reference{% endtrans %}

-

{% trans %}FAQs{% endtrans %}
- {% trans %}Frequently asked questions (with answers!){% endtrans %}

-

{% trans %}Deprecations{% endtrans %}
- {% trans %}Deprecated functionality{% endtrans %}

-
+
+ + +
+ +

{% trans %}Other resources:{% endtrans %}

+
+ + +

{% trans %}Indices, glossary, and search:{% endtrans %}

- - -
-

{% trans %}Global module index{% endtrans %}
- {% trans %}All modules and libraries{% endtrans %}

-

{% trans %}General index{% endtrans %}
- {% trans %}All functions, classes, and terms{% endtrans %}

-

{% trans %}Glossary{% endtrans %}
- {% trans %}Terms explained{% endtrans %}

- 		-

{% trans %}Search page{% endtrans %}
- {% trans %}Search this documentation{% endtrans %}

-

{% trans %}Complete table of contents{% endtrans %}
- {% trans %}Lists all sections and subsections{% endtrans %}

-
+
+ + +

{% trans %}Project information:{% endtrans %}

- - -
-

{% trans %}Reporting issues{% endtrans %}

-

{% trans %}Contributing to docs{% endtrans %}

-

{% trans %}Download the documentation{% endtrans %}

- 		-

{% trans %}History and license of Python{% endtrans %}

-

{% trans %}Copyright{% endtrans %}

-

{% trans %}About the documentation{% endtrans %}

-
+
+ + +
{% endblock %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 086f15662cf87be..3fd707babd782bb 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -1,17 +1,8 @@

{% trans %}Download{% endtrans %}

-

{% trans %}Download these documents{% endtrans %}

+

{% trans %}Download the documentation{% endtrans %}

{% trans %}Docs by version{% endtrans %}

-

{% trans %}Other resources{% endtrans %}

- diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 276e31a3056f0ee..77dd8a6e66d5cc4 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -167,7 +167,8 @@ It is also possible to use a list as a queue, where the first element added is the first element retrieved ("first-in, first-out"); however, lists are not efficient for this purpose. While appends and pops from the end of list are fast, doing inserts or pops from the beginning of a list is slow (because all -of the other elements have to be shifted by one). +of the other elements have to be shifted by one). See +:ref:`time-complexity` for more information. To implement a queue, use :class:`collections.deque` which was designed to have fast appends and pops from both ends. For example:: diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 72cac1c1e909d35..45a2ca21801437f 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -16,7 +16,7 @@ Unix shell's search path makes it possible to start it by typing the command: .. code-block:: text - python3.15 + python3.16 to the shell. [#]_ Since the choice of the directory where the interpreter lives is an installation option, other places are possible; check with your local @@ -97,8 +97,8 @@ before printing the first prompt: .. code-block:: shell-session - $ python3.15 - Python 3.15 (default, May 7 2025, 15:46:04) + $ python3.16 + Python 3.16 (default, May 7 2026, 19:03:04) [GCC 10.2.0] on linux Type "help", "copyright", "credits" or "license" for more information. >>> diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 7778e37a9adaa95..465c32d0b72431c 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -532,10 +532,13 @@ This example introduces several new features. and ``!=`` (not equal to). * The *body* of the loop is *indented*: indentation is Python's way of grouping - statements. At the interactive prompt, you have to type a tab or space(s) for - each indented line. In practice you will prepare more complicated input - for Python with a text editor; all decent text editors have an auto-indent - facility. When a compound statement is entered interactively, it must be + statements. At the interactive prompt, the default REPL automatically + indents continuation lines after compound statement headers like ``if`` or + ``while``. In the basic REPL (invoked with :envvar:`PYTHON_BASIC_REPL`) + or in older Python versions, you have to type a tab or space(s) for each + indented line manually. In practice you will prepare more complicated + input for Python with a text editor; all decent text editors have an + auto-indent facility. When a compound statement is entered interactively, it must be followed by a blank line to indicate completion (since the parser cannot guess when you have typed the last line). Note that each line within a basic block must be indented by the same amount. diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index f8105cd5441fec2..46a226be606aeba 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -584,20 +584,6 @@ Since the main module does not have a package, modules intended for use as the main module of a Python application must always use absolute imports. -Packages in Multiple Directories --------------------------------- - -Packages support one more special attribute, :attr:`~module.__path__`. This is -initialized to be a :term:`sequence` of strings containing the name of the -directory holding the -package's :file:`__init__.py` before the code in that file is executed. This -variable can be modified; doing so affects future searches for modules and -subpackages contained in the package. - -While this feature is not often needed, it can be used to extend the set of -modules found in a package. - - .. rubric:: Footnotes .. [#] In fact function definitions are also 'statements' that are 'executed'; the diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index e7c641044740504..903d8eae2b95b40 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -15,7 +15,7 @@ operating system:: >>> import os >>> os.getcwd() # Return the current working directory - 'C:\\Python315' + 'C:\\Python316' >>> os.chdir('/server/accesslogs') # Change current working directory >>> os.system('mkdir today') # Run the command mkdir in the system shell 0 @@ -36,7 +36,7 @@ aids for working with large modules like :mod:`os`:: For daily file and directory management tasks, the :mod:`shutil` module provides -a higher level interface that is easier to use:: +a higher-level interface that is easier to use:: >>> import shutil >>> shutil.copyfile('data.db', 'archive.db') @@ -63,7 +63,7 @@ wildcard searches:: Command-line arguments ====================== -Common utility scripts often need to process command line arguments. These +Common utility scripts often need to process command-line arguments. These arguments are stored in the :mod:`sys` module's *argv* attribute as a list. For instance, let's take the following :file:`demo.py` file:: @@ -77,7 +77,7 @@ line:: ['demo.py', 'one', 'two', 'three'] The :mod:`argparse` module provides a more sophisticated mechanism to process -command line arguments. The following script extracts one or more filenames +command-line arguments. The following script extracts one or more filenames and an optional number of lines to be displayed:: import argparse diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 678b71c9274c1c6..cf3a44850b0b8f2 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -1,7 +1,7 @@ .. _tut-brieftourtwo: ********************************************** -Brief Tour of the Standard Library --- Part II +Brief tour of the standard library --- part II ********************************************** This second tour covers more advanced modules that support professional @@ -10,7 +10,7 @@ programming needs. These modules rarely occur in small scripts. .. _tut-output-formatting: -Output Formatting +Output formatting ================= The :mod:`reprlib` module provides a version of :func:`repr` customized for @@ -130,7 +130,7 @@ templates for XML files, plain text reports, and HTML web reports. .. _tut-binary-formats: -Working with Binary Data Record Layouts +Working with binary data record layouts ======================================= The :mod:`struct` module provides :func:`~struct.pack` and @@ -178,14 +178,13 @@ tasks in background while the main program continues to run:: class AsyncZip(threading.Thread): def __init__(self, infile, outfile): - threading.Thread.__init__(self) + super().__init__() self.infile = infile self.outfile = outfile def run(self): - f = zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) - f.write(self.infile) - f.close() + with zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) as f: + f.write(self.infile) print('Finished background zip of:', self.infile) background = AsyncZip('mydata.txt', 'myarchive.zip') @@ -245,7 +244,7 @@ application. .. _tut-weak-references: -Weak References +Weak references =============== Python does automatic memory management (reference counting for most objects and @@ -279,14 +278,14 @@ applications include caching objects that are expensive to create:: Traceback (most recent call last): File "", line 1, in d['primary'] # entry was automatically removed - File "C:/python315/lib/weakref.py", line 46, in __getitem__ + File "C:/python316/lib/weakref.py", line 46, in __getitem__ o = self.data[key]() KeyError: 'primary' .. _tut-list-tools: -Tools for Working with Lists +Tools for working with lists ============================ Many data structure needs can be met with the built-in list type. However, @@ -352,7 +351,7 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating-Point Arithmetic +Decimal floating-point arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f362e1943b666f7..6b6b8a768e3f58d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +Note that the activated virtual environment does not alter the ``PYTHONPATH`` variable in any way. +This may lead to unexpected results if the path includes references to code which is incompatible with +the Python version the virtual environment is using. The best practice is to ``unset PYTHONPATH`` +in bash or the equivalent for the shell you are using. + To deactivate a virtual environment, type:: deactivate diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 45345d045ddfd99..36d82c11d95307e 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,14 +30,14 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ * `Termux `__ If you're sure you want to do all of this manually, read on. You can use the -:source:`testbed app ` as a guide; each step below contains a +:source:`testbed app ` as a guide; each step below contains a link to the relevant file. * First, acquire a build of Python for Android: @@ -47,10 +47,10 @@ link to the relevant file. mentioned below is at the top level of the package. * Or if you want to build it yourself, follow the instructions in - :source:`Android/README.md`. The ``prefix`` directory will be created under + :source:`Platforms/Android/README.md`. The ``prefix`` directory will be created under :samp:`cross-build/{HOST}`. -* Add code to your :source:`build.gradle ` +* Add code to your :source:`build.gradle ` file to copy the following items into your project. All except your own Python code can be copied from ``prefix/lib``: @@ -65,10 +65,10 @@ link to the relevant file. * ``python*.*/site-packages`` (your own Python code) * Add code to your app to :source:`extract the assets to the filesystem - `. + `. * Add code to your app to :source:`start Python in embedded mode - `. This will need to be C code + `. This will need to be C code called via JNI. Building a Python package for Android diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 7cbc03f5f1281ec..bd75afc18d03aea 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -50,8 +50,8 @@ additional methods of invocation: * When called with ``-c command``, it executes the Python statement(s) given as *command*. Here *command* may contain multiple statements separated by newlines. -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. +* When called with ``-m module-name``, the given module is located using the standard + import mechanism and executed as a script. In non-interactive mode, the entire input is parsed before it is executed. @@ -78,8 +78,8 @@ source. .. option:: -m - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. + Locate the module using the standard import mechanism and execute its contents + as the :mod:`__main__` module. Since the argument is a *module* name, you must not give a file extension (``.py``). The module name should be a valid absolute Python module name, but @@ -519,7 +519,25 @@ Miscellaneous options .. option:: -x Skip the first line of the source, allowing use of non-Unix forms of - ``#!cmd``. This is intended for a DOS specific hack only. + ``#!cmd``. + + This can be used to turn a Python script into a Windows batch file. + Similarly to adding a shebang line and setting the executable bit on Unix, + the extension of the Python script can be changed to ``.bat`` and the + following line can be added at the start of the script: + + .. code-block:: batch + + @py -x "%~f0" %* & exit /b + + Or, to specify the path to the Python interpreter explicitly: + + .. code-block:: batch + + @"C:\Path\to\python.exe" -x "%~f0" %* & exit /b + + Unlike a shebang line which is a Python comment, this line is not valid + Python syntax, and the :option:`-x` option is needed to skip it. .. option:: -X @@ -662,7 +680,7 @@ Miscellaneous options initialization. Python needs to be :ref:`built in debug mode ` for this option to exist. See also :envvar:`PYTHON_PRESITE`. - .. versionchanged:: next + .. versionchanged:: 3.15 Accept also ``module:func`` entry point format. .. versionadded:: 3.13 @@ -705,10 +723,9 @@ Miscellaneous options .. versionadded:: 3.14 - * :samp:`-X lazy_imports={all,none,normal}` controls lazy import behavior. - ``all`` makes all imports lazy by default, ``none`` disables lazy imports - entirely (even explicit ``lazy`` statements become eager), and ``normal`` - (the default) respects the ``lazy`` keyword in source code. + * :samp:`-X lazy_imports={all,normal}` controls lazy import behavior. + ``all`` makes all imports lazy by default, and ``normal`` (the default) + respects the ``lazy`` keyword in source code. See also :envvar:`PYTHON_LAZY_IMPORTS`. .. versionadded:: 3.15 @@ -1154,9 +1171,6 @@ conflict. 'replace', respectively. Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used. - This may also be enabled at runtime with - :func:`sys._enablelegacywindowsfsencoding`. - .. availability:: Windows. .. versionadded:: 3.6 @@ -1416,10 +1430,9 @@ conflict. .. envvar:: PYTHON_LAZY_IMPORTS - Controls lazy import behavior. Accepts three values: ``all`` makes all - imports lazy by default, ``none`` disables lazy imports entirely (even - explicit ``lazy`` statements become eager), and ``normal`` (the default) - respects the ``lazy`` keyword in source code. + Controls lazy import behavior. Accepts two values: ``all`` makes all + imports lazy by default, and ``normal`` (the default) respects the + ``lazy`` keyword in source code. See also the :option:`-X lazy_imports <-X>` command-line option. @@ -1462,7 +1475,7 @@ Debug-mode variables Needs Python configured with the :option:`--with-pydebug` build option. - .. versionchanged:: next + .. versionchanged:: 3.15 Accept also ``module:func`` entry point format. .. versionadded:: 3.13 diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 82409ac0cdcfa6e..8b4940ceb9521a7 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -71,7 +71,8 @@ Dependencies to build optional modules are: * - Dependency - Minimum version - Python module - * - `libbz2 `_ + * - `libbz2 `_ or + `libbzip2-rs `_ [1]_ - - :mod:`bz2` * - `libffi `_ @@ -82,48 +83,54 @@ Dependencies to build optional modules are: - :mod:`lzma` * - `libmpdec `_ - 2.5.0 - - :mod:`decimal` [1]_ + - :mod:`decimal` [2]_ * - `libreadline `_ or - `libedit `_ [2]_ + `libedit `_ [3]_ - - :mod:`readline` * - `libuuid `_ - - - ``_uuid`` [3]_ - * - `ncurses `_ [4]_ + - ``_uuid`` [4]_ + * - `ncurses `_ [5]_ - - :mod:`curses` * - `OpenSSL `_ - | 3.0.18 recommended | (1.1.1 minimum) - - :mod:`ssl`, :mod:`hashlib` [5]_ + - :mod:`ssl`, :mod:`hashlib` [6]_ * - `SQLite `_ - 3.15.2 - :mod:`sqlite3` * - `Tcl/Tk `_ - 8.5.12 - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` - * - `zlib `_ - - 1.2.2.1 + * - `zlib `_, + `zlib-ng `_, or + `zlib-rs `_ [7]_ + - | 1.2.2.1 (zlib, zlib-ng) + | 0.6.0 (zlib-rs) - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` * - `zstd `_ - 1.4.5 - :mod:`compression.zstd` -.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use +.. [1] See :option:`--with-bzip2` for choosing the backend for the + :mod:`bz2` module. +.. [2] If *libmpdec* is not available, the :mod:`decimal` module will use a pure-Python implementation. - See :option:`--with-system-libmpdec` for details. -.. [2] See :option:`--with-readline` for choosing the backend for the +.. [3] See :option:`--with-readline` for choosing the backend for the :mod:`readline` module. -.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. +.. [4] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. See the module documentation for details. -.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` +.. [5] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` library. The :mod:`curses.panel` module additionally requires the ``libpanel`` or ``libpanelw`` library. -.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use +.. [6] If OpenSSL is not available, the :mod:`hashlib` module will use bundled implementations of several hash functions. See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. +.. [7] See :option:`--with-zlib` for choosing the backend for the + :mod:`zlib` module. Note that the table does not include all optional modules; in particular, platform-specific modules like :mod:`winreg` are not listed here. @@ -474,6 +481,15 @@ General Options .. versionadded:: 3.15 +.. option:: --with-build-details-suffix=[yes|SUFFIX] + + Rename ``build-details.json`` to permit multiple co-located Python + installs. If a custom ``SUFFIX`` is supplied it is used verbatim, + otherwise one will be generated from the ``MULTIARCH`` tag with + ``-free-threading`` and ``-debug``, as appropriate. + + .. versionadded:: 3.16 + C compiler options ------------------ @@ -553,11 +569,6 @@ Options for third-party dependencies C compiler and linker flags for ``libmpdec``, used by :mod:`decimal` module, overriding ``pkg-config``. - .. note:: - - These environment variables have no effect unless - :option:`--with-system-libmpdec` is specified. - .. option:: LIBLZMA_CFLAGS .. option:: LIBLZMA_LIBS @@ -780,6 +791,33 @@ also be used to improve performance. .. versionadded:: 3.14 +.. option:: --without-frame-pointers + + Disable frame pointers, which are enabled by default (see :pep:`831`). + + By default, the build appends flags to generate frame or backchain + pointers to ``BASECFLAGS``: + + - ``-fno-omit-frame-pointer`` and/or ``-mno-omit-leaf-frame-pointer`` + are added when the compiler supports them. + - ``-marm`` and/or ``-mno-thumb`` is added on 32-bit ARM when supported, + - on s390x platforms, when supported, ``-mbackchain`` is added *instead*. + of the above frame pointer flags. + - on ppc64le platforms, no compiler flags is needed since the power ABI + requires that compilers maintain a back chain by default. + + Frame pointers enable profilers, debuggers, and system tracing tools + (``perf``, ``eBPF``, ``dtrace``, ``gdb``) to walk the C call stack + without DWARF metadata. The flags propagate to third-party C + extensions through :mod:`sysconfig`. On compilers that do not + understand them, the build silently skips them. + + Downstream packagers and authors of native libraries built with + custom build systems should set the same flags so the unwind chain + stays unbroken across all native frames. + + .. versionadded:: 3.15 + .. option:: --without-mimalloc Disable the fast :ref:`mimalloc ` allocator @@ -1012,6 +1050,21 @@ Linker options .. versionadded:: 3.10 +.. option:: --enable-static-libpython-for-interpreter + + Do not link the Python interpreter binary (``python3``) against the + shared Python library; instead, statically link the interpreter + against ``libpython`` as if ``--enable-shared`` had not been used, + but continue to build the shared ``libpython`` (for use by other + programs). + + This option does nothing if ``--enable-shared`` is not used. + + The default (when ``-enable-shared`` is used) is to link the Python + interpreter against the built shared library. + + .. versionadded:: 3.15 + Libraries options ----------------- @@ -1025,29 +1078,6 @@ Libraries options Build the :mod:`!pyexpat` module using an installed ``expat`` library (default is no). -.. option:: --with-system-libmpdec - - Build the ``_decimal`` extension module using an installed ``mpdecimal`` - library, see the :mod:`decimal` module (default is yes). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.13 - Default to using the installed ``mpdecimal`` library. - - .. versionchanged:: 3.15 - - A bundled copy of the library will no longer be selected - implicitly if an installed ``mpdecimal`` library is not found. - In Python 3.15 only, it can still be selected explicitly using - ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. - - .. deprecated-removed:: 3.13 3.16 - A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.16. - - .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. - .. option:: --with-readline=readline|editline Designate a backend library for the :mod:`readline` module. @@ -1065,6 +1095,80 @@ Libraries options .. versionadded:: 3.10 +.. option:: --with-curses=ncursesw|ncurses|curses + + Designate a backend library for the :mod:`curses` and :mod:`curses.panel` + modules. + + * ``ncursesw``: Use the wide-character ``libncursesw`` (and ``libpanelw``), + found via ``pkg-config``. + * ``ncurses``: Use ``libncurses`` (and ``libpanel``), found via + ``pkg-config``. + * ``curses``: Use the system's native ``libcurses`` (and ``libpanel``), such + as on NetBSD or Solaris. It is built with wide-character support when the + library provides it. + + Without this option (or with ``--with-curses=auto``, the default), Python + tries ``ncursesw`` and then ``ncurses``; the native ``curses`` is used only + when requested explicitly. + + .. versionadded:: next + +.. option:: --without-curses + + Don't build the :mod:`curses` and :mod:`curses.panel` modules, even when a + curses library is present (built by default when one is found). + + .. versionadded:: next + +.. option:: --with-zlib=zlib|zlib-ng|zlib-rs + + Designate the zlib implementation to build the :mod:`zlib` module against. + + * ``zlib``: Use the system zlib library, found via ``pkg-config`` or as + ``libz``. + * ``zlib-ng``: Same detection as ``zlib``, but fail unless the detected + library is `zlib-ng `__. + * ``zlib-rs``: Use `zlib-rs + `__, found via + ``pkg-config`` as ``libz_rs``. + + Without this option (or with ``--with-zlib=auto``, the default), the system + zlib is used when found; configure only fails when an implementation was + requested explicitly but cannot be found. + + .. versionadded:: next + +.. option:: --without-zlib + + Don't build the :mod:`zlib` module, even when a zlib library is present + (built by default when one is found). + + .. versionadded:: next + +.. option:: --with-bzip2=bzip2|bzip2-rs + + Designate the bzip2 implementation to build the :mod:`bz2` module against. + + * ``bzip2``: Use the system bzip2 library, found via ``pkg-config`` or as + ``libbz2``. + * ``bzip2-rs``: Use `libbzip2-rs + `__, found via + ``pkg-config`` as ``libbz2_rs``. + + Without this option (or with ``--with-bzip2=auto``, the default), the + system bzip2 is used when found; configure only fails when an + implementation was requested explicitly but cannot be found. + + .. versionadded:: next + +.. option:: --without-bzip2 + + Don't build the :mod:`bz2` module, even when a bzip2 library is present + (built by default when one is found). + + .. versionadded:: next + .. option:: --with-libm=STRING Override ``libm`` math library to *STRING* (default is system-dependent). @@ -1227,7 +1331,7 @@ See :source:`Mac/README.rst`. iOS Options ----------- -See :source:`iOS/README.rst`. +See :source:`Platforms/Apple/iOS/README.md`. .. option:: --enable-framework=INSTALLDIR @@ -1574,6 +1678,12 @@ Compiler flags .. versionadded:: 3.7 +.. envvar:: CFLAGS_CEVAL + + Flags used to compile ``Python/ceval.c``. + + .. versionadded:: 3.14.5 + .. envvar:: CCSHARED Compiler flags used to build a shared library. diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 5e4033fb6cec7a0..31d9e2f2c816e73 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -170,7 +170,7 @@ helpful. To add Python to an iOS Xcode project: 1. Build or obtain a Python ``XCFramework``. See the instructions in - :source:`Apple/iOS/README.md` (in the CPython source distribution) for details on + :source:`Platforms/Apple/iOS/README.md` (in the CPython source distribution) for details on how to build a Python ``XCFramework``. At a minimum, you will need a build that supports ``arm64-apple-ios``, plus one of either ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. @@ -266,13 +266,13 @@ modules in your app, some additional steps will be required: Testing a Python package ------------------------ -The CPython source tree contains :source:`a testbed project ` that +The CPython source tree contains :source:`a testbed project ` that is used to run the CPython test suite on the iOS simulator. This testbed can also be used as a testbed project for running your Python library's test suite on iOS. -After building or obtaining an iOS XCFramework (see :source:`Apple/iOS/README.md` +After building or obtaining an iOS XCFramework (see :source:`Platforms/Apple/iOS/README.md` for details), create a clone of the Python iOS testbed project. If you used the -``Apple`` build script to build the XCframework, you can run: +``Platforms/Apple`` build script to build the XCframework, you can run: .. code-block:: bash @@ -282,7 +282,7 @@ Or, if you've sourced your own XCframework, by running: .. code-block:: bash - $ python Apple/testbed clone --platform iOS --framework --app --app app-testbed + $ python Platforms/Apple/testbed clone --platform iOS --framework --app --app app-testbed Any folders specified with the ``--app`` flag will be copied into the cloned testbed project. The resulting testbed will be created in the ``app-testbed`` diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 6cf945de5b3f3b4..fd0464133ebfb3b 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -1,3 +1,4 @@ +.. highlight:: none .. _using-on-mac: @@ -29,8 +30,8 @@ Installation steps For `current Python versions `_ (other than those in ``security`` status), the release team produces a **Python for macOS** installer package for each new release. -A list of available installers -is available `here `_. +A list of available installers is provided on the +`macOS downloads page `__. We recommend using the most recent supported Python version where possible. Current installers provide a `universal2 binary `_ build @@ -39,14 +40,14 @@ supported by a wide range of macOS versions, currently typically from at least **macOS 10.15 Catalina** on. The downloaded file is a standard macOS installer package file (``.pkg``). -File integrity information (checksum, size, sigstore signature, etc) for each file is included +File integrity information (such as checksum, size, and Sigstore signature) for each file is included on the release download page. Installer packages and their contents are signed and notarized with ``Python Software Foundation`` Apple Developer ID certificates to meet `macOS Gatekeeper requirements `_. For a default installation, double-click on the downloaded installer package file. This should launch the standard macOS Installer app and display the first of several -installer windows steps. +installer steps. .. image:: mac_installer_01_introduction.png @@ -73,8 +74,8 @@ display. For most uses, the standard set of installation operations is appropria By pressing the **Customize** button, you can choose to omit or select certain package components of the installer. Click on each package name to see a description of what it installs. -To also install support for the optional free-threaded feature, -see :ref:`install-freethreaded-macos`. +For details of the optional free-threaded feature, which is installed +by default, see :ref:`install-freethreaded-macos`. .. image:: mac_installer_05_custom_install.png @@ -98,7 +99,7 @@ for its use. .. image:: mac_installer_08_install_certificates.png -If ``Successfully installed certifi`` and ``update complete`` appears +If ``Successfully installed certifi`` and ``update complete`` appear in the terminal window, the installation is complete. Close this terminal window and the installer window. @@ -171,7 +172,7 @@ be certain of what you are about to run. .. _alternative_bundles: -Alternative Distributions +Alternative distributions ========================= Besides the standard ``python.org`` for macOS installer, there are third-party @@ -199,7 +200,7 @@ other libraries, and are not maintained or supported by the core Python team. .. _mac-package-manager: -Installing Additional Python Packages +Installing additional Python packages ===================================== Refer to the `Python Packaging User Guide`_ for more information. @@ -211,7 +212,7 @@ Refer to the `Python Packaging User Guide`_ for more information. .. _gui-programming-on-the-mac: -GUI Programming +GUI programming =============== There are several options for building GUI applications on the Mac with Python. @@ -241,19 +242,22 @@ A number of alternative macOS GUI toolkits are available including: supports desktop operating systems. -Advanced Topics +Advanced topics =============== .. _install-freethreaded-macos: -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- .. versionadded:: 3.13 +.. versionchanged:: 3.15 + Free-threading support is now installed by default. + The ``python.org`` :ref:`Python for macOS ` -installer package can optionally install an additional build of -Python |version| that supports :pep:`703`, the free-threading feature +installer package installs, by default, an additional build of +Python |version| that supports :term:`free threading` (running with the :term:`global interpreter lock` disabled). Check the release page on ``python.org`` for possible updated information. @@ -262,14 +266,15 @@ there is some additional overhead in single-threaded workloads compared to the regular build. Additionally, third-party packages, in particular ones with an :term:`extension module`, may not be ready for use in a free-threaded build, and will re-enable the :term:`GIL`. -Therefore, the support for free-threading -is not installed by default. It is packaged as a separate install option, -available by clicking the **Customize** button on the **Installation Type** +As of Python 3.15, free-threading support is installed by default. +It is packaged as a separate install option, which can be deselected +by clicking the **Customize** button on the **Installation Type** step of the installer as described above. .. image:: mac_installer_09_custom_install_free_threaded.png -If the box next to the **Free-threaded Python** package name is checked, +If the box next to the **Free-threaded Python** package name is checked +(as it is by default), a separate :file:`PythonT.framework` will also be installed alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. This configuration allows a free-threaded Python |version| build to co-exist @@ -284,7 +289,7 @@ Known cautions and limitations: the free-threaded interpreter, and |python_x_dot_y_t_literal_config|, a configuration utility which may be useful for package builders. Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, - in most cases no changes to your ``PATH`` environment variables should + in most cases no changes to your ``PATH`` environment variable should be needed to use |python_x_dot_y_t_literal|. - For this release, the **Shell profile updater** package and the @@ -367,8 +372,8 @@ the macOS command line :command:`installer` utility lets you select non-default options, too. If you are not familiar with :command:`installer`, it can be somewhat cryptic (see :command:`man installer` for more information). As an example, the following shell snippet shows one way to do it, -using the |x_dot_y_b2_literal| release and selecting the free-threaded interpreter -option: +using the |x_dot_y_b2_literal| release and deselecting the free-threaded +interpreter option: .. parsed-literal:: @@ -378,7 +383,7 @@ option: curl -O \https://www.python.org/ftp/python/\ |version|\ .0/${RELEASE} # create installer choicechanges to customize the install: - # enable the PythonTFramework-\ |version|\ package + # disable the PythonTFramework-\ |version|\ package # while accepting the other defaults (install all other packages) cat > ./choicechanges.plist < @@ -387,7 +392,7 @@ option: attributeSetting - 1 + 0 choiceAttribute selected choiceIdentifier @@ -400,7 +405,7 @@ option: sudo installer -pkg ./${RELEASE} -applyChoiceChangesXML ./choicechanges.plist -target / -You can then test that both installer builds are now available with something like: +After a default install, you can test that both builds are available with something like: .. parsed-literal:: @@ -425,7 +430,7 @@ You can then test that both installer builds are now available with something li .. _distributing-python-applications-on-the-mac: -Distributing Python Applications +Distributing Python applications -------------------------------- A range of tools exist for converting your Python code into a standalone @@ -442,7 +447,7 @@ distributable application: * `PyInstaller `_: A cross-platform packaging tool that creates a single file or folder as a distributable artifact. -App Store Compliance +App Store compliance -------------------- Apps submitted for distribution through the macOS App Store must pass Apple's @@ -464,7 +469,7 @@ This patch is not normally required to use CPython on a Mac; nor is it required if you are distributing an app *outside* the macOS App Store. It is *only* required if you are using the macOS App Store as a distribution channel. -Other Resources +Other resources =============== The `python.org Help page `_ has links to many useful resources. diff --git a/Doc/using/mac_installer_01_introduction.png b/Doc/using/mac_installer_01_introduction.png index 1999f3a37590933..81e1f4e5ac5c32e 100644 Binary files a/Doc/using/mac_installer_01_introduction.png and b/Doc/using/mac_installer_01_introduction.png differ diff --git a/Doc/using/mac_installer_02_readme.png b/Doc/using/mac_installer_02_readme.png index a36efaf7d50fd6a..7e4a9b15cf94f6a 100644 Binary files a/Doc/using/mac_installer_02_readme.png and b/Doc/using/mac_installer_02_readme.png differ diff --git a/Doc/using/mac_installer_03_license.png b/Doc/using/mac_installer_03_license.png index 598c22a13d9e62e..153daa780efa9e9 100644 Binary files a/Doc/using/mac_installer_03_license.png and b/Doc/using/mac_installer_03_license.png differ diff --git a/Doc/using/mac_installer_04_installation_type.png b/Doc/using/mac_installer_04_installation_type.png index 9498fd06240a4ed..1fc597c2427ab76 100644 Binary files a/Doc/using/mac_installer_04_installation_type.png and b/Doc/using/mac_installer_04_installation_type.png differ diff --git a/Doc/using/mac_installer_05_custom_install.png b/Doc/using/mac_installer_05_custom_install.png index 3a201d2f44655af..c70d93bbcee70a8 100644 Binary files a/Doc/using/mac_installer_05_custom_install.png and b/Doc/using/mac_installer_05_custom_install.png differ diff --git a/Doc/using/mac_installer_06_summary.png b/Doc/using/mac_installer_06_summary.png index 1af6eee2c668cdb..f8a67052ef04479 100644 Binary files a/Doc/using/mac_installer_06_summary.png and b/Doc/using/mac_installer_06_summary.png differ diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png index c8b3aa1099adfc9..c39741f33ac807f 100644 Binary files a/Doc/using/mac_installer_07_applications.png and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/mac_installer_08_install_certificates.png b/Doc/using/mac_installer_08_install_certificates.png index c125eeb18aa0c14..f476c7f37d245d6 100644 Binary files a/Doc/using/mac_installer_08_install_certificates.png and b/Doc/using/mac_installer_08_install_certificates.png differ diff --git a/Doc/using/mac_installer_09_custom_install_free_threaded.png b/Doc/using/mac_installer_09_custom_install_free_threaded.png index 0f69c55eddb228b..e867e422e42e485 100644 Binary files a/Doc/using/mac_installer_09_custom_install_free_threaded.png and b/Doc/using/mac_installer_09_custom_install_free_threaded.png differ diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png deleted file mode 100644 index 12b89c0165d994d..000000000000000 Binary files a/Doc/using/win_install_freethreaded.png and /dev/null differ diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png deleted file mode 100644 index fc9605a79cfcc51..000000000000000 Binary files a/Doc/using/win_installer.png and /dev/null differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 1a913c624e99b02..3a32c193e3a8fc8 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -59,6 +59,8 @@ packages. .. _setting-envvars: .. _windows-path-mod: .. _launcher: +.. _getting-started: +.. _installation-steps: Python install manager ====================== @@ -128,7 +130,7 @@ difference between the two commands is when running without any arguments: help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a -console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` +console window. These are ``pyw``, ``pythonw`` and ``pywmanager``. A ``python3`` command is also included that mimics the ``python`` command. It is intended to catch accidental uses of the typical POSIX command on Windows, but is not meant to be widely used or recommended. @@ -162,7 +164,9 @@ omitted in cases where the tag refers to an official release and starts with $> py -V:3-arm64 ... Runtimes from other distributors may require the *company* to be included as -well. This should be separated from the tag by a slash, and may be a prefix. +well. +It should be separated from the tag by a slash (either ``/`` or ``\``), +and may be shortened to any prefix of its full value. Specifying the company is optional when it is ``PythonCore``, and specifying the tag is optional (but not the slash) when you want the latest release from a specific company. @@ -313,6 +317,8 @@ for new users, but should not generally be relied on rather than using the .. _pymanager-offline: +.. _install-layout-option: +.. _installing-without-downloading: Offline installs ---------------- @@ -376,6 +382,10 @@ again. To completely clean up all Python runtimes, run with ``--purge`` before uninstalling the Python install manager. .. _pymanager-config: +.. _customization: +.. _customizing-default-python-versions: +.. _customization-via-ini-files: +.. _launcher-ini: Configuration ------------- @@ -455,6 +465,12 @@ customization. Python runtimes, or false to prevent it. By default, true. + * - ``shebang_templates`` + - (none) + - Mapping from shebang line template to alternative command, such as + ``py -V:`` or a substitute string. + See :ref:`pymanager-shebang` for more details. + * - ``log_level`` - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` - Set the default level of output (0-50). @@ -510,6 +526,7 @@ Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. .. _pymanager-shebang: +.. _arguments-in-shebang-lines: Shebang lines ------------- @@ -568,6 +585,30 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. +Since version 26.3 of the Python install manager, custom shebang templates may +be added to your configuration file. Add the ``shebang_templates`` object with +one member for each template (the string to match) and the command to use when +the template is matched. Most commands should be ``py -V:`` (or ``pyw``) to +launch one of your installed runtimes. The ``py -3.`` form is also +allowed, as is a plain ``py`` to launch the default. No other arguments are +supported. + +.. code:: json5 + + { + "shebang_templates": { + "/usr/bin/python": "py", + "/usr/bin/my_custom_python": "py -V:MyCustomPython/3" + } + } + +If the substitute command is not ``py`` or ``pyw``, it will be written back into +the shebang and regular handling continues. If launching arbitrary executables +is permitted, then providing a full path will allow you to redirect from Python +to any executable. The template should match either the entire line (ignoring +leading and trailing whitespace), or up to the first space in the shebang line. + + .. note:: The behaviour of shebangs in the Python install manager is subtly different @@ -778,6 +819,14 @@ directory containing the configuration file that specified them. - True to suppress visible warnings when a shebang launches an application other than a Python runtime. + * - ``source_settings`` + - A mapping from source URL to settings specific to that index. + When multiple configuration files include this section, URL settings are + added or overwritten, but individual settings are not merged. + These settings are currently only for :ref:`index signatures + `. + + .. _install-freethreaded-windows: Installing free-threaded binaries @@ -799,6 +848,101 @@ installed, then ``python`` will launch this one. Otherwise, you will need to use ``py -V:3.14t ...`` or, if you have added the global aliases directory to your :envvar:`PATH` environment variable, the ``python3.14t.exe`` commands. + +.. _pymanager-index-signatures: + +Index signatures +---------------- + +.. versionadded:: 26.2 + +Index files may be signed to detect tampering. A signature is a catalog file +at the same URL as the index with ``.cat`` added to the filename. The catalog +file should contain the hash of its matching index file, and should be signed +with a valid Authenticode signature. This allows standard tooling (on Windows) +to generate a signature, and any certificate may be used as long as the client +operating system already trusts its certification authority (root CA). + +Index signatures are only downloaded and checked when the local configuration's +``source_settings`` section includes the index URL and ``requires_signature`` is +true, or the index JSON contains ``requires_signature`` set to true. When the +setting exists in local configuration, even when false, settings in the index +are ignored. + +As well as requiring a valid signature, the ``required_root_subject`` and +``required_publisher_subject`` settings can further restrict acceptable +signatures based on the certificate Subject fields. Any attribute specified in +the configuration must match the attribute in the certificate (additional +attributes in the certificate are ignored). Typical attributes are ``CN=`` for +the common name, ``O=`` for the organizational unit, and ``C=`` for the +publisher's country. + +Finally, the ``required_publisher_eku`` setting allows requiring that a specific +Enhanced Key Usage (EKU) has been assigned to the publisher certificate. For +example, the EKU ``1.3.6.1.5.5.7.3.3`` indicates that the certificate was +intended for code signing (as opposed to server or client authentication). +In combination with a specific root CA, this provides another mechanism to +verify a legitimate signature. + +This is an example ``source_settings`` section from a configuration file. In +this case, the publisher of the feed is uniquely identified by the combination +of the Microsoft Identity Verification root and the EKU assigned by that root. +The signature for this case would be found at +``https://www.python.org/ftp/python/index-windows.json.cat``. + +.. code:: json5 + + { + "source_settings": { + "https://www.python.org/ftp/python/index-windows.json": { + "requires_signature": true, + "required_root_subject": "CN=Microsoft Identity Verification Root Certificate Authority 2020", + "required_publisher_subject": "CN=Python Software Foundation", + "required_publisher_eku": "1.3.6.1.4.1.311.97.608394634.79987812.305991749.578777327" + } + } + } + +The same settings could be specified in the ``index.json`` file instead. In this +case, the root and EKU are omitted, meaning that the signature must be valid and +have a specific common name in the publisher's certificate, but no other checks +are used. + +.. code:: json5 + + { + "requires_signature": true, + "required_publisher_subject": "CN=Python Software Foundation", + "versions": [ + // ... + ] + } + +When settings from inside a feed are used, the user is notified and the settings +are shown in the log file or verbose output. It is recommended to copy these +settings into a local configuration file for feeds that will be used frequently, +so that unauthorised modifications to the feed cannot disable verification. + +It is not possible to override the location of the signature file in the feed or +through a configuration file. Administrators can provide their own +``source_settings`` in a mandatory configuration file (see +:ref:`pymanager-admin-config`). + +If signature validation fails, you will be notified and prompted to continue. +When interactive confirmation is not allowed (for example, because ``--yes`` was +specified), it will always abort. To use a feed with invalid configuration in +this scenario, you must provide a configuration file that disables signature +checking for that feed. + +.. code:: json5 + + "source_settings": { + "https://www.example.com/feed-with-invalid-signature.json": { + "requires_signature": false + } + } + + .. _pymanager-troubleshoot: Troubleshooting @@ -1041,9 +1185,9 @@ on using nuget. What follows is a summary that is sufficient for Python developers. The ``nuget.exe`` command line tool may be downloaded directly from -``https://aka.ms/nugetclidl``, for example, using curl or PowerShell. With the -tool, the latest version of Python for 64-bit or 32-bit machines is installed -using:: +``https://dist.nuget.org/win-x86-commandline/latest/nuget.exe``, for example, +using curl or PowerShell. With the tool, the latest version of Python for +64-bit or 32-bit machines is installed using:: nuget.exe install python -ExcludeVersion -OutputDirectory . nuget.exe install pythonx86 -ExcludeVersion -OutputDirectory . @@ -1082,6 +1226,10 @@ for the 64-bit version, `www.nuget.org/packages/pythonx86 `www.nuget.org/packages/pythonarm64 `_ for the ARM64 version +Resolve error ``Unable to find package 'python'`` from a misconfigured ``nuget.config`` using:: + + nuget.exe sources add -Name "nuget.org" -source "https://api.nuget.org/v3/index.json" + Free-threaded packages ---------------------- @@ -1366,725 +1514,3 @@ releases. These files are in the :file:`PCbuild` directory. Check :file:`PCbuild/readme.txt` for general information on the build process. For extension modules, consult :ref:`building-on-windows`. - - - -.. _windows-full: - -The full installer (deprecated) -=============================== - -.. deprecated:: 3.14 - - This installer is deprecated since 3.14 and will not be produced for Python - 3.16 or later. See :ref:`pymanager` for the modern installer. - - -Installation steps ------------------- - -Four Python |version| installers are available for download - two each for the -32-bit and 64-bit versions of the interpreter. The *web installer* is a small -initial download, and it will automatically download the required components as -necessary. The *offline installer* includes the components necessary for a -default installation and only requires an internet connection for optional -features. See :ref:`install-layout-option` for other ways to avoid downloading -during installation. - -After starting the installer, one of two options may be selected: - -.. image:: win_installer.png - -If you select "Install Now": - -* You will *not* need to be an administrator (unless a system update for the - C Runtime Library is required or you install the :ref:`launcher` for all - users) -* Python will be installed into your user directory -* The :ref:`launcher` will be installed according to the option at the bottom - of the first page -* The standard library, test suite, launcher and pip will be installed -* If selected, the install directory will be added to your :envvar:`PATH` -* Shortcuts will only be visible for the current user - -Selecting "Customize installation" will allow you to select the features to -install, the installation location and other options or post-install actions. -To install debugging symbols or binaries, you will need to use this option. - -To perform an all-users installation, you should select "Customize -installation". In this case: - -* You may be required to provide administrative credentials or approval -* Python will be installed into the Program Files directory -* The :ref:`launcher` will be installed into the Windows directory -* Optional features may be selected during installation -* The standard library can be pre-compiled to bytecode -* If selected, the install directory will be added to the system :envvar:`PATH` -* Shortcuts are available for all users - - -Removing the MAX_PATH limitation --------------------------------- - -Windows historically has limited path lengths to 260 characters. This meant that -paths longer than this would not resolve and errors would result. - -In the latest versions of Windows, this limitation can be expanded to -approximately 32,000 characters. Your administrator will need to activate the -"Enable Win32 long paths" group policy, or set ``LongPathsEnabled`` to ``1`` -in the registry key -``HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem``. - -This allows the :func:`open` function, the :mod:`os` module and most other -path functionality to accept and return paths longer than 260 characters. - -After changing the above option, no further configuration is required. - -.. versionchanged:: 3.6 - - Support for long paths was enabled in Python. - -.. _install-quiet-option: - -Installing without UI ---------------------- - -All of the options available in the installer UI can also be specified from the -command line, allowing scripted installers to replicate an installation on many -machines without user interaction. These options may also be set without -suppressing the UI in order to change some of the defaults. - -The following options (found by executing the installer with ``/?``) can be -passed into the installer: - -+---------------------+--------------------------------------------------------+ -| Name | Description | -+=====================+========================================================+ -| /passive | to display progress without requiring user interaction | -+---------------------+--------------------------------------------------------+ -| /quiet | to install/uninstall without displaying any UI | -+---------------------+--------------------------------------------------------+ -| /simple | to prevent user customization | -+---------------------+--------------------------------------------------------+ -| /uninstall | to remove Python (without confirmation) | -+---------------------+--------------------------------------------------------+ -| /layout [directory] | to pre-download all components | -+---------------------+--------------------------------------------------------+ -| /log [filename] | to specify log files location | -+---------------------+--------------------------------------------------------+ - -All other options are passed as ``name=value``, where the value is usually -``0`` to disable a feature, ``1`` to enable a feature, or a path. The full list -of available options is shown below. - -+---------------------------+--------------------------------------+--------------------------+ -| Name | Description | Default | -+===========================+======================================+==========================+ -| InstallAllUsers | Perform a system-wide installation. | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| TargetDir | The installation directory | Selected based on | -| | | InstallAllUsers | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultAllUsersTargetDir | The default installation directory | :file:`%ProgramFiles%\\\ | -| | for all-user installs | Python X.Y` or :file:`\ | -| | | %ProgramFiles(x86)%\\\ | -| | | Python X.Y` | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultJustForMeTargetDir | The default install directory for | :file:`%LocalAppData%\\\ | -| | just-for-me installs | Programs\\Python\\\ | -| | | PythonXY` or | -| | | :file:`%LocalAppData%\\\ | -| | | Programs\\Python\\\ | -| | | PythonXY-32` or | -| | | :file:`%LocalAppData%\\\ | -| | | Programs\\Python\\\ | -| | | PythonXY-64` | -+---------------------------+--------------------------------------+--------------------------+ -| DefaultCustomTargetDir | The default custom install directory | (empty) | -| | displayed in the UI | | -+---------------------------+--------------------------------------+--------------------------+ -| AssociateFiles | Create file associations if the | 1 | -| | launcher is also installed. | | -+---------------------------+--------------------------------------+--------------------------+ -| CompileAll | Compile all ``.py`` files to | 0 | -| | ``.pyc``. | | -+---------------------------+--------------------------------------+--------------------------+ -| PrependPath | Prepend install and Scripts | 0 | -| | directories to :envvar:`PATH` and | | -| | add ``.PY`` to :envvar:`PATHEXT` | | -+---------------------------+--------------------------------------+--------------------------+ -| AppendPath | Append install and Scripts | 0 | -| | directories to :envvar:`PATH` and | | -| | add ``.PY`` to :envvar:`PATHEXT` | | -+---------------------------+--------------------------------------+--------------------------+ -| Shortcuts | Create shortcuts for the interpreter,| 1 | -| | documentation and IDLE if installed. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_doc | Install Python manual | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_debug | Install debug binaries | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_dev | Install developer headers and | 1 | -| | libraries. Omitting this may lead to | | -| | an unusable installation. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_exe | Install :file:`python.exe` and | 1 | -| | related files. Omitting this may | | -| | lead to an unusable installation. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_launcher | Install :ref:`launcher`. | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| InstallLauncherAllUsers | Installs the launcher for all | 1 | -| | users. Also requires | | -| | ``Include_launcher`` to be set to 1 | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_lib | Install standard library and | 1 | -| | extension modules. Omitting this may | | -| | lead to an unusable installation. | | -+---------------------------+--------------------------------------+--------------------------+ -| Include_pip | Install bundled pip and setuptools | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_symbols | Install debugging symbols (``*.pdb``)| 0 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_tcltk | Install Tcl/Tk support and IDLE | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_test | Install standard library test suite | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| Include_tools | Install utility scripts | 1 | -+---------------------------+--------------------------------------+--------------------------+ -| LauncherOnly | Only installs the launcher. This | 0 | -| | will override most other options. | | -+---------------------------+--------------------------------------+--------------------------+ -| SimpleInstall | Disable most install UI | 0 | -+---------------------------+--------------------------------------+--------------------------+ -| SimpleInstallDescription | A custom message to display when the | (empty) | -| | simplified install UI is used. | | -+---------------------------+--------------------------------------+--------------------------+ - -For example, to silently install a default, system-wide Python installation, -you could use the following command (from an elevated command prompt):: - - python-3.9.0.exe /quiet InstallAllUsers=1 PrependPath=1 Include_test=0 - -To allow users to easily install a personal copy of Python without the test -suite, you could provide a shortcut with the following command. This will -display a simplified initial page and disallow customization:: - - python-3.9.0.exe InstallAllUsers=0 Include_launcher=0 Include_test=0 - SimpleInstall=1 SimpleInstallDescription="Just for me, no test suite." - -(Note that omitting the launcher also omits file associations, and is only -recommended for per-user installs when there is also a system-wide installation -that included the launcher.) - -The options listed above can also be provided in a file named ``unattend.xml`` -alongside the executable. This file specifies a list of options and values. -When a value is provided as an attribute, it will be converted to a number if -possible. Values provided as element text are always left as strings. This -example file sets the same options as the previous example: - -.. code-block:: xml - - - - - -.. _install-layout-option: - -Installing without downloading ------------------------------- - -As some features of Python are not included in the initial installer download, -selecting those features may require an internet connection. To avoid this -need, all possible components may be downloaded on-demand to create a complete -*layout* that will no longer require an internet connection regardless of the -selected features. Note that this download may be bigger than required, but -where a large number of installations are going to be performed it is very -useful to have a locally cached copy. - -Execute the following command from Command Prompt to download all possible -required files. Remember to substitute ``python-3.9.0.exe`` for the actual -name of your installer, and to create layouts in their own directories to -avoid collisions between files with the same name. - -:: - - python-3.9.0.exe /layout [optional target directory] - -You may also specify the ``/quiet`` option to hide the progress display. - -Modifying an install --------------------- - -Once Python has been installed, you can add or remove features through the -Programs and Features tool that is part of Windows. Select the Python entry and -choose "Uninstall/Change" to open the installer in maintenance mode. - -"Modify" allows you to add or remove features by modifying the checkboxes - -unchanged checkboxes will not install or remove anything. Some options cannot be -changed in this mode, such as the install directory; to modify these, you will -need to remove and then reinstall Python completely. - -"Repair" will verify all the files that should be installed using the current -settings and replace any that have been removed or modified. - -"Uninstall" will remove Python entirely, with the exception of the -:ref:`launcher`, which has its own entry in Programs and Features. - - -Installing free-threaded binaries ---------------------------------- - -.. versionadded:: 3.13 - -To install pre-built binaries with free-threading enabled (see :pep:`703`), you -should select "Customize installation". The second page of options includes the -"Download free-threaded binaries" checkbox. - -.. image:: win_install_freethreaded.png - -Selecting this option will download and install additional binaries to the same -location as the main Python install. The main executable is called -``python3.13t.exe``, and other binaries either receive a ``t`` suffix or a full -ABI suffix. Python source files and bundled third-party dependencies are shared -with the main install. - -The free-threaded version is registered as a regular Python install with the -tag ``3.13t`` (with a ``-32`` or ``-arm64`` suffix as normal for those -platforms). This allows tools to discover it, and for the :ref:`launcher` to -support ``py.exe -3.13t``. Note that the launcher will interpret ``py.exe -3`` -(or a ``python3`` shebang) as "the latest 3.x install", which will prefer the -free-threaded binaries over the regular ones, while ``py.exe -3.13`` will not. -If you use the short style of option, you may prefer to not install the -free-threaded binaries at this time. - -To specify the install option at the command line, use -``Include_freethreaded=1``. See :ref:`install-layout-option` for instructions on -pre-emptively downloading the additional binaries for offline install. The -options to include debug symbols and binaries also apply to the free-threaded -builds. - -Free-threaded binaries are also available :ref:`on nuget.org `. - - -Python launcher for Windows (deprecated) -======================================== - -.. deprecated:: 3.14 - - The launcher and this documentation have been superseded by the Python - Install Manager described above. This is preserved temporarily for historical - interest. - -.. versionadded:: 3.3 - -The Python launcher for Windows is a utility which aids in locating and -executing of different Python versions. It allows scripts (or the -command-line) to indicate a preference for a specific Python version, and -will locate and execute that version. - -Unlike the :envvar:`PATH` variable, the launcher will correctly select the most -appropriate version of Python. It will prefer per-user installations over -system-wide ones, and orders by language version rather than using the most -recently installed version. - -The launcher was originally specified in :pep:`397`. - -Getting started ---------------- - -From the command-line -^^^^^^^^^^^^^^^^^^^^^ - -.. versionchanged:: 3.6 - -System-wide installations of Python 3.3 and later will put the launcher on your -:envvar:`PATH`. The launcher is compatible with all available versions of -Python, so it does not matter which version is installed. To check that the -launcher is available, execute the following command in Command Prompt:: - - py - -You should find that the latest version of Python you have installed is -started - it can be exited as normal, and any additional command-line -arguments specified will be sent directly to Python. - -If you have multiple versions of Python installed (e.g., 3.7 and |version|) you -will have noticed that Python |version| was started - to launch Python 3.7, try -the command:: - - py -3.7 - -If you want the latest version of Python 2 you have installed, try the -command:: - - py -2 - -If you see the following error, you do not have the launcher installed:: - - 'py' is not recognized as an internal or external command, - operable program or batch file. - -The command:: - - py --list - -displays the currently installed version(s) of Python. - -The ``-x.y`` argument is the short form of the ``-V:Company/Tag`` argument, -which allows selecting a specific Python runtime, including those that may have -come from somewhere other than python.org. Any runtime registered by following -:pep:`514` will be discoverable. The ``--list`` command lists all available -runtimes using the ``-V:`` format. - -When using the ``-V:`` argument, specifying the Company will limit selection to -runtimes from that provider, while specifying only the Tag will select from all -providers. Note that omitting the slash implies a tag:: - - # Select any '3.*' tagged runtime - py -V:3 - - # Select any 'PythonCore' released runtime - py -V:PythonCore/ - - # Select PythonCore's latest Python 3 runtime - py -V:PythonCore/3 - -The short form of the argument (``-3``) only ever selects from core Python -releases, and not other distributions. However, the longer form (``-V:3``) will -select from any. - -The Company is matched on the full string, case-insensitive. The Tag is matched -on either the full string, or a prefix, provided the next character is a dot or a -hyphen. This allows ``-V:3.1`` to match ``3.1-32``, but not ``3.10``. Tags are -sorted using numerical ordering (``3.10`` is newer than ``3.1``), but are -compared using text (``-V:3.01`` does not match ``3.1``). - - -Virtual environments -^^^^^^^^^^^^^^^^^^^^ - -.. versionadded:: 3.5 - -If the launcher is run with no explicit Python version specification, and a -virtual environment (created with the standard library :mod:`venv` module or -the external ``virtualenv`` tool) active, the launcher will run the virtual -environment's interpreter rather than the global one. To run the global -interpreter, either deactivate the virtual environment, or explicitly specify -the global Python version. - -From a script -^^^^^^^^^^^^^ - -Let's create a test Python script - create a file called ``hello.py`` with the -following contents - -.. code-block:: python - - #! python - import sys - sys.stdout.write("hello from Python %s\n" % (sys.version,)) - -From the directory in which hello.py lives, execute the command:: - - py hello.py - -You should notice the version number of your latest Python 2.x installation -is printed. Now try changing the first line to be: - -.. code-block:: python - - #! python3 - -Re-executing the command should now print the latest Python 3.x information. -As with the above command-line examples, you can specify a more explicit -version qualifier. Assuming you have Python 3.7 installed, try changing -the first line to ``#! python3.7`` and you should find the 3.7 -version information printed. - -Note that unlike interactive use, a bare "python" will use the latest -version of Python 2.x that you have installed. This is for backward -compatibility and for compatibility with Unix, where the command ``python`` -typically refers to Python 2. - -From file associations -^^^^^^^^^^^^^^^^^^^^^^ - -The launcher should have been associated with Python files (i.e. ``.py``, -``.pyw``, ``.pyc`` files) when it was installed. This means that -when you double-click on one of these files from Windows explorer the launcher -will be used, and therefore you can use the same facilities described above to -have the script specify the version which should be used. - -The key benefit of this is that a single launcher can support multiple Python -versions at the same time depending on the contents of the first line. - -Shebang lines -------------- - -If the first line of a script file starts with ``#!``, it is known as a -"shebang" line. Linux and other Unix like operating systems have native -support for such lines and they are commonly used on such systems to indicate -how a script should be executed. This launcher allows the same facilities to -be used with Python scripts on Windows and the examples above demonstrate their -use. - -To allow shebang lines in Python scripts to be portable between Unix and -Windows, this launcher supports a number of 'virtual' commands to specify -which interpreter to use. The supported virtual commands are: - -* ``/usr/bin/env`` -* ``/usr/bin/python`` -* ``/usr/local/bin/python`` -* ``python`` - -For example, if the first line of your script starts with - -.. code-block:: sh - - #! /usr/bin/python - -The default Python or an active virtual environment will be located and used. -As many Python scripts written to work on Unix will already have this line, -you should find these scripts can be used by the launcher without modification. -If you are writing a new script on Windows which you hope will be useful on -Unix, you should use one of the shebang lines starting with ``/usr``. - -Any of the above virtual commands can be suffixed with an explicit version -(either just the major version, or the major and minor version). -Furthermore the 32-bit version can be requested by adding "-32" after the -minor version. I.e. ``/usr/bin/python3.7-32`` will request usage of the -32-bit Python 3.7. If a virtual environment is active, the version will be -ignored and the environment will be used. - -.. versionadded:: 3.7 - - Beginning with python launcher 3.7 it is possible to request 64-bit version - by the "-64" suffix. Furthermore it is possible to specify a major and - architecture without minor (i.e. ``/usr/bin/python3-64``). - -.. versionchanged:: 3.11 - - The "-64" suffix is deprecated, and now implies "any architecture that is - not provably i386/32-bit". To request a specific environment, use the new - :samp:`-V:{TAG}` argument with the complete tag. - -.. versionchanged:: 3.13 - - Virtual commands referencing ``python`` now prefer an active virtual - environment rather than searching :envvar:`PATH`. This handles cases where - the shebang specifies ``/usr/bin/env python3`` but :file:`python3.exe` is - not present in the active environment. - -The ``/usr/bin/env`` form of shebang line has one further special property. -Before looking for installed Python interpreters, this form will search the -executable :envvar:`PATH` for a Python executable matching the name provided -as the first argument. This corresponds to the behaviour of the Unix ``env`` -program, which performs a :envvar:`PATH` search. -If an executable matching the first argument after the ``env`` command cannot -be found, but the argument starts with ``python``, it will be handled as -described for the other virtual commands. -The environment variable :envvar:`!PYLAUNCHER_NO_SEARCH_PATH` may be set -(to any value) to skip this search of :envvar:`PATH`. - -Shebang lines that do not match any of these patterns are looked up in the -``[commands]`` section of the launcher's :ref:`.INI file `. -This may be used to handle certain commands in a way that makes sense for your -system. The name of the command must be a single argument (no spaces in the -shebang executable), and the value substituted is the full path to the -executable (additional arguments specified in the .INI will be quoted as part -of the filename). - -.. code-block:: ini - - [commands] - /bin/xpython=C:\Program Files\XPython\python.exe - -Any commands not found in the .INI file are treated as **Windows** executable -paths that are absolute or relative to the directory containing the script file. -This is a convenience for Windows-only scripts, such as those generated by an -installer, since the behavior is not compatible with Unix-style shells. -These paths may be quoted, and may include multiple arguments, after which the -path to the script and any additional arguments will be appended. - - -Arguments in shebang lines --------------------------- - -The shebang lines can also specify additional options to be passed to the -Python interpreter. For example, if you have a shebang line: - -.. code-block:: sh - - #! /usr/bin/python -v - -Then Python will be started with the ``-v`` option - -Customization -------------- - -.. _launcher-ini: - -Customization via INI files -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Two .ini files will be searched by the launcher - ``py.ini`` in the current -user's application data directory (``%LOCALAPPDATA%`` or ``$env:LocalAppData``) -and ``py.ini`` in the same directory as the launcher. The same .ini files are -used for both the 'console' version of the launcher (i.e. py.exe) and for the -'windows' version (i.e. pyw.exe). - -Customization specified in the "application directory" will have precedence over -the one next to the executable, so a user, who may not have write access to the -.ini file next to the launcher, can override commands in that global .ini file. - -Customizing default Python versions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In some cases, a version qualifier can be included in a command to dictate -which version of Python will be used by the command. A version qualifier -starts with a major version number and can optionally be followed by a period -('.') and a minor version specifier. Furthermore it is possible to specify -if a 32 or 64 bit implementation shall be requested by adding "-32" or "-64". - -For example, a shebang line of ``#!python`` has no version qualifier, while -``#!python3`` has a version qualifier which specifies only a major version. - -If no version qualifiers are found in a command, the environment -variable :envvar:`!PY_PYTHON` can be set to specify the default version -qualifier. If it is not set, the default is "3". The variable can -specify any value that may be passed on the command line, such as "3", -"3.7", "3.7-32" or "3.7-64". (Note that the "-64" option is only -available with the launcher included with Python 3.7 or newer.) - -If no minor version qualifiers are found, the environment variable -``PY_PYTHON{major}`` (where ``{major}`` is the current major version qualifier -as determined above) can be set to specify the full version. If no such option -is found, the launcher will enumerate the installed Python versions and use -the latest minor release found for the major version, which is likely, -although not guaranteed, to be the most recently installed version in that -family. - -On 64-bit Windows with both 32-bit and 64-bit implementations of the same -(major.minor) Python version installed, the 64-bit version will always be -preferred. This will be true for both 32-bit and 64-bit implementations of the -launcher - a 32-bit launcher will prefer to execute a 64-bit Python installation -of the specified version if available. This is so the behavior of the launcher -can be predicted knowing only what versions are installed on the PC and -without regard to the order in which they were installed (i.e., without knowing -whether a 32 or 64-bit version of Python and corresponding launcher was -installed last). As noted above, an optional "-32" or "-64" suffix can be -used on a version specifier to change this behaviour. - -Examples: - -* If no relevant options are set, the commands ``python`` and - ``python2`` will use the latest Python 2.x version installed and - the command ``python3`` will use the latest Python 3.x installed. - -* The command ``python3.7`` will not consult any - options at all as the versions are fully specified. - -* If ``PY_PYTHON=3``, the commands ``python`` and ``python3`` will both use - the latest installed Python 3 version. - -* If ``PY_PYTHON=3.7-32``, the command ``python`` will use the 32-bit - implementation of 3.7 whereas the command ``python3`` will use the latest - installed Python (PY_PYTHON was not considered at all as a major - version was specified.) - -* If ``PY_PYTHON=3`` and ``PY_PYTHON3=3.7``, the commands - ``python`` and ``python3`` will both use specifically 3.7 - -In addition to environment variables, the same settings can be configured -in the .INI file used by the launcher. The section in the INI file is -called ``[defaults]`` and the key name will be the same as the -environment variables without the leading ``PY_`` prefix (and note that -the key names in the INI file are case insensitive.) The contents of -an environment variable will override things specified in the INI file. - -For example: - -* Setting ``PY_PYTHON=3.7`` is equivalent to the INI file containing: - -.. code-block:: ini - - [defaults] - python=3.7 - -* Setting ``PY_PYTHON=3`` and ``PY_PYTHON3=3.7`` is equivalent to the INI file - containing: - -.. code-block:: ini - - [defaults] - python=3 - python3=3.7 - -Diagnostics ------------ - -If an environment variable :envvar:`!PYLAUNCHER_DEBUG` is set (to any value), the -launcher will print diagnostic information to stderr (i.e. to the console). -While this information manages to be simultaneously verbose *and* terse, it -should allow you to see what versions of Python were located, why a -particular version was chosen and the exact command-line used to execute the -target Python. It is primarily intended for testing and debugging. - -Dry run -------- - -If an environment variable :envvar:`!PYLAUNCHER_DRYRUN` is set (to any value), -the launcher will output the command it would have run, but will not actually -launch Python. This may be useful for tools that want to use the launcher to -detect and then launch Python directly. Note that the command written to -standard output is always encoded using UTF-8, and may not render correctly in -the console. - -Install on demand ------------------ - -If an environment variable :envvar:`!PYLAUNCHER_ALLOW_INSTALL` is set (to any -value), and the requested Python version is not installed but is available on -the Microsoft Store, the launcher will attempt to install it. This may require -user interaction to complete, and you may need to run the command again. - -An additional :envvar:`!PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher -to always try to install Python, even if it is detected. This is mainly intended -for testing (and should be used with :envvar:`!PYLAUNCHER_DRYRUN`). - -Return codes ------------- - -The following exit codes may be returned by the Python launcher. Unfortunately, -there is no way to distinguish these from the exit code of Python itself. - -The names of codes are as used in the sources, and are only for reference. There -is no way to access or resolve them apart from reading this page. Entries are -listed in alphabetical order of names. - -+-------------------+-------+-----------------------------------------------+ -| Name | Value | Description | -+===================+=======+===============================================+ -| RC_BAD_VENV_CFG | 107 | A :file:`pyvenv.cfg` was found but is corrupt.| -+-------------------+-------+-----------------------------------------------+ -| RC_CREATE_PROCESS | 101 | Failed to launch Python. | -+-------------------+-------+-----------------------------------------------+ -| RC_INSTALLING | 111 | An install was started, but the command will | -| | | need to be re-run after it completes. | -+-------------------+-------+-----------------------------------------------+ -| RC_INTERNAL_ERROR | 109 | Unexpected error. Please report a bug. | -+-------------------+-------+-----------------------------------------------+ -| RC_NO_COMMANDLINE | 108 | Unable to obtain command line from the | -| | | operating system. | -+-------------------+-------+-----------------------------------------------+ -| RC_NO_PYTHON | 103 | Unable to locate the requested version. | -+-------------------+-------+-----------------------------------------------+ -| RC_NO_VENV_CFG | 106 | A :file:`pyvenv.cfg` was required but not | -| | | found. | -+-------------------+-------+-----------------------------------------------+ - diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 7628cfefe0ec96e..87c7b6b17477b50 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -35,7 +35,7 @@ implementation and design rationale. PEP 218: Built-In Set Objects ============================= -Python 2.3 introduced the :mod:`sets` module. C implementations of set data +Python 2.3 introduced the :mod:`!sets` module. C implementations of set data types have now been added to the Python core as two new built-in types, ``set(iterable)`` and ``frozenset(iterable)``. They provide high speed operations for membership testing, for eliminating duplicates from sequences, @@ -72,8 +72,8 @@ The :func:`frozenset` type is an immutable version of :func:`set`. Since it is immutable and hashable, it may be used as a dictionary key or as a member of another set. -The :mod:`sets` module remains in the standard library, and may be useful if you -wish to subclass the :class:`Set` or :class:`ImmutableSet` classes. There are +The :mod:`!sets` module remains in the standard library, and may be useful if you +wish to subclass the :class:`!Set` or :class:`!ImmutableSet` classes. There are currently no plans to deprecate the module. @@ -189,7 +189,7 @@ aren't aware of the Python language. The format string's syntax is complicated to explain to such users, and if they make a mistake, it's difficult to provide helpful feedback to them. -PEP 292 adds a :class:`Template` class to the :mod:`string` module that uses +PEP 292 adds a :class:`~string.Template` class to the :mod:`string` module that uses ``$`` to indicate a substitution:: >>> import string @@ -197,9 +197,9 @@ PEP 292 adds a :class:`Template` class to the :mod:`string` module that uses >>> t.substitute({'page':2, 'title': 'The Best of Times'}) '2: The Best of Times' -If a key is missing from the dictionary, the :meth:`substitute` method will -raise a :exc:`KeyError`. There's also a :meth:`safe_substitute` method that -ignores missing keys:: +If a key is missing from the dictionary, the :meth:`~string.Template.substitute` +method will raise a :exc:`KeyError`. There's also a +:meth:`~string.Template.safe_substitute` method that ignores missing keys:: >>> t = string.Template('$page: $title') >>> t.safe_substitute({'page':3}) @@ -430,14 +430,14 @@ The constructor has a number of handy options: * *universal_newlines* opens the child's input and output using Python's :term:`universal newlines` feature. -Once you've created the :class:`Popen` instance, you can call its :meth:`wait` -method to pause until the subprocess has exited, :meth:`poll` to check if it's +Once you've created the :class:`~subprocess.Popen` instance, you can call its :meth:`~subprocess.Popen.wait` +method to pause until the subprocess has exited, :meth:`~subprocess.Popen.poll` to check if it's exited without pausing, or ``communicate(data)`` to send the string *data* to the subprocess's standard input. ``communicate(data)`` then reads any data that the subprocess has sent to its standard output or standard error, returning a tuple ``(stdout_data, stderr_data)``. -:func:`call` is a shortcut that passes its arguments along to the :class:`Popen` +:func:`~subprocess.call` is a shortcut that passes its arguments along to the :class:`~subprocess.Popen` constructor, waits for the command to complete, and returns the status code of the subprocess. It can serve as a safer analog to :func:`os.system`:: @@ -476,7 +476,7 @@ Python has always supported floating-point (FP) numbers, based on the underlying C :c:expr:`double` type, as a data type. However, while most programming languages provide a floating-point type, many people (even programmers) are unaware that floating-point numbers don't represent certain decimal fractions -accurately. The new :class:`Decimal` type can represent these fractions +accurately. The new :class:`~decimal.Decimal` type can represent these fractions accurately, up to a user-specified precision limit. @@ -531,20 +531,20 @@ decimal places, the error is never apparent. However, for applications where it does matter, it's a lot of work to implement your own custom arithmetic routines. -Hence, the :class:`Decimal` type was created. +Hence, the :class:`~decimal.Decimal` type was created. -The :class:`Decimal` type -------------------------- +The :class:`~decimal.Decimal` type +---------------------------------- A new module, :mod:`decimal`, was added to Python's standard library. It -contains two classes, :class:`Decimal` and :class:`Context`. :class:`Decimal` -instances represent numbers, and :class:`Context` instances are used to wrap up +contains two classes, :class:`~decimal.Decimal` and :class:`~decimal.Context`. :class:`~decimal.Decimal` +instances represent numbers, and :class:`~decimal.Context` instances are used to wrap up various settings such as the precision and default rounding mode. -:class:`Decimal` instances are immutable, like regular Python integers and FP +:class:`~decimal.Decimal` instances are immutable, like regular Python integers and FP numbers; once it's been created, you can't change the value an instance -represents. :class:`Decimal` instances can be created from integers or +represents. :class:`~decimal.Decimal` instances can be created from integers or strings:: >>> import decimal @@ -567,7 +567,7 @@ number representing 1.1 turn into the decimal number for exactly 1.1, or for 1.1 plus whatever inaccuracies are introduced? The decision was to dodge the issue and leave such a conversion out of the API. Instead, you should convert the floating-point number into a string using the desired precision and pass the -string to the :class:`Decimal` constructor:: +string to the :class:`~decimal.Decimal` constructor:: >>> f = 1.1 >>> decimal.Decimal(str(f)) @@ -575,7 +575,7 @@ string to the :class:`Decimal` constructor:: >>> decimal.Decimal('%.12f' % f) Decimal("1.100000000000") -Once you have :class:`Decimal` instances, you can perform the usual mathematical +Once you have :class:`~decimal.Decimal` instances, you can perform the usual mathematical operations on them. One limitation: exponentiation requires an integer exponent:: @@ -596,7 +596,7 @@ exponent:: ... decimal.InvalidOperation: x ** (non-integer) -You can combine :class:`Decimal` instances with integers, but not with +You can combine :class:`~decimal.Decimal` instances with integers, but not with floating-point numbers:: >>> a + 4 @@ -607,11 +607,11 @@ floating-point numbers:: TypeError: You can interact Decimal only with int, long or Decimal data types. >>> -:class:`Decimal` numbers can be used with the :mod:`math` and :mod:`cmath` +:class:`~decimal.Decimal` numbers can be used with the :mod:`math` and :mod:`cmath` modules, but note that they'll be immediately converted to floating-point numbers before the operation is performed, resulting in a possible loss of precision and accuracy. You'll also get back a regular floating-point number -and not a :class:`Decimal`. :: +and not a :class:`~decimal.Decimal`. :: >>> import math, cmath >>> d = decimal.Decimal('123456789012.345') @@ -620,32 +620,33 @@ and not a :class:`Decimal`. :: >>> cmath.sqrt(-d) 351364.18288201344j -:class:`Decimal` instances have a :meth:`sqrt` method that returns a -:class:`Decimal`, but if you need other things such as trigonometric functions +:class:`~decimal.Decimal` instances have a :meth:`~decimal.Decimal.sqrt` method that returns a +:class:`~decimal.Decimal`, but if you need other things such as trigonometric functions you'll have to implement them. :: >>> d.sqrt() Decimal("351364.1828820134592177245001") -The :class:`Context` type -------------------------- +The :class:`~decimal.Context` type +---------------------------------- -Instances of the :class:`Context` class encapsulate several settings for +Instances of the :class:`~decimal.Context` class encapsulate several settings for decimal operations: -* :attr:`prec` is the precision, the number of decimal places. +* :attr:`~decimal.Context.prec` is the precision, the number of decimal places. -* :attr:`rounding` specifies the rounding mode. The :mod:`decimal` module has - constants for the various possibilities: :const:`ROUND_DOWN`, - :const:`ROUND_CEILING`, :const:`ROUND_HALF_EVEN`, and various others. +* :attr:`~decimal.Context.rounding` specifies the rounding mode. The + :mod:`decimal` module has constants for the various possibilities: + :const:`~decimal.ROUND_DOWN`, :const:`~decimal.ROUND_CEILING`, + :const:`~decimal.ROUND_HALF_EVEN`, and various others. -* :attr:`traps` is a dictionary specifying what happens on encountering certain +* :attr:`~decimal.Context.traps` is a dictionary specifying what happens on encountering certain error conditions: either an exception is raised or a value is returned. Some examples of error conditions are division by zero, loss of precision, and overflow. -There's a thread-local default context available by calling :func:`getcontext`; +There's a thread-local default context available by calling :func:`~decimal.getcontext`; you can change the properties of this context to alter the default precision, rounding, or trap handling. The following example shows the effect of changing the precision of the default context:: @@ -671,8 +672,8 @@ raised:: Decimal("Infinity") >>> -The :class:`Context` instance also has various methods for formatting numbers -such as :meth:`to_eng_string` and :meth:`to_sci_string`. +The :class:`~decimal.Context` instance also has various methods for formatting numbers +such as :meth:`~decimal.Context.to_eng_string` and :meth:`~decimal.Context.to_sci_string`. For more information, see the documentation for the :mod:`decimal` module, which includes a quick-start tutorial and a reference. @@ -740,7 +741,7 @@ display conventions that are localized to a particular country or language. However, the module was careful to not change the numeric locale because various functions in Python's implementation required that the numeric locale remain set to the ``'C'`` locale. Often this was because the code was using the C -library's :c:func:`atof` function. +library's :c:func:`!atof` function. Not setting the numeric locale caused trouble for extensions that used third-party C libraries, however, because they wouldn't have the correct locale set. @@ -793,11 +794,11 @@ Here are all of the changes that Python 2.4 makes to the core Python language. :class:`dict` constructor. This includes any mapping, any iterable of key/value pairs, and keyword arguments. (Contributed by Raymond Hettinger.) -* The string methods :meth:`ljust`, :meth:`rjust`, and :meth:`center` now take +* The string methods :meth:`~str.ljust`, :meth:`~str.rjust`, and :meth:`~str.center` now take an optional argument for specifying a fill character other than a space. (Contributed by Raymond Hettinger.) -* Strings also gained an :meth:`rsplit` method that works like the :meth:`split` +* Strings also gained an :meth:`~str.rsplit` method that works like the :meth:`~str.split` method but splits from the end of the string. (Contributed by Sean Reifschneider.) :: @@ -807,13 +808,13 @@ Here are all of the changes that Python 2.4 makes to the core Python language. ['www.python', 'org'] * Three keyword parameters, *cmp*, *key*, and *reverse*, were added to the - :meth:`sort` method of lists. These parameters make some common usages of - :meth:`sort` simpler. All of these parameters are optional. + :meth:`~list.sort` method of lists. These parameters make some common usages of + :meth:`~list.sort` simpler. All of these parameters are optional. For the *cmp* parameter, the value should be a comparison function that takes two parameters and returns -1, 0, or +1 depending on how the parameters compare. This function will then be used to sort the list. Previously this was the only - parameter that could be provided to :meth:`sort`. + parameter that could be provided to :meth:`~list.sort`. *key* should be a single-parameter function that takes a list element and returns a comparison key for the element. The list is then sorted using the @@ -834,9 +835,9 @@ Here are all of the changes that Python 2.4 makes to the core Python language. The last example, which uses the *cmp* parameter, is the old way to perform a case-insensitive sort. It works but is slower than using a *key* parameter. - Using *key* calls :meth:`lower` method once for each element in the list while + Using *key* calls :meth:`~str.lower` method once for each element in the list while using *cmp* will call it twice for each comparison, so using *key* saves on - invocations of the :meth:`lower` method. + invocations of the :meth:`~str.lower` method. For simple key functions and comparison functions, it is often possible to avoid a :keyword:`lambda` expression by using an unbound method instead. For example, @@ -856,7 +857,7 @@ Here are all of the changes that Python 2.4 makes to the core Python language. age, resulting in a list sorted by age where people with the same age are in name-sorted order. - (All changes to :meth:`sort` contributed by Raymond Hettinger.) + (All changes to :meth:`~list.sort` contributed by Raymond Hettinger.) * There is a new built-in function ``sorted(iterable)`` that works like the in-place :meth:`list.sort` method but can be used in expressions. The @@ -891,8 +892,8 @@ Here are all of the changes that Python 2.4 makes to the core Python language. (Contributed by Raymond Hettinger.) -* Integer operations will no longer trigger an :exc:`OverflowWarning`. The - :exc:`OverflowWarning` warning will disappear in Python 2.5. +* Integer operations will no longer trigger an :exc:`!OverflowWarning`. The + :exc:`!OverflowWarning` warning will disappear in Python 2.5. * The interpreter gained a new switch, :option:`-m`, that takes a name, searches for the corresponding module on ``sys.path``, and runs the module as a script. @@ -904,7 +905,7 @@ Here are all of the changes that Python 2.4 makes to the core Python language. for the *locals* parameter. Previously this had to be a regular Python dictionary. (Contributed by Raymond Hettinger.) -* The :func:`zip` built-in function and :func:`itertools.izip` now return an +* The :func:`zip` built-in function and :func:`!itertools.izip` now return an empty list if called with no arguments. Previously they raised a :exc:`TypeError` exception. This makes them more suitable for use with variable length argument lists:: @@ -935,8 +936,8 @@ Optimizations * The inner loops for list and tuple slicing were optimized and now run about one-third faster. The inner loops for dictionaries were also optimized, - resulting in performance boosts for :meth:`keys`, :meth:`values`, :meth:`items`, - :meth:`iterkeys`, :meth:`itervalues`, and :meth:`iteritems`. (Contributed by + resulting in performance boosts for :meth:`~dict.keys`, :meth:`~dict.values`, :meth:`~dict.items`, + :meth:`!iterkeys`, :meth:`!itervalues`, and :meth:`!iteritems`. (Contributed by Raymond Hettinger.) * The machinery for growing and shrinking lists was optimized for speed and for @@ -948,11 +949,11 @@ Optimizations * :func:`list`, :func:`tuple`, :func:`map`, :func:`filter`, and :func:`zip` now run several times faster with non-sequence arguments that supply a - :meth:`__len__` method. (Contributed by Raymond Hettinger.) + :meth:`~object.__len__` method. (Contributed by Raymond Hettinger.) -* The methods :meth:`list.__getitem__`, :meth:`dict.__getitem__`, and - :meth:`dict.__contains__` are now implemented as :class:`method_descriptor` - objects rather than :class:`wrapper_descriptor` objects. This form of access +* The methods :meth:`!list.__getitem__`, :meth:`!dict.__getitem__`, and + :meth:`!dict.__contains__` are now implemented as :class:`!method_descriptor` + objects rather than :class:`!wrapper_descriptor` objects. This form of access doubles their performance and makes them more suitable for use as arguments to functionals: ``map(mydict.__getitem__, keylist)``. (Contributed by Raymond Hettinger.) @@ -968,7 +969,7 @@ Optimizations * String concatenations in statements of the form ``s = s + "abc"`` and ``s += "abc"`` are now performed more efficiently in certain circumstances. This optimization won't be present in other Python implementations such as Jython, so - you shouldn't rely on it; using the :meth:`join` method of strings is still + you shouldn't rely on it; using the :meth:`~str.join` method of strings is still recommended when you want to efficiently glue a large number of strings together. (Contributed by Armin Rigo.) @@ -1023,13 +1024,13 @@ complete list of changes, or look through the CVS logs for all the details. PCTP-154, and TIS-620. * The UTF-8 and UTF-16 codecs now cope better with receiving partial input. - Previously the :class:`StreamReader` class would try to read more data, making - it impossible to resume decoding from the stream. The :meth:`read` method will + Previously the :class:`~codecs.StreamReader` class would try to read more data, making + it impossible to resume decoding from the stream. The :meth:`~codecs.StreamReader.read` method will now return as much data as it can and future calls will resume decoding where previous ones left off. (Implemented by Walter Dörwald.) * There is a new :mod:`collections` module for various specialized collection - datatypes. Currently it contains just one type, :class:`deque`, a double-ended + datatypes. Currently it contains just one type, :class:`~collections.deque`, a double-ended queue that supports efficiently adding and removing elements from either end:: @@ -1048,7 +1049,7 @@ complete list of changes, or look through the CVS logs for all the details. >>> 'h' in d # search the deque True - Several modules, such as the :mod:`Queue` and :mod:`threading` modules, now take + Several modules, such as the :mod:`queue` and :mod:`threading` modules, now take advantage of :class:`collections.deque` for improved performance. (Contributed by Raymond Hettinger.) @@ -1058,49 +1059,49 @@ complete list of changes, or look through the CVS logs for all the details. isn't a string. (Contributed by John Belmonte and David Goodger.) * The :mod:`curses` module now supports the ncurses extension - :func:`use_default_colors`. On platforms where the terminal supports + :func:`~curses.use_default_colors`. On platforms where the terminal supports transparency, this makes it possible to use a transparent background. (Contributed by Jörg Lehmann.) -* The :mod:`difflib` module now includes an :class:`HtmlDiff` class that creates +* The :mod:`difflib` module now includes an :class:`~difflib.HtmlDiff` class that creates an HTML table showing a side by side comparison of two versions of a text. (Contributed by Dan Gass.) * The :mod:`email` package was updated to version 3.0, which dropped various deprecated APIs and removes support for Python versions earlier than 2.3. The 3.0 version of the package uses a new incremental parser for MIME messages, - available in the :mod:`email.FeedParser` module. The new parser doesn't require + available in the :mod:`!email.FeedParser` module. The new parser doesn't require reading the entire message into memory, and doesn't raise exceptions if a - message is malformed; instead it records any problems in the :attr:`defect` + message is malformed; instead it records any problems in the :attr:`!defect` attribute of the message. (Developed by Anthony Baxter, Barry Warsaw, Thomas Wouters, and others.) * The :mod:`heapq` module has been converted to C. The resulting tenfold improvement in speed makes the module suitable for handling high volumes of - data. In addition, the module has two new functions :func:`nlargest` and - :func:`nsmallest` that use heaps to find the N largest or smallest values in a + data. In addition, the module has two new functions :func:`~heapq.nlargest` and + :func:`~heapq.nsmallest` that use heaps to find the N largest or smallest values in a dataset without the expense of a full sort. (Contributed by Raymond Hettinger.) * The :mod:`httplib ` module now contains constants for HTTP status codes defined in various HTTP-related RFC documents. Constants have names such as - :const:`OK`, :const:`CREATED`, :const:`CONTINUE`, and - :const:`MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by + :const:`!OK`, :const:`!CREATED`, :const:`!CONTINUE`, and + :const:`!MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by Andrew Eland.) * The :mod:`imaplib` module now supports IMAP's THREAD command (contributed by - Yves Dionne) and new :meth:`deleteacl` and :meth:`myrights` methods (contributed - by Arnaud Mazin). + Yves Dionne) and new :meth:`~imaplib.IMAP4.deleteacl` and + :meth:`~imaplib.IMAP4.myrights` methods (contributed by Arnaud Mazin). * The :mod:`itertools` module gained a ``groupby(iterable[, *func*])`` function. *iterable* is something that can be iterated over to return a stream of elements, and the optional *func* parameter is a function that takes an element and returns a key value; if omitted, the key is simply the element - itself. :func:`groupby` then groups the elements into subsequences which have + itself. :func:`~itertools.groupby` then groups the elements into subsequences which have matching values of the key, and returns a series of 2-tuples containing the key value and an iterator over the subsequence. Here's an example to make this clearer. The *key* function simply returns - whether a number is even or odd, so the result of :func:`groupby` is to return + whether a number is even or odd, so the result of :func:`~itertools.groupby` is to return consecutive runs of odd or even numbers. :: >>> import itertools @@ -1115,8 +1116,8 @@ complete list of changes, or look through the CVS logs for all the details. 0 [12, 14] >>> - :func:`groupby` is typically used with sorted input. The logic for - :func:`groupby` is similar to the Unix ``uniq`` filter which makes it handy for + :func:`~itertools.groupby` is typically used with sorted input. The logic for + :func:`~itertools.groupby` is similar to the Unix ``uniq`` filter which makes it handy for eliminating, counting, or identifying duplicate elements:: >>> word = 'abracadabra' @@ -1153,22 +1154,22 @@ complete list of changes, or look through the CVS logs for all the details. >>> list(i2) # Run the second iterator to exhaustion [1, 2, 3] - Note that :func:`tee` has to keep copies of the values returned by the + Note that :func:`~itertools.tee` has to keep copies of the values returned by the iterator; in the worst case, it may need to keep all of them. This should therefore be used carefully if the leading iterator can run far ahead of the trailing iterator in a long stream of inputs. If the separation is large, then you might as well use :func:`list` instead. When the iterators track closely - with one another, :func:`tee` is ideal. Possible applications include + with one another, :func:`~itertools.tee` is ideal. Possible applications include bookmarking, windowing, or lookahead iterators. (Contributed by Raymond Hettinger.) * A number of functions were added to the :mod:`locale` module, such as - :func:`bind_textdomain_codeset` to specify a particular encoding and a family of + :func:`~locale.bind_textdomain_codeset` to specify a particular encoding and a family of :func:`!l\*gettext` functions that return messages in the chosen encoding. (Contributed by Gustavo Niemeyer.) * Some keyword arguments were added to the :mod:`logging` package's - :func:`basicConfig` function to simplify log configuration. The default + :func:`~logging.basicConfig` function to simplify log configuration. The default behavior is to log messages to standard error, but various keyword arguments can be specified to log to a particular file, change the logging format, or set the logging level. For example:: @@ -1179,10 +1180,10 @@ complete list of changes, or look through the CVS logs for all the details. format='%(levelname):%(process):%(thread):%(message)') Other additions to the :mod:`logging` package include a ``log(level, msg)`` - convenience method, as well as a :class:`TimedRotatingFileHandler` class that + convenience method, as well as a :class:`~logging.handlers.TimedRotatingFileHandler` class that rotates its log files at a timed interval. The module already had - :class:`RotatingFileHandler`, which rotated logs once the file exceeded a - certain size. Both classes derive from a new :class:`BaseRotatingHandler` class + :class:`~logging.handlers.RotatingFileHandler`, which rotated logs once the file exceeded a + certain size. Both classes derive from a new :class:`~logging.handlers.BaseRotatingHandler` class that can be used to implement other rotating handlers. (Changes implemented by Vinay Sajip.) @@ -1192,8 +1193,8 @@ complete list of changes, or look through the CVS logs for all the details. effect is to make :file:`.pyc` files significantly smaller. (Contributed by Martin von Löwis.) -* The :mod:`!nntplib` module's :class:`NNTP` class gained :meth:`description` and - :meth:`descriptions` methods to retrieve newsgroup descriptions for a single +* The :mod:`!nntplib` module's :class:`!NNTP` class gained :meth:`!description` and + :meth:`!descriptions` methods to retrieve newsgroup descriptions for a single group or for a range of groups. (Contributed by Jürgen A. Erhard.) * Two new functions were added to the :mod:`operator` module, @@ -1235,7 +1236,7 @@ complete list of changes, or look through the CVS logs for all the details. *path* is a symlink that points to a destination that doesn't exist. (Contributed by Beni Cherniavsky.) -* A new :func:`getsid` function was added to the :mod:`posix` module that +* A new :func:`~os.getsid` function was added to the :mod:`posix` module that underlies the :mod:`os` module. (Contributed by J. Raynor.) * The :mod:`poplib` module now supports POP over SSL. (Contributed by Hector @@ -1245,8 +1246,8 @@ complete list of changes, or look through the CVS logs for all the details. by Nick Bastin.) * The :mod:`random` module has a new method called ``getrandbits(N)`` that - returns a long integer *N* bits in length. The existing :meth:`randrange` - method now uses :meth:`getrandbits` where appropriate, making generation of + returns a long integer *N* bits in length. The existing :meth:`~random.randrange` + method now uses :meth:`~random.getrandbits` where appropriate, making generation of arbitrarily large random numbers more efficient. (Contributed by Raymond Hettinger.) @@ -1269,24 +1270,24 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`signal` module now performs tighter error-checking on the parameters to the :func:`signal.signal` function. For example, you can't set a handler on - the :const:`SIGKILL` signal; previous versions of Python would quietly accept + the :const:`~signal.SIGKILL` signal; previous versions of Python would quietly accept this, but 2.4 will raise a :exc:`RuntimeError` exception. -* Two new functions were added to the :mod:`socket` module. :func:`socketpair` +* Two new functions were added to the :mod:`socket` module. :func:`~socket.socketpair` returns a pair of connected sockets and ``getservbyport(port)`` looks up the service name for a given port number. (Contributed by Dave Cole and Barry Warsaw.) -* The :func:`sys.exitfunc` function has been deprecated. Code should be using +* The :func:`!sys.exitfunc` function has been deprecated. Code should be using the existing :mod:`atexit` module, which correctly handles calling multiple exit - functions. Eventually :func:`sys.exitfunc` will become a purely internal + functions. Eventually :func:`!sys.exitfunc` will become a purely internal interface, accessed only by :mod:`atexit`. * The :mod:`tarfile` module now generates GNU-format tar files by default. (Contributed by Lars Gustäbel.) * The :mod:`threading` module now has an elegantly simple way to support - thread-local data. The module contains a :class:`local` class whose attribute + thread-local data. The module contains a :class:`~threading.local` class whose attribute values are local to different threads. :: import threading @@ -1295,8 +1296,8 @@ complete list of changes, or look through the CVS logs for all the details. data.number = 42 data.url = ('www.python.org', 80) - Other threads can assign and retrieve their own values for the :attr:`number` - and :attr:`url` attributes. You can subclass :class:`local` to initialize + Other threads can assign and retrieve their own values for the :attr:`!number` + and :attr:`!url` attributes. You can subclass :class:`~threading.local` to initialize attributes or to add methods. (Contributed by Jim Fulton.) * The :mod:`timeit` module now automatically disables periodic garbage @@ -1312,7 +1313,7 @@ complete list of changes, or look through the CVS logs for all the details. transmitting multiple XML-RPC calls in a single HTTP operation. (Contributed by Brian Quinlan.) -* The :mod:`mpz`, :mod:`rotor`, and :mod:`xreadlines` modules have been +* The :mod:`!mpz`, :mod:`!rotor`, and :mod:`!xreadlines` modules have been removed. .. ====================================================================== @@ -1336,7 +1337,7 @@ use the Mozilla or Lynx cookie files, and one that stores cookies in the same format as the Perl libwww library. :mod:`urllib2 ` has been changed to interact with :mod:`cookielib `: -:class:`HTTPCookieProcessor` manages a cookie jar that is used when accessing +:class:`~urllib.request.HTTPCookieProcessor` manages a cookie jar that is used when accessing URLs. This module was contributed by John J. Lee. @@ -1352,7 +1353,7 @@ Loper and Tim Peters. Testing can still be as simple as running :func:`doctest.testmod`, but the refactorings allow customizing the module's operation in various ways -The new :class:`DocTestFinder` class extracts the tests from a given object's +The new :class:`~doctest.DocTestFinder` class extracts the tests from a given object's docstrings:: def f (x, y): @@ -1368,7 +1369,7 @@ docstrings:: # Get list of DocTest instances tests = finder.find(f) -The new :class:`DocTestRunner` class then runs individual tests and can produce +The new :class:`~doctest.DocTestRunner` class then runs individual tests and can produce a summary of the results:: runner = doctest.DocTestRunner() @@ -1385,10 +1386,10 @@ The above example produces the following output:: 2 passed and 0 failed. Test passed. -:class:`DocTestRunner` uses an instance of the :class:`OutputChecker` class to +:class:`~doctest.DocTestRunner` uses an instance of the :class:`~doctest.OutputChecker` class to compare the expected output with the actual output. This class takes a number of different flags that customize its behaviour; ambitious users can also write -a completely new subclass of :class:`OutputChecker`. +a completely new subclass of :class:`~doctest.OutputChecker`. The default output checker provides a number of handy features. For example, with the :const:`doctest.ELLIPSIS` option flag, an ellipsis (``...``) in the @@ -1465,7 +1466,7 @@ Some of the changes to Python's build process and to the C API are: lookups without masking exceptions raised during the look-up process. (Contributed by Raymond Hettinger.) -* The :c:expr:`Py_IS_NAN(X)` macro returns 1 if its float or double argument +* The :c:macro:`Py_IS_NAN` macro returns 1 if its float or double argument *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new @@ -1479,7 +1480,7 @@ Some of the changes to Python's build process and to the C API are: * A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve - the access time for a method such as :meth:`set.__contains__`. (Contributed by + the access time for a method such as :meth:`!set.__contains__`. (Contributed by Raymond Hettinger.) * Python can now be built with additional profiling for the interpreter itself, @@ -1493,7 +1494,7 @@ Some of the changes to Python's build process and to the C API are: register". (Contributed by Jeremy Hylton.) * The :c:type:`!tracebackobject` type has been renamed to - :c:type:`PyTracebackObject`. + :c:type:`!PyTracebackObject`. .. ====================================================================== @@ -1517,14 +1518,14 @@ code: trigger a :exc:`FutureWarning` and return a value limited to 32 or 64 bits; instead they return a long integer. -* Integer operations will no longer trigger an :exc:`OverflowWarning`. The - :exc:`OverflowWarning` warning will disappear in Python 2.5. +* Integer operations will no longer trigger an :exc:`!OverflowWarning`. The + :exc:`!OverflowWarning` warning will disappear in Python 2.5. -* The :func:`zip` built-in function and :func:`itertools.izip` now return an +* The :func:`zip` built-in function and :func:`!itertools.izip` now return an empty list instead of raising a :exc:`TypeError` exception if called with no arguments. -* You can no longer compare the :class:`date` and :class:`~datetime.datetime` instances +* You can no longer compare the :class:`~datetime.date` and :class:`~datetime.datetime` instances provided by the :mod:`datetime` module. Two instances of different classes will now always be unequal, and relative comparisons (``<``, ``>``) will raise a :exc:`TypeError`. @@ -1532,7 +1533,8 @@ code: * :func:`!dircache.listdir` now passes exceptions to the caller instead of returning empty lists. -* :func:`LexicalHandler.startDTD` used to receive the public and system IDs in +* :meth:`LexicalHandler.startDTD ` + used to receive the public and system IDs in the wrong order. This has been corrected; applications relying on the wrong order need to be fixed. @@ -1547,9 +1549,9 @@ code: * :const:`None` is now a constant; code that binds a new value to the name ``None`` is now a syntax error. -* The :func:`signals.signal` function now raises a :exc:`RuntimeError` exception +* The :func:`signal.signal` function now raises a :exc:`RuntimeError` exception for certain illegal values; previously these errors would pass silently. For - example, you can no longer set a handler on the :const:`SIGKILL` signal. + example, you can no longer set a handler on the :const:`~signal.SIGKILL` signal. .. ====================================================================== diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 03e612fb651e71b..33d8b741ba17668 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -16,8 +16,8 @@ release schedule. Python 2.5 was released on September 19, 2006. The changes in Python 2.5 are an interesting mix of language and library improvements. The library enhancements will be more important to Python's user community, I think, because several widely useful packages were added. New -modules include ElementTree for XML processing (:mod:`xml.etree`), -the SQLite database module (:mod:`sqlite`), and the :mod:`ctypes` +modules include ElementTree for XML processing (:mod:`xml.etree.ElementTree`), +the SQLite database module (:mod:`sqlite3`), and the :mod:`ctypes` module for calling C functions. The language changes are of middling significance. Some pleasant new features @@ -134,14 +134,14 @@ PEP 309: Partial Function Application The :mod:`functools` module is intended to contain tools for functional-style programming. -One useful tool in this module is the :func:`partial` function. For programs +One useful tool in this module is the :func:`~functools.partial` function. For programs written in a functional style, you'll sometimes want to construct variants of existing functions that have some of the parameters filled in. Consider a Python function ``f(a, b, c)``; you could create a new function ``g(b, c)`` that was equivalent to ``f(1, b, c)``. This is called "partial function application". -:func:`partial` takes the arguments ``(function, arg1, arg2, ... kwarg1=value1, +:func:`~functools.partial` takes the arguments ``(function, arg1, arg2, ... kwarg1=value1, kwarg2=value2)``. The resulting object is callable, so you can just call it to invoke *function* with the filled-in arguments. @@ -159,7 +159,7 @@ Here's a small but realistic example:: Here's another example, from a program that uses PyGTK. Here a context-sensitive pop-up menu is being constructed dynamically. The callback provided -for the menu option is a partially applied version of the :meth:`open_item` +for the menu option is a partially applied version of the ``open_item()`` method, where the first argument has been provided. :: ... @@ -172,7 +172,7 @@ method, where the first argument has been provided. :: Another function in the :mod:`functools` module is the ``update_wrapper(wrapper, wrapped)`` function that helps you write -well-behaved decorators. :func:`update_wrapper` copies the name, module, and +well-behaved decorators. :func:`~functools.update_wrapper` copies the name, module, and docstring attribute to a wrapper function so that tracebacks inside the wrapped function are easier to understand. For example, you might write:: @@ -183,7 +183,7 @@ function are easier to understand. For example, you might write:: functools.update_wrapper(wrapper, f) return wrapper -:func:`wraps` is a decorator that can be used inside your own decorators to copy +:func:`~functools.wraps` is a decorator that can be used inside your own decorators to copy the wrapped function's information. An alternate version of the previous example would be:: @@ -209,7 +209,7 @@ example would be:: PEP 314: Metadata for Python Software Packages v1.1 =================================================== -Some simple dependency support was added to Distutils. The :func:`setup` +Some simple dependency support was added to Distutils. The :func:`!setup` function now has ``requires``, ``provides``, and ``obsoletes`` keyword parameters. When you build a source distribution using the ``sdist`` command, the dependency information will be recorded in the :file:`PKG-INFO` file. @@ -271,26 +271,26 @@ Let's say you have a package directory like this:: pkg/main.py pkg/string.py -This defines a package named :mod:`pkg` containing the :mod:`pkg.main` and -:mod:`pkg.string` submodules. +This defines a package named ``pkg`` containing the ``pkg.main`` and +``pkg.string`` submodules. Consider the code in the :file:`main.py` module. What happens if it executes the statement ``import string``? In Python 2.4 and earlier, it will first look in the package's directory to perform a relative import, finds :file:`pkg/string.py`, imports the contents of that file as the -:mod:`pkg.string` module, and that module is bound to the name ``string`` in the -:mod:`pkg.main` module's namespace. +``pkg.string`` module, and that module is bound to the name ``string`` in the +``pkg.main`` module's namespace. -That's fine if :mod:`pkg.string` was what you wanted. But what if you wanted +That's fine if ``pkg.string`` was what you wanted. But what if you wanted Python's standard :mod:`string` module? There's no clean way to ignore -:mod:`pkg.string` and look for the standard module; generally you had to look at +``pkg.string`` and look for the standard module; generally you had to look at the contents of ``sys.modules``, which is slightly unclean. Holger Krekel's -:mod:`py.std` package provides a tidier way to perform imports from the standard +:mod:`!py.std` package provides a tidier way to perform imports from the standard library, ``import py; py.std.string.join()``, but that package isn't available on all Python installations. Reading code which relies on relative imports is also less clear, because a -reader may be confused about which module, :mod:`string` or :mod:`pkg.string`, +reader may be confused about which module, :mod:`string` or ``pkg.string``, is intended to be used. Python users soon learned not to duplicate the names of standard library modules in the names of their packages' submodules, but you can't protect against having your submodule's name being used for a new module @@ -313,9 +313,9 @@ name when using the ``from ... import`` form:: from . import string This imports the :mod:`string` module relative to the current package, so in -:mod:`pkg.main` this will import *name1* and *name2* from :mod:`pkg.string`. +``pkg.main`` this will import *name1* and *name2* from ``pkg.string``. Additional leading periods perform the relative import starting from the parent -of the current package. For example, code in the :mod:`A.B.C` module can do:: +of the current package. For example, code in the ``A.B.C`` module can do:: from . import D # Imports A.B.D from .. import E # Imports A.E @@ -331,7 +331,7 @@ statement, only the ``from ... import`` form. PEP written by Aahz; implemented by Thomas Wouters. https://pylib.readthedocs.io/ - The py library by Holger Krekel, which contains the :mod:`py.std` package. + The py library by Holger Krekel, which contains the :mod:`!py.std` package. .. ====================================================================== @@ -347,7 +347,7 @@ Python interpreter, the switch now uses an implementation in a new module, :mod:`runpy`. The :mod:`runpy` module implements a more sophisticated import mechanism so that -it's now possible to run modules in a package such as :mod:`pychecker.checker`. +it's now possible to run modules in a package such as :mod:`!pychecker.checker`. The module also supports alternative import mechanisms such as the :mod:`zipimport` module. This means you can add a .zip archive's path to ``sys.path`` and then use the :option:`-m` switch to execute code from the @@ -392,8 +392,8 @@ write:: The code in *block-1* is executed. If the code raises an exception, the various :keyword:`except` blocks are tested: if the exception is of class -:class:`Exception1`, *handler-1* is executed; otherwise if it's of class -:class:`Exception2`, *handler-2* is executed, and so forth. If no exception is +``Exception1``, *handler-1* is executed; otherwise if it's of class +``Exception2``, *handler-2* is executed, and so forth. If no exception is raised, the *else-block* is executed. No matter what happened previously, the *final-block* is executed once the code @@ -492,21 +492,21 @@ And here's an example of changing the counter:: :keyword:`yield` will usually return :const:`None`, so you should always check for this case. Don't just use its value in expressions unless you're sure that -the :meth:`send` method will be the only method used to resume your generator +the :meth:`~generator.send` method will be the only method used to resume your generator function. -In addition to :meth:`send`, there are two other new methods on generators: +In addition to :meth:`~generator.send`, there are two other new methods on generators: * ``throw(type, value=None, traceback=None)`` is used to raise an exception inside the generator; the exception is raised by the :keyword:`yield` expression where the generator's execution is paused. -* :meth:`close` raises a new :exc:`GeneratorExit` exception inside the generator +* :meth:`~generator.close` raises a new :exc:`GeneratorExit` exception inside the generator to terminate the iteration. On receiving this exception, the generator's code must either raise :exc:`GeneratorExit` or :exc:`StopIteration`. Catching the :exc:`GeneratorExit` exception and returning a value is illegal and will trigger a :exc:`RuntimeError`; if the function raises some other exception, that - exception is propagated to the caller. :meth:`close` will also be called by + exception is propagated to the caller. :meth:`~generator.close` will also be called by Python's garbage collector when the generator is garbage-collected. If you need to run cleanup code when a :exc:`GeneratorExit` occurs, I suggest @@ -521,8 +521,8 @@ function, and a :keyword:`return` statement), but coroutines can be entered, exited, and resumed at many different points (the :keyword:`yield` statements). We'll have to figure out patterns for using coroutines effectively in Python. -The addition of the :meth:`close` method has one side effect that isn't obvious. -:meth:`close` is called when a generator is garbage-collected, so this means the +The addition of the :meth:`~generator.close` method has one side effect that isn't obvious. +:meth:`~generator.close` is called when a generator is garbage-collected, so this means the generator's code gets one last chance to run before the generator is destroyed. This last chance means that ``try...finally`` statements in generators can now be guaranteed to work; the :keyword:`finally` clause will now always get a @@ -534,8 +534,8 @@ seems like a minor bit of language trivia, but using generators and in the following section. Another even more esoteric effect of this change: previously, the -:attr:`gi_frame` attribute of a generator was always a frame object. It's now -possible for :attr:`gi_frame` to be ``None`` once the generator has been +:attr:`!gi_frame` attribute of a generator was always a frame object. It's now +possible for :attr:`!gi_frame` to be ``None`` once the generator has been exhausted. @@ -622,7 +622,7 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The new :func:`localcontext` function in the :mod:`decimal` module makes it easy +The new :func:`~decimal.localcontext` function in the :mod:`decimal` module makes it easy to save and restore the current decimal context, which encapsulates the desired precision and rounding characteristics for computations:: @@ -692,7 +692,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +``DatabaseConnection`` that I'll assume:: class DatabaseConnection: # Database interface @@ -745,7 +745,7 @@ The contextlib module The new :mod:`contextlib` module provides some functions and a decorator that are useful for writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will be the method's return @@ -910,7 +910,7 @@ may therefore need to have some variables changed to :c:type:`Py_ssize_t`. The :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` functions have a new conversion code, ``n``, for :c:type:`Py_ssize_t`. :c:func:`PyArg_ParseTuple`'s ``s#`` and ``t#`` still output :c:expr:`int` by default, but you can define the -macro :c:macro:`PY_SSIZE_T_CLEAN` before including :file:`Python.h` to make +macro :c:macro:`!PY_SSIZE_T_CLEAN` before including :file:`Python.h` to make them return :c:type:`Py_ssize_t`. :pep:`353` has a section on conversion guidelines that extension authors should @@ -931,19 +931,19 @@ PEP 357: The '__index__' method =============================== The NumPy developers had a problem that could only be solved by adding a new -special method, :meth:`__index__`. When using slice notation, as in +special method, :meth:`~object.__index__`. When using slice notation, as in ``[start:stop:step]``, the values of the *start*, *stop*, and *step* indexes must all be either integers or long integers. NumPy defines a variety of specialized integer types corresponding to unsigned and signed integers of 8, 16, 32, and 64 bits, but there was no way to signal that these types could be used as slice indexes. -Slicing can't just use the existing :meth:`__int__` method because that method +Slicing can't just use the existing :meth:`~object.__int__` method because that method is also used to implement coercion to integers. If slicing used -:meth:`__int__`, floating-point numbers would also become legal slice indexes +:meth:`~object.__int__`, floating-point numbers would also become legal slice indexes and that's clearly an undesirable behaviour. -Instead, a new special method called :meth:`__index__` was added. It takes no +Instead, a new special method called :meth:`~object.__index__` was added. It takes no arguments and returns an integer giving the slice index to use. For example:: class C: @@ -957,7 +957,7 @@ interpreter will check that the type returned is correct, and raises a A corresponding :c:member:`~PyNumberMethods.nb_index` slot was added to the C-level :c:type:`PyNumberMethods` structure to let C extensions implement this protocol. ``PyNumber_Index(obj)`` can be used in extension code to call the -:meth:`__index__` function and retrieve its result. +:meth:`~object.__index__` function and retrieve its result. .. seealso:: @@ -978,7 +978,7 @@ Here are all of the changes that Python 2.5 makes to the core Python language. * The :class:`dict` type has a new hook for letting subclasses provide a default value when a key isn't contained in the dictionary. When a key isn't found, the dictionary's ``__missing__(key)`` method will be called. This hook is used - to implement the new :class:`defaultdict` class in the :mod:`collections` + to implement the new :class:`~collections.defaultdict` class in the :mod:`collections` module. The following example defines a dictionary that returns zero for any missing key:: @@ -1017,7 +1017,7 @@ Here are all of the changes that Python 2.5 makes to the core Python language. (Implemented by Fredrik Lundh following a suggestion by Raymond Hettinger.) -* The :meth:`startswith` and :meth:`endswith` methods of string types now accept +* The :meth:`~str.startswith` and :meth:`~str.endswith` methods of string types now accept tuples of strings to check for. :: def is_image_file (filename): @@ -1028,7 +1028,7 @@ Here are all of the changes that Python 2.5 makes to the core Python language. .. RFE #1491485 * The :func:`min` and :func:`max` built-in functions gained a ``key`` keyword - parameter analogous to the ``key`` argument for :meth:`sort`. This parameter + parameter analogous to the ``key`` argument for :meth:`~list.sort`. This parameter supplies a function that takes a single argument and is called for every value in the list; :func:`min`/:func:`max` will return the element with the smallest/largest return value from this function. For example, to find the @@ -1049,12 +1049,12 @@ Here are all of the changes that Python 2.5 makes to the core Python language. returned by the iterator evaluate as true. (Suggested by Guido van Rossum, and implemented by Raymond Hettinger.) -* The result of a class's :meth:`__hash__` method can now be either a long +* The result of a class's :meth:`~object.__hash__` method can now be either a long integer or a regular integer. If a long integer is returned, the hash of that value is taken. In earlier versions the hash value was required to be a regular integer, but in 2.5 the :func:`id` built-in was changed to always return non-negative numbers, and users often seem to use ``id(self)`` in - :meth:`__hash__` methods (though this is discouraged). + :meth:`~object.__hash__` methods (though this is discouraged). .. Bug #1536021 @@ -1166,7 +1166,7 @@ marked in the following list. .. Patch 1442927 * It's now illegal to mix iterating over a file with ``for line in file`` and - calling the file object's :meth:`read`/:meth:`readline`/:meth:`readlines` + calling the file object's :meth:`!read`/:meth:`!readline`/:meth:`!readlines` methods. Iteration uses an internal buffer and the :meth:`!read\*` methods don't use that buffer. Instead they would return the data following the buffer, causing the data to appear out of order. Mixing iteration and these @@ -1204,7 +1204,7 @@ marked in the following list. Sean Reifschneider at the NeedForSpeed sprint.) * Importing now caches the paths tried, recording whether they exist or not so - that the interpreter makes fewer :c:func:`open` and :c:func:`stat` calls on + that the interpreter makes fewer :c:func:`!open` and :c:func:`!stat` calls on startup. (Contributed by Martin von Löwis and Georg Brandl.) .. Patch 921466 @@ -1226,23 +1226,24 @@ complete list of changes, or look through the SVN logs for all the details. u-LAW encoding has been improved. (Contributed by Lars Immisch.) * The :mod:`codecs` module gained support for incremental codecs. The - :func:`codec.lookup` function now returns a :class:`CodecInfo` instance instead - of a tuple. :class:`CodecInfo` instances behave like a 4-tuple to preserve - backward compatibility but also have the attributes :attr:`encode`, - :attr:`decode`, :attr:`incrementalencoder`, :attr:`incrementaldecoder`, - :attr:`streamwriter`, and :attr:`streamreader`. Incremental codecs can receive + :func:`codecs.lookup` function now returns a :class:`~codecs.CodecInfo` instance instead + of a tuple. :class:`~codecs.CodecInfo` instances behave like a 4-tuple to preserve + backward compatibility but also have the attributes :attr:`~codecs.CodecInfo.encode`, + :attr:`~codecs.CodecInfo.decode`, :attr:`~codecs.CodecInfo.incrementalencoder`, + :attr:`~codecs.CodecInfo.incrementaldecoder`, :attr:`~codecs.CodecInfo.streamwriter`, + and :attr:`~codecs.CodecInfo.streamreader`. Incremental codecs can receive input and produce output in multiple chunks; the output is the same as if the entire input was fed to the non-incremental codec. See the :mod:`codecs` module documentation for details. (Designed and implemented by Walter Dörwald.) .. Patch 1436130 -* The :mod:`collections` module gained a new type, :class:`defaultdict`, that +* The :mod:`collections` module gained a new type, :class:`~collections.defaultdict`, that subclasses the standard :class:`dict` type. The new type mostly behaves like a dictionary but constructs a default value when a key isn't present, automatically adding it to the dictionary for the requested key value. - The first argument to :class:`defaultdict`'s constructor is a factory function + The first argument to :class:`~collections.defaultdict`'s constructor is a factory function that gets called whenever a key is requested but not found. This factory function receives no arguments, so you can use built-in type constructors such as :func:`list` or :func:`int`. For example, you can make an index of words @@ -1268,7 +1269,7 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Guido van Rossum.) -* The :class:`deque` double-ended queue type supplied by the :mod:`collections` +* The :class:`~collections.deque` double-ended queue type supplied by the :mod:`collections` module now has a ``remove(value)`` method that removes the first occurrence of *value* in the queue, raising :exc:`ValueError` if the value isn't found. (Contributed by Raymond Hettinger.) @@ -1287,15 +1288,15 @@ complete list of changes, or look through the SVN logs for all the details. Also, the :mod:`pstats` module for analyzing the data measured by the profiler now supports directing the output to any file object by supplying a *stream* - argument to the :class:`Stats` constructor. (Contributed by Skip Montanaro.) + argument to the :class:`~pstats.Stats` constructor. (Contributed by Skip Montanaro.) * The :mod:`csv` module, which parses files in comma-separated value format, received several enhancements and a number of bugfixes. You can now set the maximum size in bytes of a field by calling the ``csv.field_size_limit(new_limit)`` function; omitting the *new_limit* - argument will return the currently set limit. The :class:`reader` class now has - a :attr:`line_num` attribute that counts the number of physical lines read from - the source; records can span multiple physical lines, so :attr:`line_num` is not + argument will return the currently set limit. The :class:`~csv.reader` class now has + a :attr:`~csv.csvreader.line_num` attribute that counts the number of physical lines read from + the source; records can span multiple physical lines, so :attr:`~csv.csvreader.line_num` is not the same as the number of records read. The CSV parser is now stricter about multi-line quoted fields. Previously, if a @@ -1318,7 +1319,7 @@ complete list of changes, or look through the SVN logs for all the details. ts = dt.datetime.strptime('10:13:15 2006-03-07', '%H:%M:%S %Y-%m-%d') -* The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` +* The :meth:`difflib.SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching subsequences. Previously, the algorithm would occasionally break a block of matching elements into two list entries. (Enhancement by Tim Peters.) @@ -1327,8 +1328,8 @@ complete list of changes, or look through the SVN logs for all the details. being executed at all. This is intended for code snippets that are usage examples intended for the reader and aren't actually test cases. - An *encoding* parameter was added to the :func:`testfile` function and the - :class:`DocFileSuite` class to specify the file's encoding. This makes it + An *encoding* parameter was added to the :func:`~doctest.testfile` function and the + :class:`~doctest.DocFileSuite` class to specify the file's encoding. This makes it easier to use non-ASCII characters in tests contained within a docstring. (Contributed by Bjorn Tillenius.) @@ -1347,20 +1348,20 @@ complete list of changes, or look through the SVN logs for all the details. :func:`input` function to allow opening files in binary or :term:`universal newlines` mode. Another new parameter, *openhook*, lets you use a function other than :func:`open` to open the input files. Once you're iterating over - the set of files, the :class:`FileInput` object's new :meth:`~fileinput.fileno` returns + the set of files, the :class:`~fileinput.FileInput` object's new :meth:`~fileinput.fileno` returns the file descriptor for the currently opened file. (Contributed by Georg Brandl.) -* In the :mod:`gc` module, the new :func:`get_count` function returns a 3-tuple +* In the :mod:`gc` module, the new :func:`~gc.get_count` function returns a 3-tuple containing the current collection counts for the three GC generations. This is accounting information for the garbage collector; when these counts reach a specified threshold, a garbage collection sweep will be made. The existing :func:`gc.collect` function now takes an optional *generation* argument of 0, 1, or 2 to specify which generation to collect. (Contributed by Barry Warsaw.) -* The :func:`nsmallest` and :func:`nlargest` functions in the :mod:`heapq` +* The :func:`~heapq.nsmallest` and :func:`~heapq.nlargest` functions in the :mod:`heapq` module now support a ``key`` keyword parameter similar to the one provided by - the :func:`min`/:func:`max` functions and the :meth:`sort` methods. For + the :func:`min`/:func:`max` functions and the :meth:`~list.sort` methods. For example:: >>> import heapq @@ -1382,7 +1383,7 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Raymond Hettinger.) * The :func:`format` function in the :mod:`locale` module has been modified and - two new functions were added, :func:`format_string` and :func:`currency`. + two new functions were added, :func:`~locale.format_string` and :func:`~locale.currency`. The :func:`format` function's *val* parameter could previously be a string as long as no more than one %char specifier appeared; now the parameter must be @@ -1391,10 +1392,10 @@ complete list of changes, or look through the SVN logs for all the details. formatting currency in placing a separator between groups of three digits. To format strings with multiple %char specifiers, use the new - :func:`format_string` function that works like :func:`format` but also supports + :func:`~locale.format_string` function that works like :func:`format` but also supports mixing %char specifiers with arbitrary text. - A new :func:`currency` function was also added that formats a number according + A new :func:`~locale.currency` function was also added that formats a number according to the current locale's settings. (Contributed by Georg Brandl.) @@ -1403,9 +1404,11 @@ complete list of changes, or look through the SVN logs for all the details. * The :mod:`mailbox` module underwent a massive rewrite to add the capability to modify mailboxes in addition to reading them. A new set of classes that include - :class:`mbox`, :class:`MH`, and :class:`Maildir` are used to read mailboxes, and + :class:`~mailbox.mbox`, :class:`~mailbox.MH`, and :class:`~mailbox.Maildir` + are used to read mailboxes, and have an ``add(message)`` method to add messages, ``remove(key)`` to - remove messages, and :meth:`lock`/:meth:`unlock` to lock/unlock the mailbox. + remove messages, and :meth:`~mailbox.Mailbox.lock`/:meth:`~mailbox.Mailbox.unlock` + to lock/unlock the mailbox. The following example converts a maildir-format mailbox into an mbox-format one:: @@ -1430,19 +1433,19 @@ complete list of changes, or look through the SVN logs for all the details. default domain by supplying a *domain* argument to the :func:`!nis.match` and :func:`!nis.maps` functions. (Contributed by Ben Bell.) -* The :mod:`operator` module's :func:`itemgetter` and :func:`attrgetter` +* The :mod:`operator` module's :func:`~operator.itemgetter` and :func:`~operator.attrgetter` functions now support multiple fields. A call such as ``operator.attrgetter('a', 'b')`` will return a function that retrieves the - :attr:`a` and :attr:`b` attributes. Combining this new feature with the - :meth:`sort` method's ``key`` parameter lets you easily sort lists using + ``a`` and ``b`` attributes. Combining this new feature with the + :meth:`~list.sort` method's ``key`` parameter lets you easily sort lists using multiple fields. (Contributed by Raymond Hettinger.) * The :mod:`optparse` module was updated to version 1.5.1 of the Optik library. - The :class:`OptionParser` class gained an :attr:`epilog` attribute, a string - that will be printed after the help message, and a :meth:`destroy` method to + The :class:`~optparse.OptionParser` class gained an :attr:`!epilog` attribute, a string + that will be printed after the help message, and a :meth:`!destroy` method to break reference cycles created by the object. (Contributed by Greg Ward.) -* The :mod:`os` module underwent several changes. The :attr:`stat_float_times` +* The :mod:`os` module underwent several changes. The :attr:`!stat_float_times` variable now defaults to true, meaning that :func:`os.stat` will now return time values as floats. (This doesn't necessarily mean that :func:`os.stat` will return times that are precise to fractions of a second; not all systems support @@ -1453,19 +1456,20 @@ complete list of changes, or look through the SVN logs for all the details. :func:`os.lseek` function. Two new constants for locking are :const:`os.O_SHLOCK` and :const:`os.O_EXLOCK`. - Two new functions, :func:`wait3` and :func:`wait4`, were added. They're similar - the :func:`waitpid` function which waits for a child process to exit and returns - a tuple of the process ID and its exit status, but :func:`wait3` and - :func:`wait4` return additional information. :func:`wait3` doesn't take a + Two new functions, :func:`~os.wait3` and :func:`~os.wait4`, were added. They're similar + the :func:`~os.waitpid` function which waits for a child process to exit and returns + a tuple of the process ID and its exit status, but :func:`~os.wait3` and + :func:`~os.wait4` return additional information. :func:`~os.wait3` doesn't take a process ID as input, so it waits for any child process to exit and returns a 3-tuple of *process-id*, *exit-status*, *resource-usage* as returned from the :func:`resource.getrusage` function. ``wait4(pid)`` does take a process ID. (Contributed by Chad J. Schroeder.) On FreeBSD, the :func:`os.stat` function now returns times with nanosecond - resolution, and the returned object now has :attr:`st_gen` and - :attr:`st_birthtime`. The :attr:`st_flags` attribute is also available, if the - platform supports it. (Contributed by Antti Louko and Diego Pettenò.) + resolution, and the returned object now has :attr:`~os.stat_result.st_gen` and + :attr:`~os.stat_result.st_birthtime`. The :attr:`~os.stat_result.st_flags` + attribute is also available, if the platform supports it. + (Contributed by Antti Louko and Diego Pettenò.) .. (Patch 1180695, 1212117) @@ -1495,21 +1499,21 @@ complete list of changes, or look through the SVN logs for all the details. instead of performing many different operations and reducing the result to a single number as :file:`pystone.py` does. -* The :mod:`pyexpat` module now uses version 2.0 of the Expat parser. +* The :mod:`!pyexpat` module now uses version 2.0 of the Expat parser. (Contributed by Trent Mick.) -* The :class:`~queue.Queue` class provided by the :mod:`Queue` module gained two new - methods. :meth:`join` blocks until all items in the queue have been retrieved +* The :class:`~queue.Queue` class provided by the :mod:`queue` module gained two new + methods. :meth:`~queue.Queue.join` blocks until all items in the queue have been retrieved and all processing work on the items have been completed. Worker threads call - the other new method, :meth:`task_done`, to signal that processing for an item + the other new method, :meth:`~queue.Queue.task_done`, to signal that processing for an item has been completed. (Contributed by Raymond Hettinger.) -* The old :mod:`regex` and :mod:`regsub` modules, which have been deprecated +* The old :mod:`!regex` and :mod:`!regsub` modules, which have been deprecated ever since Python 2.0, have finally been deleted. Other deleted modules: - :mod:`statcache`, :mod:`tzparse`, :mod:`whrandom`. + :mod:`!statcache`, :mod:`!tzparse`, :mod:`!whrandom`. * Also deleted: the :file:`lib-old` directory, which includes ancient modules - such as :mod:`dircmp` and :mod:`ni`, was removed. :file:`lib-old` wasn't on the + such as :mod:`!dircmp` and :mod:`!ni`, was removed. :file:`lib-old` wasn't on the default ``sys.path``, so unless your programs explicitly added the directory to ``sys.path``, this removal shouldn't affect your code. @@ -1519,14 +1523,16 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #1472854 -* The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer ` classes now have a - :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set +* The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer + ` classes now have a :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` + attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting - :attr:`rpc_paths` to ``None`` or an empty tuple disables this path checking. + :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` to ``None`` or an + empty tuple disables this path checking. .. Bug #1473048 -* The :mod:`socket` module now supports :const:`AF_NETLINK` sockets on Linux, +* The :mod:`socket` module now supports :const:`!AF_NETLINK` sockets on Linux, thanks to a patch from Philippe Biondi. Netlink sockets are a Linux-specific mechanism for communications between a user-space process and kernel code; an introductory article about them is at https://www.linuxjournal.com/article/7356. @@ -1538,19 +1544,19 @@ complete list of changes, or look through the SVN logs for all the details. supports the buffer protocol instead of returning the data as a string. This means you can put the data directly into an array or a memory-mapped file. - Socket objects also gained :meth:`getfamily`, :meth:`gettype`, and - :meth:`getproto` accessor methods to retrieve the family, type, and protocol + Socket objects also gained :meth:`!getfamily`, :meth:`!gettype`, and + :meth:`!getproto` accessor methods to retrieve the family, type, and protocol values for the socket. * New module: the :mod:`!spwd` module provides functions for accessing the shadow password database on systems that support shadow passwords. * The :mod:`struct` is now faster because it compiles format strings into - :class:`Struct` objects with :meth:`pack` and :meth:`unpack` methods. This is + :class:`~struct.Struct` objects with :meth:`~struct.Struct.pack` and :meth:`~struct.Struct.unpack` methods. This is similar to how the :mod:`re` module lets you create compiled regular expression - objects. You can still use the module-level :func:`pack` and :func:`unpack` - functions; they'll create :class:`Struct` objects and cache them. Or you can - use :class:`Struct` instances directly:: + objects. You can still use the module-level :func:`~struct.pack` and :func:`~struct.unpack` + functions; they'll create :class:`~struct.Struct` objects and cache them. Or you can + use :class:`~struct.Struct` instances directly:: s = struct.Struct('ih3s') @@ -1562,7 +1568,7 @@ complete list of changes, or look through the SVN logs for all the details. offset)`` methods. This lets you store data directly into an array or a memory-mapped file. - (:class:`Struct` objects were implemented by Bob Ippolito at the NeedForSpeed + (:class:`~struct.Struct` objects were implemented by Bob Ippolito at the NeedForSpeed sprint. Support for buffer objects was added by Martin Blais, also at the NeedForSpeed sprint.) @@ -1582,8 +1588,8 @@ complete list of changes, or look through the SVN logs for all the details. topmost stack frame currently active in that thread at the time the function is called. (Contributed by Tim Peters.) -* The :class:`TarFile` class in the :mod:`tarfile` module now has an - :meth:`extractall` method that extracts all members from the archive into the +* The :class:`~tarfile.TarFile` class in the :mod:`tarfile` module now has an + :meth:`~tarfile.TarFile.extractall` method that extracts all members from the archive into the current working directory. It's also possible to set a different directory as the extraction target, and to unpack only a subset of the archive's members. @@ -1607,8 +1613,8 @@ complete list of changes, or look through the SVN logs for all the details. * New module: the :mod:`uuid` module generates universally unique identifiers (UUIDs) according to :rfc:`4122`. The RFC defines several different UUID versions that are generated from a starting string, from system properties, or - purely randomly. This module contains a :class:`UUID` class and functions - named :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, and :func:`uuid5` to + purely randomly. This module contains a :class:`~uuid.UUID` class and functions + named :func:`~uuid.uuid1`, :func:`~uuid.uuid3`, :func:`~uuid.uuid4`, and :func:`~uuid.uuid5` to generate different versions of UUID. (Version 2 UUIDs are not specified in :rfc:`4122` and are not supported by this module.) :: @@ -1631,18 +1637,18 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Ka-Ping Yee.) -* The :mod:`weakref` module's :class:`WeakKeyDictionary` and - :class:`WeakValueDictionary` types gained new methods for iterating over the - weak references contained in the dictionary. :meth:`iterkeyrefs` and - :meth:`keyrefs` methods were added to :class:`WeakKeyDictionary`, and - :meth:`itervaluerefs` and :meth:`valuerefs` were added to - :class:`WeakValueDictionary`. (Contributed by Fred L. Drake, Jr.) +* The :mod:`weakref` module's :class:`~weakref.WeakKeyDictionary` and + :class:`~weakref.WeakValueDictionary` types gained new methods for iterating over the + weak references contained in the dictionary. :meth:`!iterkeyrefs` and + :meth:`~weakref.WeakKeyDictionary.keyrefs` methods were added to :class:`~weakref.WeakKeyDictionary`, and + :meth:`!itervaluerefs` and :meth:`~weakref.WeakValueDictionary.valuerefs` were added to + :class:`~weakref.WeakValueDictionary`. (Contributed by Fred L. Drake, Jr.) * The :mod:`webbrowser` module received a number of enhancements. It's now usable as a script with ``python -m webbrowser``, taking a URL as the argument; there are a number of switches to control the behaviour (:option:`!-n` for a new browser window, :option:`!-t` for a new tab). New module-level functions, - :func:`open_new` and :func:`open_new_tab`, were added to support this. The + :func:`~webbrowser.open_new` and :func:`~webbrowser.open_new_tab`, were added to support this. The module's :func:`open` function supports an additional feature, an *autoraise* parameter that signals whether to raise the open window when possible. A number of additional browsers were added to the supported list such as Firefox, Opera, @@ -1663,9 +1669,9 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch 1446489 -* The :mod:`zlib` module's :class:`Compress` and :class:`Decompress` objects now +* The :mod:`zlib` module's :class:`!Compress` and :class:`!Decompress` objects now support a :meth:`copy` method that makes a copy of the object's internal state - and returns a new :class:`Compress` or :class:`Decompress` object. + and returns a new :class:`!Compress` or :class:`!Decompress` object. (Contributed by Chris AtLee.) .. Patch 1435422 @@ -1685,34 +1691,34 @@ provides functions for loading shared libraries and calling functions in them. The :mod:`ctypes` package is much fancier. To load a shared library or DLL, you must create an instance of the -:class:`CDLL` class and provide the name or path of the shared library or DLL. +:class:`~ctypes.CDLL` class and provide the name or path of the shared library or DLL. Once that's done, you can call arbitrary functions by accessing them as -attributes of the :class:`CDLL` object. :: +attributes of the :class:`~ctypes.CDLL` object. :: import ctypes libc = ctypes.CDLL('libc.so.6') result = libc.printf("Line of output\n") -Type constructors for the various C types are provided: :func:`c_int`, -:func:`c_float`, :func:`c_double`, :func:`c_char_p` (equivalent to :c:expr:`char +Type constructors for the various C types are provided: :func:`~ctypes.c_int`, +:func:`~ctypes.c_float`, :func:`~ctypes.c_double`, :func:`~ctypes.c_char_p` (equivalent to :c:expr:`char \*`), and so forth. Unlike Python's types, the C versions are all mutable; you -can assign to their :attr:`value` attribute to change the wrapped value. Python +can assign to their :attr:`~ctypes._SimpleCData.value` attribute to change the wrapped value. Python integers and strings will be automatically converted to the corresponding C types, but for other types you must call the correct type constructor. (And I mean *must*; getting it wrong will often result in the interpreter crashing with a segmentation fault.) -You shouldn't use :func:`c_char_p` with a Python string when the C function will +You shouldn't use :func:`~ctypes.c_char_p` with a Python string when the C function will be modifying the memory area, because Python strings are supposed to be immutable; breaking this rule will cause puzzling bugs. When you need a -modifiable memory area, use :func:`create_string_buffer`:: +modifiable memory area, use :func:`~ctypes.create_string_buffer`:: s = "this is a string" buf = ctypes.create_string_buffer(s) libc.strfry(buf) -C functions are assumed to return integers, but you can set the :attr:`restype` +C functions are assumed to return integers, but you can set the :attr:`~ctypes._CFuncPtr.restype` attribute of the function object to change this:: >>> libc.atof('2.71828') @@ -1760,9 +1766,9 @@ The ElementTree package ----------------------- A subset of Fredrik Lundh's ElementTree library for processing XML has been -added to the standard library as :mod:`xml.etree`. The available modules are -:mod:`ElementTree`, :mod:`ElementPath`, and :mod:`ElementInclude` from -ElementTree 1.2.6. The :mod:`cElementTree` accelerator module is also +added to the standard library as :mod:`!xml.etree`. The available modules are +:mod:`!ElementTree`, :mod:`!ElementPath`, and :mod:`!ElementInclude` from +ElementTree 1.2.6. The :mod:`!cElementTree` accelerator module is also included. The rest of this section will provide a brief overview of using ElementTree. @@ -1770,14 +1776,15 @@ Full documentation for ElementTree is available at https://web.archive.org/web/20201124024954/http://effbot.org/zone/element-index.htm. ElementTree represents an XML document as a tree of element nodes. The text -content of the document is stored as the :attr:`text` and :attr:`tail` +content of the document is stored as the :attr:`~xml.etree.ElementTree.Element.text` +and :attr:`~xml.etree.ElementTree.Element.tail` attributes of (This is one of the major differences between ElementTree and the Document Object Model; in the DOM there are many different types of node, -including :class:`TextNode`.) +including :class:`!TextNode`.) -The most commonly used parsing function is :func:`parse`, that takes either a +The most commonly used parsing function is :func:`~xml.etree.ElementTree.parse`, that takes either a string (assumed to contain a filename) or a file-like object and returns an -:class:`ElementTree` instance:: +:class:`~xml.etree.ElementTree.ElementTree` instance:: from xml.etree import ElementTree as ET @@ -1787,11 +1794,13 @@ string (assumed to contain a filename) or a file-like object and returns an 'http://planet.python.org/rss10.xml') tree = ET.parse(feed) -Once you have an :class:`ElementTree` instance, you can call its :meth:`getroot` -method to get the root :class:`Element` node. +Once you have an :class:`~xml.etree.ElementTree.ElementTree` instance, you can +call its :meth:`~xml.etree.ElementTree.ElementTree.getroot` +method to get the root :class:`~xml.etree.ElementTree.Element` node. -There's also an :func:`XML` function that takes a string literal and returns an -:class:`Element` node (not an :class:`ElementTree`). This function provides a +There's also an :func:`~xml.etree.ElementTree.XML` function that takes a string +literal and returns an :class:`~xml.etree.ElementTree.Element` node (not an +:class:`~xml.etree.ElementTree.ElementTree`). This function provides a tidy way to incorporate XML fragments, approaching the convenience of an XML literal:: @@ -1834,7 +1843,7 @@ list-like operations are used to access child nodes. | ``del elem.attrib[name]`` | Deletes attribute *name*. | +-------------------------------+--------------------------------------------+ -Comments and processing instructions are also represented as :class:`Element` +Comments and processing instructions are also represented as :class:`~xml.etree.ElementTree.Element` nodes. To check if a node is a comment or processing instructions:: if elem.tag is ET.Comment: @@ -1842,8 +1851,8 @@ nodes. To check if a node is a comment or processing instructions:: elif elem.tag is ET.ProcessingInstruction: ... -To generate XML output, you should call the :meth:`ElementTree.write` method. -Like :func:`parse`, it can take either a string or a file-like object:: +To generate XML output, you should call the :meth:`xml.etree.ElementTree.ElementTree.write` method. +Like :func:`~xml.etree.ElementTree.parse`, it can take either a string or a file-like object:: # Encoding is US-ASCII tree.write('output.xml') @@ -1913,7 +1922,7 @@ differently. :: Once a hash object has been created, its methods are the same as before: ``update(string)`` hashes the specified string into the current digest -state, :meth:`digest` and :meth:`hexdigest` return the digest value as a binary +state, :meth:`~hashlib.hash.digest` and :meth:`~hashlib.hash.hexdigest` return the digest value as a binary string or a string of hex digits, and :meth:`copy` returns a new hashing object with the same digest state. @@ -1949,7 +1958,7 @@ doesn't include the SQLite code, only the wrapper module. You'll need to have the SQLite libraries and headers installed before compiling Python, and the build process will compile the module when the necessary headers are available. -To use the module, you must first create a :class:`Connection` object that +To use the module, you must first create a :class:`~sqlite3.Connection` object that represents the database. Here the data will be stored in the :file:`/tmp/example` file:: @@ -1957,8 +1966,8 @@ represents the database. Here the data will be stored in the You can also supply the special name ``:memory:`` to create a database in RAM. -Once you have a :class:`Connection`, you can create a :class:`Cursor` object -and call its :meth:`execute` method to perform SQL commands:: +Once you have a :class:`~sqlite3.Connection`, you can create a :class:`~sqlite3.Cursor` object +and call its :meth:`~sqlite3.Cursor.execute` method to perform SQL commands:: c = conn.cursor() @@ -1977,7 +1986,7 @@ is insecure; it makes your program vulnerable to an SQL injection attack. Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`execute` method. (Other database modules +second argument to the cursor's :meth:`~sqlite3.Cursor.execute` method. (Other database modules may use a different placeholder, such as ``%s`` or ``:1``.) For example:: # Never do this -- insecure! @@ -1996,8 +2005,8 @@ may use a different placeholder, such as ``%s`` or ``:1``.) For example:: c.execute('insert into stocks values (?,?,?,?,?)', t) To retrieve data after executing a SELECT statement, you can either treat the -cursor as an iterator, call the cursor's :meth:`fetchone` method to retrieve a -single matching row, or call :meth:`fetchall` to get a list of the matching +cursor as an iterator, call the cursor's :meth:`~sqlite3.Cursor.fetchone` method to retrieve a +single matching row, or call :meth:`~sqlite3.Cursor.fetchall` to get a list of the matching rows. This example uses the iterator form:: @@ -2118,7 +2127,7 @@ Changes to Python's build process and to the C API include: discusses the design. To start learning about the code, read the definition of the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in - :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and + :file:`Include/Python-ast.h`. The :c:func:`!PyParser_ASTFromString` and :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For @@ -2235,9 +2244,9 @@ code: encoding declaration. In Python 2.4 this triggered a warning, not a syntax error. -* Previously, the :attr:`gi_frame` attribute of a generator was always a frame +* Previously, the :attr:`!gi_frame` attribute of a generator was always a frame object. Because of the :pep:`342` changes described in section :ref:`pep-342`, - it's now possible for :attr:`gi_frame` to be ``None``. + it's now possible for :attr:`!gi_frame` to be ``None``. * A new warning, :class:`UnicodeWarning`, is triggered when you attempt to compare a Unicode string and an 8-bit string that can't be converted to Unicode @@ -2258,11 +2267,12 @@ code: return a tuple of arguments instead. The modules also no longer accept the deprecated *bin* keyword parameter. -* Library: The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer ` classes now - have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a +* Library: The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer + ` classes now have a :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` + attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. - Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path - checking. + Setting :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` to ``None`` + or an empty tuple disables this path checking. * C API: Many functions now use :c:type:`Py_ssize_t` instead of :c:expr:`int` to allow processing more data on 64-bit machines. Extension code may need to make diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 1215601a09d681c..3f4645d2b5b4830 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -101,10 +101,10 @@ to break, they've been backported to 2.6 and are described in this document in the appropriate place. Some of the 3.0-derived features are: -* A :meth:`__complex__` method for converting objects to a complex number. +* A :meth:`~object.__complex__` method for converting objects to a complex number. * Alternate syntax for catching exceptions: ``except TypeError as exc``. * The addition of :func:`functools.reduce` as a synonym for the built-in - :func:`reduce` function. + :func:`!reduce` function. Python 3.0 adds several new built-in functions and changes the semantics of some existing builtins. Functions that are new in 3.0 @@ -431,7 +431,7 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`~contextlib.contextmanager`, and lets you write +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will @@ -1145,7 +1145,7 @@ constraints upon the memory returned. Some examples are: * :c:macro:`PyBUF_WRITABLE` indicates that the memory must be writable. -* :c:macro:`PyBUF_LOCK` requests a read-only or exclusive lock on the memory. +* :c:macro:`!PyBUF_LOCK` requests a read-only or exclusive lock on the memory. * :c:macro:`PyBUF_C_CONTIGUOUS` and :c:macro:`PyBUF_F_CONTIGUOUS` requests a C-contiguous (last dimension varies the fastest) or @@ -1339,7 +1339,7 @@ builtin returns the binary representation for a number:: >>> bin(173) '0b10101101' -The :func:`int` and :func:`long` builtins will now accept the "0o" +The :func:`int` and :func:`!long` builtins will now accept the "0o" and "0b" prefixes when base-8 or base-2 are requested, or when the *base* argument is zero (signalling that the base used should be determined from the string):: @@ -1392,35 +1392,36 @@ This is equivalent to:: .. _pep-3141: PEP 3141: A Type Hierarchy for Numbers -===================================================== +====================================== Python 3.0 adds several abstract base classes for numeric types inspired by Scheme's numeric tower. These classes were backported to 2.6 as the :mod:`numbers` module. -The most general ABC is :class:`Number`. It defines no operations at +The most general ABC is :class:`~numbers.Number`. It defines no operations at all, and only exists to allow checking if an object is a number by doing ``isinstance(obj, Number)``. -:class:`Complex` is a subclass of :class:`Number`. Complex numbers +:class:`~numbers.Complex` is a subclass of :class:`~numbers.Number`. Complex numbers can undergo the basic operations of addition, subtraction, multiplication, division, and exponentiation, and you can retrieve the real and imaginary parts and obtain a number's conjugate. Python's built-in -complex type is an implementation of :class:`Complex`. +complex type is an implementation of :class:`~numbers.Complex`. -:class:`Real` further derives from :class:`Complex`, and adds -operations that only work on real numbers: :func:`floor`, :func:`trunc`, +:class:`~numbers.Real` further derives from :class:`~numbers.Complex`, and adds +operations that only work on real numbers: :func:`~math.floor`, :func:`~math.trunc`, rounding, taking the remainder mod N, floor division, and comparisons. -:class:`Rational` numbers derive from :class:`Real`, have -:attr:`numerator` and :attr:`denominator` properties, and can be +:class:`~numbers.Rational` numbers derive from :class:`~numbers.Real`, have +:attr:`~numbers.Rational.numerator` and :attr:`~numbers.Rational.denominator` +properties, and can be converted to floats. Python 2.6 adds a simple rational-number class, -:class:`Fraction`, in the :mod:`fractions` module. (It's called -:class:`Fraction` instead of :class:`Rational` to avoid +:class:`~fractions.Fraction`, in the :mod:`fractions` module. (It's called +:class:`~fractions.Fraction` instead of ``Rational`` to avoid a name clash with :class:`numbers.Rational`.) -:class:`Integral` numbers derive from :class:`Rational`, and +:class:`~numbers.Integral` numbers derive from :class:`~numbers.Rational`, and can be shifted left and right with ``<<`` and ``>>``, combined using bitwise operations such as ``&`` and ``|``, and can be used as array indexes and slice boundaries. @@ -1429,7 +1430,7 @@ In Python 3.0, the PEP slightly redefines the existing builtins :func:`round`, :func:`math.floor`, :func:`math.ceil`, and adds a new one, :func:`math.trunc`, that's been backported to Python 2.6. :func:`math.trunc` rounds toward zero, returning the closest -:class:`Integral` that's between the function's argument and zero. +:class:`~numbers.Integral` that's between the function's argument and zero. .. seealso:: @@ -1442,7 +1443,7 @@ one, :func:`math.trunc`, that's been backported to Python 2.6. The :mod:`fractions` Module --------------------------------------------------- +--------------------------- To fill out the hierarchy of numeric types, the :mod:`fractions` module provides a rational-number class. Rational numbers store their @@ -1450,7 +1451,7 @@ values as a numerator and denominator forming a fraction, and can exactly represent numbers such as ``2/3`` that floating-point numbers can only approximate. -The :class:`Fraction` constructor takes two :class:`Integral` values +The :class:`~fractions.Fraction` constructor takes two :class:`~numbers.Integral` values that will be the numerator and denominator of the resulting fraction. :: >>> from fractions import Fraction @@ -1464,7 +1465,7 @@ that will be the numerator and denominator of the resulting fraction. :: Fraction(5, 3) For converting floating-point numbers to rationals, -the float type now has an :meth:`as_integer_ratio` method that returns +the float type now has an :meth:`~float.as_integer_ratio` method that returns the numerator and denominator for a fraction that evaluates to the same floating-point value:: @@ -1499,7 +1500,7 @@ Some smaller changes made to the core Python language are: :issue:`1739468`.) * The :func:`hasattr` function was catching and ignoring all errors, - under the assumption that they meant a :meth:`__getattr__` method + under the assumption that they meant a :meth:`~object.__getattr__` method was failing somehow and the return value of :func:`hasattr` would therefore be ``False``. This logic shouldn't be applied to :exc:`KeyboardInterrupt` and :exc:`SystemExit`, however; Python 2.6 @@ -1557,8 +1558,9 @@ Some smaller changes made to the core Python language are: .. Revision 57619 -* Properties now have three attributes, :attr:`getter`, :attr:`setter` - and :attr:`deleter`, that are decorators providing useful shortcuts +* Properties now have three decorators, :deco:`~property.getter`, + :deco:`~property.setter` and :deco:`~property.deleter`, that are + decorators providing useful shortcuts for adding a getter, setter or deleter function to an existing property. You would use them like this:: @@ -1585,10 +1587,10 @@ Some smaller changes made to the core Python language are: self._x = value / 2 * Several methods of the built-in set types now accept multiple iterables: - :meth:`intersection`, - :meth:`intersection_update`, - :meth:`union`, :meth:`update`, - :meth:`difference` and :meth:`difference_update`. + :meth:`~set.intersection`, + :meth:`~set.intersection_update`, + :meth:`~set.union`, :meth:`~set.update`, + :meth:`~set.difference` and :meth:`~set.difference_update`. :: @@ -1606,8 +1608,8 @@ Some smaller changes made to the core Python language are: positive or negative infinity. This works on any platform with IEEE 754 semantics. (Contributed by Christian Heimes; :issue:`1635`.) - Other functions in the :mod:`math` module, :func:`isinf` and - :func:`isnan`, return true if their floating-point argument is + Other functions in the :mod:`math` module, :func:`~math.isinf` and + :func:`~math.isnan`, return true if their floating-point argument is infinite or Not A Number. (:issue:`1640`) Conversion functions were added to convert floating-point numbers @@ -1632,17 +1634,17 @@ Some smaller changes made to the core Python language are: :func:`complex` constructor will now preserve the sign of the zero. (Fixed by Mark T. Dickinson; :issue:`1507`.) -* Classes that inherit a :meth:`__hash__` method from a parent class +* Classes that inherit a :meth:`~object.__hash__` method from a parent class can set ``__hash__ = None`` to indicate that the class isn't hashable. This will make ``hash(obj)`` raise a :exc:`TypeError` and the class will not be indicated as implementing the :class:`Hashable` ABC. - You should do this when you've defined a :meth:`__cmp__` or - :meth:`__eq__` method that compares objects by their value rather + You should do this when you've defined a :meth:`!__cmp__` or + :meth:`~object.__eq__` method that compares objects by their value rather than by identity. All objects have a default hash method that uses ``id(obj)`` as the hash value. There's no tidy way to remove the - :meth:`__hash__` method inherited from a parent class, so + :meth:`~object.__hash__` method inherited from a parent class, so assigning ``None`` was implemented as an override. At the C level, extensions can set ``tp_hash`` to :c:func:`PyObject_HashNotImplemented`. @@ -1654,7 +1656,7 @@ Some smaller changes made to the core Python language are: will not inadvertently catch :exc:`GeneratorExit`. (Contributed by Chad Austin; :issue:`1537`.) -* Generator objects now have a :attr:`gi_code` attribute that refers to +* Generator objects now have a :attr:`!gi_code` attribute that refers to the original code object backing the generator. (Contributed by Collin Winter; :issue:`1473257`.) @@ -1667,17 +1669,17 @@ Some smaller changes made to the core Python language are: will now round-trip values. For example, ``complex('(3+4j)')`` now returns the value (3+4j). (:issue:`1491866`) -* The string :meth:`translate` method now accepts ``None`` as the +* The string :meth:`~str.translate` method now accepts ``None`` as the translation table parameter, which is treated as the identity transformation. This makes it easier to carry out operations that only delete characters. (Contributed by Bengt Richter and implemented by Raymond Hettinger; :issue:`1193128`.) -* The built-in :func:`dir` function now checks for a :meth:`__dir__` +* The built-in :func:`dir` function now checks for a :meth:`~object.__dir__` method on the objects it receives. This method must return a list of strings containing the names of valid attributes for the object, and lets the object control the value that :func:`dir` produces. - Objects that have :meth:`__getattr__` or :meth:`__getattribute__` + Objects that have :meth:`~object.__getattr__` or :meth:`~object.__getattribute__` methods can use this to advertise pseudo-attributes they will honor. (:issue:`1591665`) @@ -1738,8 +1740,8 @@ Optimizations one of these types. (Contributed by Neal Norwitz.) * Unicode strings now use faster code for detecting - whitespace and line breaks; this speeds up the :meth:`split` method - by about 25% and :meth:`splitlines` by 35%. + whitespace and line breaks; this speeds up the :meth:`~str.split` method + by about 25% and :meth:`~str.splitlines` by 35%. (Contributed by Antoine Pitrou.) Memory usage is reduced by using pymalloc for the Unicode string's data. @@ -1755,7 +1757,7 @@ Optimizations .. _new-26-interpreter: Interpreter Changes -------------------------------- +------------------- Two command-line options have been reserved for use by other Python implementations. The :option:`!-J` switch has been reserved for use by @@ -1799,15 +1801,15 @@ changes, or look through the Subversion logs for all the details. were applied. (Maintained by Josiah Carlson; see :issue:`1736190` for one patch.) -* The :mod:`bsddb` module also has a new maintainer, Jesús Cea Avión, and the package - is now available as a standalone package. The web page for the package is - `www.jcea.es/programacion/pybsddb.htm +* The :mod:`!bsddb` module also has a new maintainer, Jesús Cea Avión, and the + package is now available as a standalone package. The web page for the package + is `www.jcea.es/programacion/pybsddb.htm `__. The plan is to remove the package from the standard library in Python 3.0, because its pace of releases is much more frequent than Python's. - The :mod:`bsddb.dbshelve` module now uses the highest pickling protocol + The :mod:`!bsddb.dbshelve` module now uses the highest pickling protocol available, instead of restricting itself to protocol 1. (Contributed by W. Barnes.) @@ -1817,9 +1819,9 @@ changes, or look through the Subversion logs for all the details. "/cgi-bin/add.py?category=1". (Contributed by Alexandre Fiori and Nubis; :issue:`1817`.) - The :func:`parse_qs` and :func:`parse_qsl` functions have been - relocated from the :mod:`!cgi` module to the :mod:`urlparse ` module. - The versions still available in the :mod:`!cgi` module will + The :func:`~urllib.parse.parse_qs` and :func:`~urllib.parse.parse_qsl` functions + have been relocated from the :mod:`!cgi` module to the :mod:`urlparse ` + module. The versions still available in the :mod:`!cgi` module will trigger :exc:`PendingDeprecationWarning` messages in 2.6 (:issue:`600362`). @@ -1827,27 +1829,27 @@ changes, or look through the Subversion logs for all the details. contributed by Mark Dickinson and Christian Heimes. Five new functions were added: - * :func:`polar` converts a complex number to polar form, returning + * :func:`~cmath.polar` converts a complex number to polar form, returning the modulus and argument of the complex number. - * :func:`rect` does the opposite, turning a modulus, argument pair + * :func:`~cmath.rect` does the opposite, turning a modulus, argument pair back into the corresponding complex number. - * :func:`phase` returns the argument (also called the angle) of a complex + * :func:`~cmath.phase` returns the argument (also called the angle) of a complex number. - * :func:`isnan` returns True if either + * :func:`~cmath.isnan` returns True if either the real or imaginary part of its argument is a NaN. - * :func:`isinf` returns True if either the real or imaginary part of + * :func:`~cmath.isinf` returns True if either the real or imaginary part of its argument is infinite. The revisions also improved the numerical soundness of the :mod:`cmath` module. For all functions, the real and imaginary parts of the results are accurate to within a few units of least precision (ulps) whenever possible. See :issue:`1381` for the - details. The branch cuts for :func:`asinh`, :func:`atanh`: and - :func:`atan` have also been corrected. + details. The branch cuts for :func:`~cmath.asinh`, :func:`~cmath.atanh`, and + :func:`~cmath.atan` have also been corrected. The tests for the module have been greatly expanded; nearly 2000 new test cases exercise the algebraic functions. @@ -1879,14 +1881,14 @@ changes, or look through the Subversion logs for all the details. variable(id=1, name='amplitude', type='int', size=4) Several places in the standard library that returned tuples have - been modified to return :func:`namedtuple` instances. For example, - the :meth:`Decimal.as_tuple` method now returns a named tuple with - :attr:`sign`, :attr:`digits`, and :attr:`exponent` fields. + been modified to return :func:`~collections.namedtuple` instances. For example, + the :meth:`decimal.Decimal.as_tuple` method now returns a named tuple with + :attr:`!sign`, :attr:`!digits`, and :attr:`!exponent` fields. (Contributed by Raymond Hettinger.) * Another change to the :mod:`collections` module is that the - :class:`deque` type now supports an optional *maxlen* parameter; + :class:`~collections.deque` type now supports an optional *maxlen* parameter; if supplied, the deque's size will be restricted to no more than *maxlen* items. Adding more items to a full deque causes old items to be discarded. @@ -1912,7 +1914,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Arvin Schnell; :issue:`1638033`.) * A new window method in the :mod:`curses` module, - :meth:`chgat`, changes the display attributes for a certain number of + :meth:`~curses.window.chgat`, changes the display attributes for a certain number of characters on a single line. (Contributed by Fabian Kreutz.) :: @@ -1921,20 +1923,20 @@ changes, or look through the Subversion logs for all the details. # and affecting the rest of the line. stdscr.chgat(0, 21, curses.A_BOLD) - The :class:`Textbox` class in the :mod:`curses.textpad` module + The :class:`~curses.textpad.Textbox` class in the :mod:`curses.textpad` module now supports editing in insert mode as well as overwrite mode. Insert mode is enabled by supplying a true value for the *insert_mode* - parameter when creating the :class:`Textbox` instance. + parameter when creating the :class:`~curses.textpad.Textbox` instance. -* The :mod:`datetime` module's :meth:`strftime` methods now support a - ``%f`` format code that expands to the number of microseconds in the +* The :mod:`datetime` module's :meth:`~datetime.datetime.strftime` methods now + support a ``%f`` format code that expands to the number of microseconds in the object, zero-padded on the left to six places. (Contributed by Skip Montanaro; :issue:`1158`.) * The :mod:`decimal` module was updated to version 1.66 of `the General Decimal Specification `__. New features include some methods for some basic mathematical functions such as - :meth:`exp` and :meth:`log10`:: + :meth:`~decimal.Decimal.exp` and :meth:`~decimal.Decimal.log10`:: >>> Decimal(1).exp() Decimal("2.718281828459045235360287471") @@ -1943,28 +1945,29 @@ changes, or look through the Subversion logs for all the details. >>> Decimal(1000).log10() Decimal("3") - The :meth:`as_tuple` method of :class:`Decimal` objects now returns a - named tuple with :attr:`sign`, :attr:`digits`, and :attr:`exponent` fields. + The :meth:`~decimal.Decimal.as_tuple` method of :class:`~decimal.Decimal` + objects now returns a named tuple with :attr:`!sign`, :attr:`!digits`, and + :attr:`!exponent` fields. (Implemented by Facundo Batista and Mark Dickinson. Named tuple support added by Raymond Hettinger.) -* The :mod:`difflib` module's :class:`SequenceMatcher` class +* The :mod:`difflib` module's :class:`~difflib.SequenceMatcher` class now returns named tuples representing matches, - with :attr:`a`, :attr:`b`, and :attr:`size` attributes. + with :attr:`!a`, :attr:`!b`, and :attr:`!size` attributes. (Contributed by Raymond Hettinger.) * An optional ``timeout`` parameter, specifying a timeout measured in seconds, was added to the :class:`ftplib.FTP` class constructor as - well as the :meth:`connect` method. (Added by Facundo Batista.) - Also, the :class:`FTP` class's :meth:`storbinary` and - :meth:`storlines` now take an optional *callback* parameter that + well as the :meth:`~ftplib.FTP.connect` method. (Added by Facundo Batista.) + Also, the :class:`~ftplib.FTP` class's :meth:`~ftplib.FTP.storbinary` and + :meth:`~ftplib.FTP.storlines` now take an optional *callback* parameter that will be called with each block of data after the data has been sent. (Contributed by Phil Schwartz; :issue:`1221598`.) -* The :func:`reduce` built-in function is also available in the +* The :func:`!reduce` built-in function is also available in the :mod:`functools` module. In Python 3.0, the builtin has been - dropped and :func:`reduce` is only available from :mod:`functools`; + dropped and :func:`~functools.reduce` is only available from :mod:`functools`; currently there are no plans to drop the builtin in the 2.x series. (Patched by Christian Heimes; :issue:`1739906`.) @@ -1988,8 +1991,8 @@ changes, or look through the Subversion logs for all the details. Another new function, ``heappushpop(heap, item)``, pushes *item* onto *heap*, then pops off and returns the smallest item. - This is more efficient than making a call to :func:`heappush` and then - :func:`heappop`. + This is more efficient than making a call to :func:`~heapq.heappush` and then + :func:`~heapq.heappop`. :mod:`heapq` is now implemented to only use less-than comparison, instead of the less-than-or-equal comparison it previously used. @@ -2003,14 +2006,14 @@ changes, or look through the Subversion logs for all the details. Batista.) * Most of the :mod:`inspect` module's functions, such as - :func:`getmoduleinfo` and :func:`getargs`, now return named tuples. + :func:`!getmoduleinfo` and :func:`!getargs`, now return named tuples. In addition to behaving like tuples, the elements of the return value can also be accessed as attributes. (Contributed by Raymond Hettinger.) Some new functions in the module include - :func:`isgenerator`, :func:`isgeneratorfunction`, - and :func:`isabstract`. + :func:`~inspect.isgenerator`, :func:`~inspect.isgeneratorfunction`, + and :func:`~inspect.isabstract`. * The :mod:`itertools` module gained several new functions. @@ -2071,7 +2074,7 @@ changes, or look through the Subversion logs for all the details. ``itertools.chain(*iterables)`` is an existing function in :mod:`itertools` that gained a new constructor in Python 2.6. ``itertools.chain.from_iterable(iterable)`` takes a single - iterable that should return other iterables. :func:`chain` will + iterable that should return other iterables. :func:`~itertools.chain` will then return all the elements of the first iterable, then all the elements of the second, and so on. :: @@ -2080,14 +2083,15 @@ changes, or look through the Subversion logs for all the details. (All contributed by Raymond Hettinger.) -* The :mod:`logging` module's :class:`FileHandler` class - and its subclasses :class:`WatchedFileHandler`, :class:`RotatingFileHandler`, - and :class:`TimedRotatingFileHandler` now +* The :mod:`logging` module's :class:`~logging.FileHandler` class + and its subclasses :class:`~logging.handlers.WatchedFileHandler`, + :class:`~logging.handlers.RotatingFileHandler`, + and :class:`~logging.handlers.TimedRotatingFileHandler` now have an optional *delay* parameter to their constructors. If *delay* is true, opening of the log file is deferred until the first - :meth:`emit` call is made. (Contributed by Vinay Sajip.) + :meth:`~logging.FileHandler.emit` call is made. (Contributed by Vinay Sajip.) - :class:`TimedRotatingFileHandler` also has a *utc* constructor + :class:`~logging.handlers.TimedRotatingFileHandler` also has a *utc* constructor parameter. If the argument is true, UTC time will be used in determining when midnight occurs and in generating filenames; otherwise local time will be used. @@ -2116,8 +2120,8 @@ changes, or look through the Subversion logs for all the details. * :func:`~math.log1p` returns the natural logarithm of *1+x* (base *e*). - * :func:`trunc` rounds a number toward zero, returning the closest - :class:`Integral` that's between the function's argument and zero. + * :func:`~math.trunc` rounds a number toward zero, returning the closest + :class:`~numbers.Integral` that's between the function's argument and zero. Added as part of the backport of `PEP 3141's type hierarchy for numbers <#pep-3141>`__. @@ -2137,14 +2141,15 @@ changes, or look through the Subversion logs for all the details. (Contributed by Christian Heimes and Mark Dickinson.) -* :class:`~mmap.mmap` objects now have a :meth:`rfind` method that searches for a +* :class:`~mmap.mmap` objects now have a :meth:`~mmap.mmap.rfind` method that + searches for a substring beginning at the end of the string and searching - backwards. The :meth:`find` method also gained an *end* parameter + backwards. The :meth:`~mmap.mmap.find` method also gained an *end* parameter giving an index at which to stop searching. (Contributed by John Lenton.) * The :mod:`operator` module gained a - :func:`methodcaller` function that takes a name and an optional + :func:`~operator.methodcaller` function that takes a name and an optional set of arguments, returning a callable that will call the named function on any arguments passed to it. For example:: @@ -2155,7 +2160,7 @@ changes, or look through the Subversion logs for all the details. (Contributed by Georg Brandl, after a suggestion by Gregory Petrosyan.) - The :func:`attrgetter` function now accepts dotted names and performs + The :func:`~operator.attrgetter` function now accepts dotted names and performs the corresponding attribute lookups:: >>> inst_name = operator.attrgetter( @@ -2173,12 +2178,12 @@ changes, or look through the Subversion logs for all the details. the mode of a symlink. (Contributed by Georg Brandl and Christian Heimes.) - :func:`chflags` and :func:`lchflags` are wrappers for the + :func:`~os.chflags` and :func:`~os.lchflags` are wrappers for the corresponding system calls (where they're available), changing the flags set on a file. Constants for the flag values are defined in the :mod:`stat` module; some possible values include - :const:`UF_IMMUTABLE` to signal the file may not be changed and - :const:`UF_APPEND` to indicate that data can only be appended to the + :const:`~stat.UF_IMMUTABLE` to signal the file may not be changed and + :const:`~stat.UF_APPEND` to indicate that data can only be appended to the file. (Contributed by M. Levinson.) ``os.closerange(low, high)`` efficiently closes all file descriptors @@ -2186,7 +2191,7 @@ changes, or look through the Subversion logs for all the details. This function is now used by the :mod:`subprocess` module to make starting processes faster. (Contributed by Georg Brandl; :issue:`1663329`.) -* The ``os.environ`` object's :meth:`clear` method will now unset the +* The ``os.environ`` object's :meth:`!clear` method will now unset the environment variables using :func:`os.unsetenv` in addition to clearing the object's keys. (Contributed by Martin Horcicka; :issue:`1181`.) @@ -2197,7 +2202,7 @@ changes, or look through the Subversion logs for all the details. into an infinite recursion if there's a symlink that points to a parent directory. (:issue:`1273829`) -* In the :mod:`os.path` module, the :func:`splitext` function +* In the :mod:`os.path` module, the :func:`~os.path.splitext` function has been changed to not split on leading period characters. This produces better results when operating on Unix's dot-files. For example, ``os.path.splitext('.ipython')`` @@ -2224,12 +2229,12 @@ changes, or look through the Subversion logs for all the details. if no traceback is supplied. (Contributed by Facundo Batista; :issue:`1106316`.) -* The :mod:`pickletools` module now has an :func:`optimize` function +* The :mod:`pickletools` module now has an :func:`~pickletools.optimize` function that takes a string containing a pickle and removes some unused opcodes, returning a shorter pickle that contains the same data structure. (Contributed by Raymond Hettinger.) -* A :func:`get_data` function was added to the :mod:`pkgutil` +* A :func:`~pkgutil.get_data` function was added to the :mod:`pkgutil` module that returns the contents of resource files included with an installed Python package. For example:: @@ -2246,22 +2251,22 @@ changes, or look through the Subversion logs for all the details. (Contributed by Paul Moore; :issue:`2439`.) -* The :mod:`pyexpat` module's :class:`Parser` objects now allow setting - their :attr:`buffer_size` attribute to change the size of the buffer +* The :mod:`!pyexpat` module's :class:`!Parser` objects now allow setting + their :attr:`!buffer_size` attribute to change the size of the buffer used to hold character data. (Contributed by Achim Gaedke; :issue:`1137`.) -* The :mod:`Queue` module now provides queue variants that retrieve entries - in different orders. The :class:`PriorityQueue` class stores +* The :mod:`queue` module now provides queue variants that retrieve entries + in different orders. The :class:`~queue.PriorityQueue` class stores queued items in a heap and retrieves them in priority order, - and :class:`LifoQueue` retrieves the most recently added entries first, + and :class:`~queue.LifoQueue` retrieves the most recently added entries first, meaning that it behaves like a stack. (Contributed by Raymond Hettinger.) -* The :mod:`random` module's :class:`Random` objects can +* The :mod:`random` module's :class:`~random.Random` objects can now be pickled on a 32-bit system and unpickled on a 64-bit system, and vice versa. Unfortunately, this change also means - that Python 2.6's :class:`Random` objects can't be unpickled correctly + that Python 2.6's :class:`~random.Random` objects can't be unpickled correctly on earlier versions of Python. (Contributed by Shawn Ligocki; :issue:`1727780`.) @@ -2284,11 +2289,11 @@ changes, or look through the Subversion logs for all the details. (Contributed by Guido van Rossum from work for Google App Engine; :issue:`3487`.) -* The :mod:`rlcompleter` module's :meth:`Completer.complete` method +* The :mod:`rlcompleter` module's :meth:`~rlcompleter.Completer.complete` method will now ignore exceptions triggered while evaluating a name. (Fixed by Lorenz Quack; :issue:`2250`.) -* The :mod:`sched` module's :class:`scheduler` instances now +* The :mod:`sched` module's :class:`~sched.scheduler` instances now have a read-only :attr:`queue` attribute that returns the contents of the scheduler's queue, represented as a list of named tuples with the fields ``(time, priority, action, argument)``. @@ -2296,7 +2301,7 @@ changes, or look through the Subversion logs for all the details. * The :mod:`select` module now has wrapper functions for the Linux :c:func:`!epoll` and BSD :c:func:`!kqueue` system calls. - :meth:`modify` method was added to the existing :class:`poll` + :meth:`~select.poll.modify` method was added to the existing ``poll`` objects; ``pollobj.modify(fd, eventmask)`` takes a file descriptor or file object and an event mask, modifying the recorded event mask for that file. @@ -2307,8 +2312,8 @@ changes, or look through the Subversion logs for all the details. and a list of the directory's contents, and returns a list of names that will be ignored, not copied. - The :mod:`shutil` module also provides an :func:`ignore_patterns` - function for use with this new parameter. :func:`ignore_patterns` + The :mod:`shutil` module also provides an :func:`~shutil.ignore_patterns` + function for use with this new parameter. :func:`~shutil.ignore_patterns` takes an arbitrary number of glob-style patterns and returns a callable that will ignore any files and directories that match any of these patterns. The following example copies a directory tree, @@ -2332,7 +2337,7 @@ changes, or look through the Subversion logs for all the details. Event loops will use this by opening a pipe to create two descriptors, one for reading and one for writing. The writable descriptor - will be passed to :func:`set_wakeup_fd`, and the readable descriptor + will be passed to :func:`~signal.set_wakeup_fd`, and the readable descriptor will be added to the list of descriptors monitored by the event loop via :c:func:`!select` or :c:func:`!poll`. On receiving a signal, a byte will be written and the main event loop @@ -2340,20 +2345,20 @@ changes, or look through the Subversion logs for all the details. (Contributed by Adam Olsen; :issue:`1583`.) - The :func:`siginterrupt` function is now available from Python code, + The :func:`~signal.siginterrupt` function is now available from Python code, and allows changing whether signals can interrupt system calls or not. (Contributed by Ralf Schmitt.) - The :func:`setitimer` and :func:`getitimer` functions have also been - added (where they're available). :func:`setitimer` + The :func:`~signal.setitimer` and :func:`~signal.getitimer` functions have also been + added (where they're available). :func:`~signal.setitimer` allows setting interval timers that will cause a signal to be delivered to the process after a specified time, measured in wall-clock time, consumed process time, or combined process+system time. (Contributed by Guilherme Polo; :issue:`2240`.) * The :mod:`smtplib` module now supports SMTP over SSL thanks to the - addition of the :class:`SMTP_SSL` class. This class supports an - interface identical to the existing :class:`SMTP` class. + addition of the :class:`~smtplib.SMTP_SSL` class. This class supports an + interface identical to the existing :class:`~smtplib.SMTP` class. (Contributed by Monty Taylor.) Both class constructors also have an optional ``timeout`` parameter that specifies a timeout for the initial connection attempt, measured in seconds. (Contributed by @@ -2364,7 +2369,7 @@ changes, or look through the Subversion logs for all the details. e-mail between agents that don't manage a mail queue. (LMTP implemented by Leif Hedstrom; :issue:`957003`.) - :meth:`SMTP.starttls` now complies with :rfc:`3207` and forgets any + :meth:`smtplib.SMTP.starttls` now complies with :rfc:`3207` and forgets any knowledge obtained from the server not obtained from the TLS negotiation itself. (Patch contributed by Bill Fenner; :issue:`829951`.) @@ -2374,11 +2379,11 @@ changes, or look through the Subversion logs for all the details. environments. TIPC addresses are 4- or 5-tuples. (Contributed by Alberto Bertogli; :issue:`1646`.) - A new function, :func:`create_connection`, takes an address and + A new function, :func:`~socket.create_connection`, takes an address and connects to it using an optional timeout value, returning the connected socket object. This function also looks up the address's type and connects to it using IPv4 or IPv6 as appropriate. Changing - your code to use :func:`create_connection` instead of + your code to use :func:`~socket.create_connection` instead of ``socket(socket.AF_INET, ...)`` may be all that's required to make your code work with IPv6. @@ -2406,15 +2411,15 @@ changes, or look through the Subversion logs for all the details. :c:func:`!TerminateProcess`. (Contributed by Christian Heimes.) -* A new variable in the :mod:`sys` module, :attr:`float_info`, is an +* A new variable in the :mod:`sys` module, :data:`~sys.float_info`, is an object containing information derived from the :file:`float.h` file about the platform's floating-point support. Attributes of this - object include :attr:`mant_dig` (number of digits in the mantissa), - :attr:`epsilon` (smallest difference between 1.0 and the next + object include :attr:`~sys.float_info.mant_dig` (number of digits in the mantissa), + :attr:`~sys.float_info.epsilon` (smallest difference between 1.0 and the next largest value representable), and several others. (Contributed by Christian Heimes; :issue:`1534`.) - Another new variable, :attr:`dont_write_bytecode`, controls whether Python + Another new variable, :data:`~sys.dont_write_bytecode`, controls whether Python writes any :file:`.pyc` or :file:`.pyo` files on importing a module. If this variable is true, the compiled files are not written. The variable is initially set on start-up by supplying the :option:`-B` @@ -2427,16 +2432,16 @@ changes, or look through the Subversion logs for all the details. Information about the command-line arguments supplied to the Python interpreter is available by reading attributes of a named - tuple available as ``sys.flags``. For example, the :attr:`verbose` + tuple available as ``sys.flags``. For example, the :attr:`~sys.flags.verbose` attribute is true if Python - was executed in verbose mode, :attr:`debug` is true in debugging mode, etc. + was executed in verbose mode, :attr:`~sys.flags.debug` is true in debugging mode, etc. These attributes are all read-only. (Contributed by Christian Heimes.) - A new function, :func:`getsizeof`, takes a Python object and returns + A new function, :func:`~sys.getsizeof`, takes a Python object and returns the amount of memory used by the object, measured in bytes. Built-in objects return correct results; third-party extensions may not, - but can define a :meth:`__sizeof__` method to return the + but can define a :meth:`!__sizeof__` method to return the object's size. (Contributed by Robert Schuppenies; :issue:`2898`.) @@ -2460,12 +2465,12 @@ changes, or look through the Subversion logs for all the details. their UTF-8 representation. (Character conversions occur because the PAX format supports Unicode filenames, defaulting to UTF-8 encoding.) - The :meth:`TarFile.add` method now accepts an ``exclude`` argument that's + The :meth:`tarfile.TarFile.add` method now accepts an ``exclude`` argument that's a function that can be used to exclude certain filenames from an archive. The function must take a filename and return true if the file should be excluded or false if it should be archived. - The function is applied to both the name initially passed to :meth:`add` + The function is applied to both the name initially passed to :meth:`~tarfile.TarFile.add` and to the names of files in recursively added directories. (All changes contributed by Lars Gustäbel). @@ -2479,12 +2484,13 @@ changes, or look through the Subversion logs for all the details. behaviour can now be changed by passing ``delete=False`` to the constructor. (Contributed by Damien Miller; :issue:`1537850`.) - A new class, :class:`SpooledTemporaryFile`, behaves like + A new class, :class:`~tempfile.SpooledTemporaryFile`, behaves like a temporary file but stores its data in memory until a maximum size is exceeded. On reaching that limit, the contents will be written to an on-disk temporary file. (Contributed by Dustin J. Mitchell.) - The :class:`NamedTemporaryFile` and :class:`SpooledTemporaryFile` classes + The :class:`~tempfile.NamedTemporaryFile` and + :class:`~tempfile.SpooledTemporaryFile` classes both work as context managers, so you can write ``with tempfile.NamedTemporaryFile() as tmp: ...``. (Contributed by Alexander Belopolsky; :issue:`2021`.) @@ -2495,7 +2501,7 @@ changes, or look through the Subversion logs for all the details. context manager that temporarily changes environment variables and automatically restores them to their old values. - Another context manager, :class:`TransientResource`, can surround calls + Another context manager, :class:`!TransientResource`, can surround calls to resources that may or may not be available; it will catch and ignore a specified list of exceptions. For example, a network test may ignore certain failures when connecting to an @@ -2506,9 +2512,9 @@ changes, or look through the Subversion logs for all the details. f = urllib.urlopen('https://sf.net') ... - Finally, :func:`check_warnings` resets the :mod:`warning` module's - warning filters and returns an object that will record all warning - messages triggered (:issue:`3781`):: + Finally, :func:`~test.support.warnings_helper.check_warnings` resets the + :mod:`warnings` module's warning filters and returns an object that will record + all warning messages triggered (:issue:`3781`):: with test_support.check_warnings() as wrec: warnings.simplefilter("always") @@ -2540,31 +2546,31 @@ changes, or look through the Subversion logs for all the details. (Contributed by Dwayne Bailey; :issue:`1581073`.) * The :mod:`threading` module API is being changed to use properties - such as :attr:`daemon` instead of :meth:`setDaemon` and - :meth:`isDaemon` methods, and some methods have been renamed to use + such as :attr:`~threading.Thread.daemon` instead of :meth:`!setDaemon` and + :meth:`!isDaemon` methods, and some methods have been renamed to use underscores instead of camel-case; for example, the - :meth:`activeCount` method is renamed to :meth:`active_count`. Both + :meth:`!activeCount` method is renamed to :func:`~threading.active_count`. Both the 2.6 and 3.0 versions of the module support the same properties and renamed methods, but don't remove the old methods. No date has been set for the deprecation of the old APIs in Python 3.x; the old APIs won't be removed in any 2.x version. (Carried out by several people, most notably Benjamin Peterson.) - The :mod:`threading` module's :class:`Thread` objects - gained an :attr:`ident` property that returns the thread's + The :mod:`threading` module's :class:`~threading.Thread` objects + gained an :attr:`~threading.Thread.ident` property that returns the thread's identifier, a nonzero integer. (Contributed by Gregory P. Smith; :issue:`2871`.) * The :mod:`timeit` module now accepts callables as well as strings for the statement being timed and for the setup code. Two convenience functions were added for creating - :class:`Timer` instances: + :class:`~timeit.Timer` instances: ``repeat(stmt, setup, time, repeat, number)`` and ``timeit(stmt, setup, time, number)`` create an instance and call the corresponding method. (Contributed by Erik Demaine; :issue:`1533909`.) -* The :mod:`Tkinter` module now accepts lists and tuples for options, +* The :mod:`tkinter` module now accepts lists and tuples for options, separating the elements by spaces before passing the resulting value to Tcl/Tk. (Contributed by Guilherme Polo; :issue:`2906`.) @@ -2573,11 +2579,11 @@ changes, or look through the Subversion logs for all the details. Gregor Lingl. New features in the module include: * Better animation of turtle movement and rotation. - * Control over turtle movement using the new :meth:`delay`, - :meth:`tracer`, and :meth:`speed` methods. + * Control over turtle movement using the new :func:`~turtle.delay`, + :func:`~turtle.tracer`, and :func:`~turtle.speed` methods. * The ability to set new shapes for the turtle, and to define a new coordinate system. - * Turtles now have an :meth:`undo` method that can roll back actions. + * Turtles now have an :func:`~turtle.undo` method that can roll back actions. * Simple support for reacting to input events such as mouse and keyboard activity, making it possible to write simple games. * A :file:`turtle.cfg` file can be used to customize the starting appearance @@ -2589,7 +2595,7 @@ changes, or look through the Subversion logs for all the details. * An optional ``timeout`` parameter was added to the :func:`urllib.urlopen ` function and the - :class:`urllib.ftpwrapper` class constructor, as well as the + :class:`!urllib.ftpwrapper` class constructor, as well as the :func:`urllib2.urlopen ` function. The parameter specifies a timeout measured in seconds. For example:: @@ -2606,12 +2612,13 @@ changes, or look through the Subversion logs for all the details. has been updated to version 5.1.0. (Updated by Martin von Löwis; :issue:`3811`.) -* The :mod:`warnings` module's :func:`formatwarning` and :func:`showwarning` +* The :mod:`warnings` module's :func:`~warnings.formatwarning` and + :func:`~warnings.showwarning` gained an optional *line* argument that can be used to supply the line of source code. (Added as part of :issue:`1631171`, which re-implemented part of the :mod:`warnings` module in C code.) - A new function, :func:`catch_warnings`, is a context manager + A new function, :func:`~warnings.catch_warnings`, is a context manager intended for testing purposes that lets you temporarily modify the warning filters and then restore their original values (:issue:`3781`). @@ -2619,12 +2626,13 @@ changes, or look through the Subversion logs for all the details. classes can now be prevented from immediately opening and binding to their socket by passing ``False`` as the *bind_and_activate* constructor parameter. This can be used to modify the instance's - :attr:`allow_reuse_address` attribute before calling the - :meth:`server_bind` and :meth:`server_activate` methods to + :attr:`~socketserver.BaseServer.allow_reuse_address` attribute before calling the + :meth:`~socketserver.BaseServer.server_bind` and + :meth:`~socketserver.BaseServer.server_activate` methods to open the socket and begin listening for connections. (Contributed by Peter Parente; :issue:`1599845`.) - :class:`SimpleXMLRPCServer` also has a :attr:`_send_traceback_header` + :class:`~xmlrpc.server.SimpleXMLRPCServer` also has a :attr:`!_send_traceback_header` attribute; if true, the exception and formatted traceback are returned as HTTP headers "X-Exception" and "X-Traceback". This feature is for debugging purposes only and should not be used on production servers @@ -2636,14 +2644,15 @@ changes, or look through the Subversion logs for all the details. :class:`datetime.date` and :class:`datetime.time` to the :class:`xmlrpclib.DateTime ` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`~datetime.date` and :class:`~datetime.time` instances. (:issue:`1330538`) The code can also handle dates before 1900 (contributed by Ralf Schmitt; :issue:`2014`) and 64-bit integers represented by using ```` in XML-RPC responses (contributed by Riku Lindblad; :issue:`2985`). -* The :mod:`zipfile` module's :class:`ZipFile` class now has - :meth:`extract` and :meth:`extractall` methods that will unpack +* The :mod:`zipfile` module's :class:`~zipfile.ZipFile` class now has + :meth:`~zipfile.ZipFile.extract` and :meth:`~zipfile.ZipFile.extractall` + methods that will unpack a single file or all the files in the archive to the current directory, or to a specified directory:: @@ -2658,8 +2667,9 @@ changes, or look through the Subversion logs for all the details. (Contributed by Alan McIntyre; :issue:`467924`.) - The :meth:`open`, :meth:`read` and :meth:`extract` methods can now - take either a filename or a :class:`ZipInfo` object. This is useful when an + The :meth:`~zipfile.ZipFile.open`, :meth:`~zipfile.ZipFile.read` and + :meth:`~zipfile.ZipFile.extract` methods can now + take either a filename or a :class:`~zipfile.ZipInfo` object. This is useful when an archive accidentally contains a duplicated filename. (Contributed by Graham Horler; :issue:`1775025`.) @@ -2670,7 +2680,7 @@ changes, or look through the Subversion logs for all the details. .. whole new modules get described in subsections here The :mod:`ast` module ----------------------- +--------------------- The :mod:`ast` module provides an Abstract Syntax Tree representation of Python code, and Armin Ronacher @@ -2679,8 +2689,8 @@ common tasks. These will be useful for HTML templating packages, code analyzers, and similar tools that process Python code. -The :func:`parse` function takes an expression and returns an AST. -The :func:`dump` function outputs a representation of a tree, suitable +The :func:`~ast.parse` function takes an expression and returns an AST. +The :func:`~ast.dump` function outputs a representation of a tree, suitable for debugging:: import ast @@ -2726,13 +2736,13 @@ This outputs a deeply nested tree:: ], nl=True) ]) -The :func:`literal_eval` method takes a string or an AST +The :func:`~ast.literal_eval` method takes a string or an AST representing a literal expression, parses and evaluates it, and returns the resulting value. A literal expression is a Python expression containing only strings, numbers, dictionaries, etc. but no statements or function calls. If you need to evaluate an expression but cannot accept the security risk of using an -:func:`eval` call, :func:`literal_eval` will handle it safely:: +:func:`eval` call, :func:`~ast.literal_eval` will handle it safely:: >>> literal = '("a", "b", {2:4, 3:8, 1:2})' >>> print ast.literal_eval(literal) @@ -2742,15 +2752,15 @@ evaluate an expression but cannot accept the security risk of using an ... ValueError: malformed string -The module also includes :class:`NodeVisitor` and -:class:`NodeTransformer` classes for traversing and modifying an AST, +The module also includes :class:`~ast.NodeVisitor` and +:class:`~ast.NodeTransformer` classes for traversing and modifying an AST, and functions for common transformations such as changing line numbers. .. ====================================================================== The :mod:`!future_builtins` module --------------------------------------- +---------------------------------- Python 3.0 makes many changes to the repertoire of built-in functions, and most of the changes can't be introduced in the Python @@ -2770,15 +2780,15 @@ The functions in this module currently include: return iterators, unlike the 2.x builtins which return lists. * ``hex(value)``, ``oct(value)``: instead of calling the - :meth:`__hex__` or :meth:`__oct__` methods, these versions will - call the :meth:`__index__` method and convert the result to hexadecimal + :meth:`!__hex__` or :meth:`!__oct__` methods, these versions will + call the :meth:`~object.__index__` method and convert the result to hexadecimal or octal. :func:`oct` will use the new ``0o`` notation for its result. .. ====================================================================== The :mod:`json` module: JavaScript Object Notation --------------------------------------------------------------------- +-------------------------------------------------- The new :mod:`json` module supports the encoding and decoding of Python types in JSON (Javascript Object Notation). JSON is a lightweight interchange format @@ -2845,12 +2855,12 @@ Using the module is simple:: .. ====================================================================== ctypes Enhancements --------------------------------------------------- +------------------- Thomas Heller continued to maintain and enhance the :mod:`ctypes` module. -:mod:`ctypes` now supports a :class:`c_bool` datatype +:mod:`ctypes` now supports a :class:`~ctypes.c_bool` datatype that represents the C99 ``bool`` type. (Contributed by David Remahl; :issue:`1649190`.) @@ -2862,11 +2872,11 @@ where various combinations of ``(start, stop, step)`` are supplied. .. Revision 57769 All :mod:`ctypes` data types now support -:meth:`from_buffer` and :meth:`from_buffer_copy` +:meth:`~ctypes._CData.from_buffer` and :meth:`~ctypes._CData.from_buffer_copy` methods that create a ctypes instance based on a -provided buffer object. :meth:`from_buffer_copy` copies +provided buffer object. :meth:`~ctypes._CData.from_buffer_copy` copies the contents of the object, -while :meth:`from_buffer` will share the same memory area. +while :meth:`~ctypes._CData.from_buffer` will share the same memory area. A new calling convention tells :mod:`ctypes` to clear the ``errno`` or Win32 LastError variables at the outset of each wrapped call. @@ -2874,24 +2884,24 @@ Win32 LastError variables at the outset of each wrapped call. You can now retrieve the Unix ``errno`` variable after a function call. When creating a wrapped function, you can supply -``use_errno=True`` as a keyword parameter to the :func:`DLL` function -and then call the module-level methods :meth:`set_errno` and -:meth:`get_errno` to set and retrieve the error value. +``use_errno=True`` as a keyword parameter to the ``DLL`` function +and then call the module-level methods :func:`~ctypes.set_errno` and +:func:`~ctypes.get_errno` to set and retrieve the error value. The Win32 LastError variable is similarly supported by -the :func:`DLL`, :func:`OleDLL`, and :func:`WinDLL` functions. +the ``DLL``, :func:`~ctypes.OleDLL`, and :func:`~ctypes.WinDLL` functions. You supply ``use_last_error=True`` as a keyword parameter -and then call the module-level methods :meth:`set_last_error` -and :meth:`get_last_error`. +and then call the module-level methods :func:`~ctypes.set_last_error` +and :func:`~ctypes.get_last_error`. -The :func:`byref` function, used to retrieve a pointer to a ctypes +The :func:`~ctypes.byref` function, used to retrieve a pointer to a ctypes instance, now has an optional *offset* parameter that is a byte count that will be added to the returned pointer. .. ====================================================================== Improved SSL Support --------------------------------------------------- +-------------------- Bill Janssen made extensive improvements to Python 2.6's support for the Secure Sockets Layer by adding a new module, :mod:`ssl`, that's @@ -2903,9 +2913,9 @@ in the :mod:`socket` module hasn't been removed and continues to work, though it will be removed in Python 3.0. To use the new module, you must first create a TCP connection in the -usual way and then pass it to the :func:`ssl.wrap_socket` function. +usual way and then pass it to the :func:`!ssl.wrap_socket` function. It's possible to specify whether a certificate is required, and to -obtain certificate info by calling the :meth:`getpeercert` method. +obtain certificate info by calling the :meth:`~ssl.SSLSocket.getpeercert` method. .. seealso:: @@ -3031,7 +3041,7 @@ Changes to Python's build process and to the C API include: * The BerkeleyDB module now has a C API object, available as ``bsddb.db.api``. This object can be used by other C extensions - that wish to use the :mod:`bsddb` module for their own purposes. + that wish to use the :mod:`!bsddb` module for their own purposes. (Contributed by Duncan Grisby.) * The new buffer interface, previously described in @@ -3073,7 +3083,7 @@ Changes to Python's build process and to the C API include: * C functions and methods that use :c:func:`PyComplex_AsCComplex` will now accept arguments that - have a :meth:`__complex__` method. In particular, the functions in the + have a :meth:`~object.__complex__` method. In particular, the functions in the :mod:`cmath` module will now accept objects with this method. This is a backport of a Python 3.0 change. (Contributed by Mark Dickinson; :issue:`1675423`.) @@ -3126,7 +3136,7 @@ Changes to Python's build process and to the C API include: .. ====================================================================== Port-Specific Changes: Windows ------------------------------------ +------------------------------ * The support for Windows 95, 98, ME and NT4 has been dropped. Python 2.6 requires at least Windows 2000 SP4. @@ -3152,7 +3162,7 @@ Port-Specific Changes: Windows * The :mod:`socket` module's socket objects now have an :meth:`~socket.socket.ioctl` method that provides a limited interface to the - :c:func:`WSAIoctl` system interface. + :c:func:`!WSAIoctl` system interface. * The :mod:`_winreg ` module now has a function, :func:`~winreg.ExpandEnvironmentStrings`, @@ -3175,7 +3185,7 @@ Port-Specific Changes: Windows .. ====================================================================== Port-Specific Changes: Mac OS X ------------------------------------ +------------------------------- * When compiling a framework build of Python, you can now specify the framework name to be used by providing the @@ -3229,7 +3239,7 @@ Port-Specific Changes: Mac OS X .. ====================================================================== Port-Specific Changes: IRIX ------------------------------------ +--------------------------- A number of old IRIX-specific modules were deprecated and will be removed in Python 3.0: @@ -3273,7 +3283,7 @@ that may require changes to your code: * String exceptions have been removed. Attempting to use them raises a :exc:`TypeError`. -* The :meth:`__init__` method of :class:`collections.deque` +* The :meth:`!__init__` method of :class:`collections.deque` now clears any existing contents of the deque before adding elements from the iterable. This change makes the behavior match ``list.__init__()``. @@ -3281,16 +3291,16 @@ that may require changes to your code: * :meth:`object.__init__` previously accepted arbitrary arguments and keyword arguments, ignoring them. In Python 2.6, this is no longer allowed and will result in a :exc:`TypeError`. This will affect - :meth:`__init__` methods that end up calling the corresponding + :meth:`!__init__` methods that end up calling the corresponding method on :class:`object` (perhaps through using :func:`super`). See :issue:`1683368` for discussion. -* The :class:`Decimal` constructor now accepts leading and trailing +* The :class:`~decimal.Decimal` constructor now accepts leading and trailing whitespace when passed a string. Previously it would raise an - :exc:`InvalidOperation` exception. On the other hand, the - :meth:`create_decimal` method of :class:`Context` objects now + :exc:`~decimal.InvalidOperation` exception. On the other hand, the + :meth:`~decimal.Context.create_decimal` method of :class:`~decimal.Context` objects now explicitly disallows extra whitespace, raising a - :exc:`ConversionSyntax` exception. + :exc:`!ConversionSyntax` exception. * Due to an implementation accident, if you passed a file path to the built-in :func:`__import__` function, it would actually import @@ -3308,14 +3318,14 @@ that may require changes to your code: * The :mod:`socket` module exception :exc:`socket.error` now inherits from :exc:`IOError`. Previously it wasn't a subclass of - :exc:`StandardError` but now it is, through :exc:`IOError`. + :exc:`!StandardError` but now it is, through :exc:`IOError`. (Implemented by Gregory P. Smith; :issue:`1706815`.) * The :mod:`xmlrpclib ` module no longer automatically converts :class:`datetime.date` and :class:`datetime.time` to the :class:`xmlrpclib.DateTime ` type; the conversion semantics were not necessarily correct for all applications. Code using - :mod:`!xmlrpclib` should convert :class:`date` and :class:`~datetime.time` + :mod:`!xmlrpclib` should convert :class:`~datetime.date` and :class:`~datetime.time` instances. (:issue:`1330538`) * (3.0-warning mode) The :class:`Exception` class now warns diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 7296296d144803d..de90c37a38d38ff 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1267,7 +1267,7 @@ changes, or look through the Subversion logs for all the details. uploads thanks to an added *rest* parameter (patch by Pablo Mouzo; :issue:`6845`.) -* New class decorator: :func:`~functools.total_ordering` in the :mod:`functools` +* New class decorator: :deco:`~functools.total_ordering` in the :mod:`functools` module takes a class that defines an :meth:`~object.__eq__` method and one of :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, and generates the missing comparison methods. Since the diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 4b092b139595304..390b82d5a1fdc1b 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -402,7 +402,7 @@ Tracing events, with the correct line number, are generated for all lines of cod The :attr:`~frame.f_lineno` attribute of frame objects will always contain the expected line number. -The :attr:`~codeobject.co_lnotab` attribute of +The :attr:`!codeobject.co_lnotab` attribute of :ref:`code objects ` is deprecated and will be removed in 3.12. Code that needs to convert from offset to line number should use the new @@ -996,7 +996,7 @@ dataclasses __slots__ ~~~~~~~~~ -Added ``slots`` parameter in :func:`dataclasses.dataclass` decorator. +Added ``slots`` parameter in :deco:`dataclasses.dataclass` decorator. (Contributed by Yurii Karabas in :issue:`42269`) Keyword-only fields @@ -1446,8 +1446,8 @@ as a positional-only argument. types ----- -Reintroduce the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set +Reintroduce the :class:`types.EllipsisType`, :class:`types.NoneType` +and :class:`types.NotImplementedType` classes, providing a new set of types readily interpretable by type checkers. (Contributed by Bas van Beek in :issue:`41810`.) @@ -1485,9 +1485,9 @@ is a :class:`typing.TypedDict`. Subclasses of ``typing.Protocol`` which only have data variables declared will now raise a ``TypeError`` when checked with ``isinstance`` unless they -are decorated with :func:`~typing.runtime_checkable`. Previously, these checks +are decorated with :deco:`~typing.runtime_checkable`. Previously, these checks passed silently. Users should decorate their -subclasses with the :func:`!runtime_checkable` decorator +subclasses with the :deco:`!runtime_checkable` decorator if they want runtime protocols. (Contributed by Yurii Karabas in :issue:`38908`.) @@ -1928,7 +1928,7 @@ Changes in the Python API (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley in :issue:`42392`.) -* The :data:`types.FunctionType` constructor now inherits the current builtins +* The :class:`types.FunctionType` constructor now inherits the current builtins if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index a095d887352127a..4adb5cbd1b7d3b9 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -388,7 +388,7 @@ Kumar Srinivasan and Graham Bleaney.) PEP 681: Data class transforms ------------------------------ -:data:`~typing.dataclass_transform` may be used to +:deco:`~typing.dataclass_transform` may be used to decorate a class, metaclass, or a function that is itself a decorator. The presence of ``@dataclass_transform()`` tells a static type checker that the decorated object performs runtime "magic" that transforms a class, @@ -522,7 +522,7 @@ Other CPython Implementation Changes have been removed, as they can be derived from ``exc_value``. (Contributed by Irit Katriel in :issue:`45711`.) -* A new :ref:`command line option `, ``AppendPath``, +* A new command line option, ``AppendPath``, has been added for the Windows installer. It behaves similarly to ``PrependPath``, but appends the install and scripts directories instead of prepending them. @@ -686,8 +686,8 @@ enum * Added the :func:`~enum.member` and :func:`~enum.nonmember` decorators, to ensure the decorated object is/is not converted to an enum member. -* Added the :func:`~enum.property` decorator, - which works like :func:`property` except for enums. +* Added the :deco:`~enum.property` decorator, + which works like :deco:`property` except for enums. Use this instead of :func:`types.DynamicClassAttribute`. * Added the :func:`~enum.global_enum` enum decorator, @@ -740,7 +740,7 @@ fractions functools --------- -* :func:`functools.singledispatch` now supports :class:`types.UnionType` +* :deco:`functools.singledispatch` now supports :class:`types.UnionType` and :class:`typing.Union` as annotations to the dispatch argument.:: >>> from functools import singledispatch @@ -1164,7 +1164,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. type checker errors related to highly dynamic class, such as mocks. (Contributed by Shantanu Jain in :gh:`91154`.) -* The :func:`typing.final` decorator now sets the ``__final__`` attributed on +* The :deco:`typing.final` decorator now sets the ``__final__`` attributed on the decorated object. (Contributed by Jelle Zijlstra in :gh:`90500`.) @@ -1197,7 +1197,7 @@ For major changes, see :ref:`new-feat-related-type-hints-311`. :data:`~typing.ClassVar` annotations. (Contributed by Gregory Beauregard in :gh:`90711`.) -* :func:`typing.no_type_check` no longer modifies external classes and functions. +* :deco:`typing.no_type_check` no longer modifies external classes and functions. It also now correctly marks classmethods as not to be type checked. (Contributed by Nikita Sobolev in :gh:`90729`.) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 221956f3dd38195..9e48b3ab7393c87 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -146,7 +146,7 @@ New typing features: * :ref:`PEP 692 `, using :class:`~typing.TypedDict` to annotate :term:`**kwargs ` -* :ref:`PEP 698 `, :func:`typing.override` decorator +* :ref:`PEP 698 `, :deco:`typing.override` decorator Important deprecations, removals or restrictions: @@ -511,7 +511,7 @@ See :pep:`692` for more details. PEP 698: Override Decorator for Static Typing --------------------------------------------- -A new decorator :func:`typing.override` has been added to the :mod:`typing` +A new decorator :deco:`typing.override` has been added to the :mod:`typing` module. It indicates to type checkers that the method is intended to override a method in a superclass. This allows type checkers to catch mistakes where a method that is intended to override something in a base class @@ -1021,7 +1021,7 @@ typing ``__orig_bases__`` attribute. (Contributed by Adrian Garcia Badaracco in :gh:`103699`.) -* Add ``frozen_default`` parameter to :func:`typing.dataclass_transform`. +* Add ``frozen_default`` parameter to :deco:`typing.dataclass_transform`. (Contributed by Erik De Bonte in :gh:`99957`.) unicodedata @@ -1347,7 +1347,7 @@ Deprecated ``int``, convert to int explicitly: ``~int(x)``. (Contributed by Tim Hoffmann in :gh:`103487`.) -* Accessing :attr:`~codeobject.co_lnotab` on code objects was deprecated in +* Accessing :attr:`!codeobject.co_lnotab` on code objects was deprecated in Python 3.10 via :pep:`626`, but it only got a proper :exc:`DeprecationWarning` in 3.12. May be removed in 3.15. @@ -1367,6 +1367,8 @@ Deprecated .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst .. _whatsnew312-removed: @@ -1747,7 +1749,7 @@ Changes in the Python API around process-global resources, which are best managed from the main interpreter. (Contributed by Donghee Na in :gh:`99127`.) -* The undocumented locking behavior of :func:`~functools.cached_property` +* The undocumented locking behavior of :deco:`~functools.cached_property` is removed, because it locked across all instances of the class, leading to high lock contention. This means that a cached property getter function could now run more than once for a single instance, if two threads race. For most simple diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index f4489bfa1b74e85..d8845fe921ededa 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1958,14 +1958,14 @@ New Deprecations * :mod:`sys`: - * Deprecate the :func:`~sys._enablelegacywindowsfsencoding` function, + * Deprecate the :func:`!_enablelegacywindowsfsencoding` function, to be removed in Python 3.16. Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. (Contributed by Inada Naoki in :gh:`73427`.) * :mod:`tarfile`: - * Deprecate the undocumented and unused :attr:`!TarFile.tarfile` attribute, + * Deprecate the undocumented and unused :attr:`!TarInfo.tarfile` attribute, to be removed in Python 3.16. (Contributed in :gh:`115256`.) @@ -1994,7 +1994,7 @@ New Deprecations use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})``. (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) - * Deprecate the :func:`!typing.no_type_check_decorator` decorator function, + * Deprecate the :deco:`!typing.no_type_check_decorator` decorator function, to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. @@ -2031,6 +2031,8 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2597,7 +2599,7 @@ Build Changes .. _mimalloc library: https://github.com/microsoft/mimalloc/ -* The :file:`configure` option :option:`--with-system-libmpdec` +* The :file:`configure` option :option:`!--with-system-libmpdec` now defaults to ``yes``. The bundled copy of ``libmpdec`` will be removed in Python 3.16. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index dfdfe66be7e6cc2..6e3e6331a1b7e88 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -648,7 +648,7 @@ Improved error messages :pep:`784`: Zstandard support in the standard library ----------------------------------------------------- -The new :mod:`!compression` package contains modules :mod:`!compression.lzma`, +The new :mod:`compression` package contains modules :mod:`!compression.lzma`, :mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` modules respectively. The new import names under :mod:`!compression` are the @@ -657,7 +657,7 @@ the existing modules names have not been deprecated. Any deprecation or removal of the existing compression modules will occur no sooner than five years after the release of 3.14. -The new :mod:`!compression.zstd` module provides compression and decompression +The new :mod:`compression.zstd` module provides compression and decompression APIs for the Zstandard format via bindings to `Meta's zstd library `__. Zstandard is a widely adopted, highly efficient, and fast compression format. In addition to the APIs introduced in @@ -953,10 +953,24 @@ when a module is imported) will still emit the syntax warning. (Contributed by Irit Katriel in :gh:`130080`.) +.. _incremental-garbage-collection: .. _whatsnew314-incremental-gc: -Incremental garbage collection ------------------------------- +Garbage collection +------------------ + +**From Python 3.14.5 onwards:** + +The garbage collector (GC) has changed in Python 3.14.5. + +Python 3.14.0-3.14.4 shipped with a new incremental GC. +However, due to a number of `reports +`__ +of significant memory pressure in production environments, +it has been reverted back to the generational GC from 3.13. +This is the GC now used in Python 3.14.5 and later. + +**Previously in Python 3.14.0-3.14.4:** The cycle garbage collector is now incremental. This means that maximum pause times are reduced @@ -2203,7 +2217,18 @@ difflib gc -- -* The new :ref:`incremental garbage collector ` +* **From Python 3.14.5 onwards:** + + Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later. + +* **Previously in Python 3.14.0-3.14.4:** + + The new :ref:`incremental garbage collector ` means that maximum pause times are reduced by an order of magnitude or more for larger heaps. @@ -2603,12 +2628,12 @@ New deprecations and will be removed in Python 3.16. In particular, the following classes and functions are deprecated: - * :class:`asyncio.AbstractEventLoopPolicy` - * :class:`asyncio.DefaultEventLoopPolicy` - * :class:`asyncio.WindowsSelectorEventLoopPolicy` - * :class:`asyncio.WindowsProactorEventLoopPolicy` - * :func:`asyncio.get_event_loop_policy` - * :func:`asyncio.set_event_loop_policy` + * :class:`!asyncio.AbstractEventLoopPolicy` + * :class:`!asyncio.DefaultEventLoopPolicy` + * :class:`!asyncio.WindowsSelectorEventLoopPolicy` + * :class:`!asyncio.WindowsProactorEventLoopPolicy` + * :func:`!asyncio.get_event_loop_policy` + * :func:`!asyncio.set_event_loop_policy` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with the *loop_factory* argument to use the desired event loop implementation. @@ -2693,7 +2718,7 @@ New deprecations (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) * :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest, + Deprecate :meth:`!symtable.Class.get_methods` due to the lack of interest, scheduled for removal in Python 3.16. (Contributed by Bénédikt Tran in :gh:`119698`.) @@ -2723,6 +2748,8 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst @@ -3285,6 +3312,16 @@ Changes in the Python API This temporary change affects other threads. (Contributed by Serhiy Storchaka in :gh:`69998`.) +* :func:`pickle.dump` and :func:`pickle.dumps` now raise + :exc:`~pickle.PicklingError` for some failures that previously raised + :exc:`AttributeError`, :exc:`ImportError`, :exc:`ValueError`, + :exc:`UnicodeEncodeError` or :exc:`!PicklingError`, + depending on the implementation + (for example, pickling a local object, or an object whose module cannot be imported). + The original exception is chained to the :exc:`!PicklingError`. + Code that caught these exceptions should also catch :exc:`!PicklingError`. + (Contributed by Serhiy Storchaka in :gh:`122311`.) + * :class:`types.UnionType` is now an alias for :class:`typing.Union`, causing changes in some behaviors. See :ref:`above ` for more details. @@ -3447,3 +3484,17 @@ Changes in the C API functions on Python 3.13 and older. .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ + + +Notable changes in 3.14.5 +========================= + +gc +-- + +* The incremental garbage collector shipped in Python 3.14.0-3.14.4 has been + reverted back to the generational garbage collector from 3.13, + due to a number of `reports + `__ + of significant memory pressure in production environments. + See :ref:`whatsnew314-incremental-gc` for details. diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 21327b611120a2e..bf9aceaa20b2b47 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -69,23 +69,35 @@ Summary -- Release highlights ` * :pep:`814`: :ref:`Add frozendict built-in type ` +* :pep:`661`: :ref:`Add sentinel built-in type + ` * :pep:`799`: :ref:`A dedicated profiling package for organizing Python profiling tools ` * :pep:`799`: :ref:`Tachyon: High frequency statistical sampling profiler ` +* :pep:`831`: :ref:`Frame pointers are enabled by default for improved + system-level observability ` * :pep:`798`: :ref:`Unpacking in comprehensions ` * :pep:`686`: :ref:`Python now uses UTF-8 as the default encoding ` -* :pep:`728`: ``TypedDict`` with typed extra items +* :pep:`829`: :ref:`Package startup configuration files ` +* :pep:`728`: :ref:`TypedDict with typed extra items ` * :pep:`747`: :ref:`Annotating type forms with TypeForm ` +* :pep:`800`: :ref:`Disjoint bases in the type system ` * :pep:`782`: :ref:`A new PyBytesWriter C API to create a Python bytes object ` +* :pep:`803`, :pep:`820 <820>`, :pep:`793 <793>`: + :ref:`Stable ABI for free-threaded builds ` and + related C API +* :pep:`788`: :ref:`Protection against finalization in the C API ` * :ref:`The JIT compiler has been significantly upgraded ` -* :ref:`Improved error messages ` * :ref:`The official Windows 64-bit binaries now use the tail-calling interpreter ` +* :ref:`Improved error messages ` +* :ref:`More color ` + New features ============ @@ -140,16 +152,15 @@ In the case where loading a lazily imported module fails (for example, if the module does not exist), Python raises the exception at the point of first use rather than at import time. The associated traceback includes both the location where the name was accessed and the original import statement, -making it straightforward to diagnose & debug the failure. +making it straightforward to diagnose and debug the failure. For cases where you want to enable lazy loading globally without modifying source code, Python provides the :option:`-X lazy_imports <-X>` command-line option and the :envvar:`PYTHON_LAZY_IMPORTS` environment variable. Both -accept three values: ``all`` makes all imports lazy by default, ``none`` -disables lazy imports entirely (even explicit ``lazy`` statements become -eager), and ``normal`` (the default) respects the ``lazy`` keyword in source -code. The :func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` -functions allow changing and querying this mode at runtime. +accept two values: ``all`` makes all imports lazy by default, and ``normal`` +(the default) respects the ``lazy`` keyword in source code. The +:func:`sys.set_lazy_imports` and :func:`sys.get_lazy_imports` functions allow +changing and querying this mode at runtime. For more selective control, :func:`sys.set_lazy_imports_filter` accepts a callable that determines whether a specific module should be loaded lazily. @@ -172,7 +183,7 @@ eager: import myapp.slow_module # lazy (matches filter) import json # eager (does not match filter) -The proxy type itself is available as :data:`types.LazyImportType` for code +The proxy type itself is available as :class:`types.LazyImportType` for code that needs to detect lazy imports programmatically. There are some restrictions on where the ``lazy`` keyword can be used. Lazy @@ -182,6 +193,18 @@ function, class body, or ``try``/``except``/``finally`` block raises a (``lazy from module import *`` and ``lazy from __future__ import ...`` both raise :exc:`SyntaxError`). +For code that cannot use the ``lazy`` keyword directly (for example, when +supporting Python versions older than 3.15 while still using lazy +imports on 3.15+), a module can define +:attr:`~module.__lazy_modules__` as a container of fully qualified module +name strings. Regular ``import`` statements for those modules are then treated +as lazy, with the same semantics as the ``lazy`` keyword:: + + __lazy_modules__ = ["json", "pathlib"] + + import json # lazy + import os # still eager + .. seealso:: :pep:`810` for the full specification and rationale. (Contributed by Pablo Galindo Salgado and Dino Viehland in :gh:`142349`.) @@ -233,6 +256,22 @@ to accept also other mapping types such as :class:`~types.MappingProxyType`. (Contributed by Victor Stinner and Donghee Na in :gh:`141510`.) +.. _whatsnew315-sentinel: + +:pep:`661`: Add sentinel built-in type +-------------------------------------- + +A new :class:`sentinel` type is added to the :mod:`builtins` module for +creating unique sentinel values with a concise representation. Sentinel +objects preserve identity when copied, support use in type expressions with +the ``|`` operator, and can be pickled when they are importable by module and +name. + +(PEP by Tal Einat; contributed by Jelle Zijlstra in :gh:`148829`.) + +.. seealso:: :pep:`661` for further details. + + .. _whatsnew315-profiling-package: :pep:`799`: A dedicated profiling package @@ -258,7 +297,7 @@ The :mod:`profile` module is deprecated and will be removed in Python 3.17. Tachyon: High frequency statistical sampling profiler ----------------------------------------------------- -.. image:: ../library/tachyon-logo.png +.. image:: ../../Lib/profiling/sampling/_assets/tachyon-logo.png :alt: Tachyon profiler logo :align: center :width: 200px @@ -290,6 +329,9 @@ Key features include: * Profile running processes by PID (``attach``) - attach to already-running applications * Run and profile scripts directly (``run``) - profile from the very start of execution * Execute and profile modules (``run -m``) - profile packages run as ``python -m module`` + * Capture a one-shot snapshot of a running process (``dump``) - print a + traceback-style stack of every thread (or all asyncio tasks with + ``--async-aware``). Useful for investigating hung processes. * **Multiple profiling modes**: Choose what to measure based on your performance investigation: @@ -341,9 +383,42 @@ available output formats, profiling modes, and configuration options. (Contributed by Pablo Galindo and László Kiss Kollár in :gh:`135953` and :gh:`138122`.) +.. _whatsnew315-frame-pointers: + +:pep:`831`: Frame pointers enabled by default +--------------------------------------------- + +CPython is now built with frame pointers by default on platforms that support +them. This uses the compiler flags ``-fno-omit-frame-pointer`` and +``-mno-omit-leaf-frame-pointer``, making native stack unwinding faster and +more reliable for system profilers, debuggers, crash analysis tools, and +eBPF-based observability tools. + +The flags are exposed through :mod:`sysconfig`, so extension modules built by +tools that consume Python's build configuration inherit frame pointers by +default. This propagation is intentional: mixed Python/native profiling needs +an unbroken frame-pointer chain through the interpreter, extension modules, +embedding applications, and native libraries. + +.. important:: + + Third-party build backends and native build systems should preserve these + flags when they consume Python's :mod:`sysconfig` values. Build systems + that compile C, C++, Rust, or other native code without inheriting Python's + compiler flags should enable equivalent frame-pointer flags themselves. A + single native component built without frame pointers can break stack + unwinding for the whole Python process. + +(Contributed by Pablo Galindo Salgado and Savannah Ostrowski in +:gh:`149201`; PEP 831 written by Pablo Galindo Salgado, Ken Jin, +Savannah Ostrowski, and Diego Russo.) + +.. seealso:: :pep:`831` for further details. + + .. _whatsnew315-unpacking-in-comprehensions: -:pep:`798`: Unpacking in Comprehensions +:pep:`798`: Unpacking in comprehensions --------------------------------------- List, set, and dictionary comprehensions, as well as generator expressions, now @@ -381,6 +456,116 @@ agen() for x in a)``. (Contributed by Adam Hartz in :gh:`143055`.) +.. _whatsnew315-startup-files: + +:pep:`829`: Package startup configuration files +----------------------------------------------- + +Loaded by the :mod:`site` module when ``-S`` is not given, :ref:`.pth files ` +can contain lines that both extend :data:`sys.path` and execute arbitrary code +when the line starts with ``import`` (followed by a space or tab). The latter +functionality can be problematic, since it is difficult to know exactly what +gets executed when Python starts up. + +As a step towards improving the ability to audit pre-start executable code, +Python 3.15 introduces :ref:`.start files ` which contain entry point +specifications of the form ``pkg.mod:callable`` where ``pkg.mod`` is the +import path to the given callable. When Python starts up, the callable is +located and called with no arguments. + +``import`` lines in :file:`.pth` files are silently deprecated. When a +matching :file:`.start` file is found, ``import`` lines in :file:`.pth` files +are ignored. There is no change to :data:`sys.path` extension lines in +:file:`.pth` files. + +The :mod:`site` module also provides :class:`site.StartupState` to batch +startup processing for multiple site directories, ensuring all static path +extensions are applied before any startup code is executed. :func:`site.main` +uses an instance of this class implicitly to batch process all startup +configuration files during normal interpreter startup. Callers needing the +same batching behavior can build a :class:`~site.StartupState` directly and +drive it with :meth:`~site.StartupState.addsitedir`, +:meth:`~site.StartupState.addusersitepackages`, and +:meth:`~site.StartupState.addsitepackages`, then call +:meth:`~site.StartupState.process` once at the end of the batch. + +(Contributed by Barry Warsaw in :gh:`148641` and :gh:`150228`.) + + +.. _whatsnew315-abi3t: + +:pep:`803`: Stable ABI for free-threaded builds +----------------------------------------------- + +C extensions that target the :ref:`Stable ABI ` can now be +compiled for the new *Stable ABI for Free-Threaded Builds* (also known +as ``abi3t``), which makes them compatible with +:term:`free-threaded builds ` of CPython. +This usually requires some non-trivial changes to the source code; +specifically: + +- Switching to API introduced in :pep:`697` (Python 3.12), such as + negative :c:member:`~PyType_Spec.basicsize` and + :c:func:`PyObject_GetTypeData`, rather than making :c:type:`PyObject` + part of the instance struct; and +- Switching from a ``PyInit_`` function to a new export hook, + :c:func:`PyModExport_* `, introduced for this + purpose in :pep:`793`, with a new :c:type:`PySlot` structure + introduced in :pep:`820`. + +Note that Stable ABI does not offer all the functionality that CPython +has to offer. +Extensions that cannot switch to ``abi3t`` should continue to build for +the existing Stable ABI (``abi3``) and the version-specific ABI for +free-threading (``cp315t``) separately. + +Stable ABI for Free-Threaded Builds should typically +be selected in a build tool (such as, for example, Setuptools, meson-python, +scikit-build-core, or Maturin). +At the time of writing, these tools do **not** support ``abi3t``. +If this is the case for your tool, compile for ``cp315t`` separately. +If not using a build tool -- or when writing such a tool -- you can select +``abi3t`` by setting the macro :c:macro:`!Py_TARGET_ABI3T` as discussed +in :ref:`abi3-compiling`. + +A practical :ref:`migration guide ` for switching to +``abi3t`` is available. + +.. seealso:: :pep:`803` for further details. + + +.. _whatsnew315-c-api-interpreter-finalization: + +:pep:`788`: Protecting the C API from interpreter finalization +-------------------------------------------------------------- + +In the C API, :term:`interpreter finalization ` can be +problematic for many extensions, because :term:`attaching ` a thread state will permanently hang the thread, resulting in deadlocks +and other spurious issues. Additionally, it has historically been impossible +to safely check whether an interpreter is alive before using it, leading to crashes +when a thread concurrently deletes an interpreter while another thread is +trying to attach to it. + +There are now several new suites of APIs to circumvent these problems: + +* :ref:`Interpreter guards `, which prevent an interpreter + from finalizing. +* :ref:`Interpreter views `, which allow thread-safe access + to an interpreter that may be concurrently finalizing or deleted. +* :ref:`New APIs ` to automatically attach and detach + thread states that come with built-in protection against finalization. + +In addition, APIs in the ``PyGILState`` family (most notably +:c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release`) have been +:term:`soft deprecated`. There is **no** plan to remove them, and existing +code will continue to work, but there will be no new ``PyGILState`` APIs +in future versions of Python. + +.. seealso:: :pep:`788` for further details. + +(Contributed by Peter Bierma in :gh:`149101`.) + .. _whatsnew315-improved-error-messages: @@ -414,14 +599,77 @@ Improved error messages Running this code now produces a clearer suggestion: - .. code-block:: pycon + .. code-block:: pytb Traceback (most recent call last): - File "/home/pablogsal/github/python/main/lel.py", line 42, in - print(container.area) - ^^^^^^^^^^^^^^ + File "/home/pablogsal/github/python/main/lel.py", line 42, in + print(container.area) + ^^^^^^^^^^^^^^ AttributeError: 'Container' object has no attribute 'area'. Did you mean '.inner.area' instead of '.area'? +* When an :exc:`AttributeError` on a builtin type has no close match via + Levenshtein distance, the error message now checks a static table of common + method names from other languages (JavaScript, Java, Ruby, C#) and suggests + the Python equivalent: + + .. doctest:: + + >>> [1, 2, 3].push(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'list' object has no attribute 'push'. Did you mean '.append'? + + >>> 'hello'.toUpperCase() # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'str' object has no attribute 'toUpperCase'. Did you mean '.upper'? + + When the Python equivalent is a language construct rather than a method, + the hint describes the construct directly: + + .. doctest:: + + >>> {}.put("a", 1) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'dict' object has no attribute 'put'. Use d[k] = v. + + When a mutable method is called on an immutable type, the hint suggests + the mutable counterpart: + + .. doctest:: + + >>> (1, 2, 3).append(4) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'tuple' object has no attribute 'append'. Did you mean to use a 'list' object? + + These hints also work for subclasses of builtin types. + + (Contributed by Matt Van Horn in :gh:`146406`.) + +* The interpreter now tries to provide a suggestion when + :func:`delattr` fails due to a missing attribute. + When an attribute name that closely resembles an existing attribute is used, + the interpreter will suggest the correct attribute name in the error message. + For example: + + .. doctest:: + + >>> class A: + ... pass + >>> a = A() + >>> a.abcde = 1 + >>> del a.abcdf # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? + + (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + +* Several error messages incorrectly using the term "argument" have been corrected. + (Contributed by Stan Ulbrych in :gh:`133382`.) + Other language changes ====================== @@ -453,32 +701,38 @@ Other language changes (Contributed by Adam Turner in :gh:`133711`; PEP 686 written by Inada Naoki.) -* Several error messages incorrectly using the term "argument" have been corrected. - (Contributed by Stan Ulbrych in :gh:`133382`.) + .. _whatsnew315-color-interpreter-help: -* The interpreter now tries to provide a suggestion when - :func:`delattr` fails due to a missing attribute. - When an attribute name that closely resembles an existing attribute is used, - the interpreter will suggest the correct attribute name in the error message. - For example: - - .. doctest:: - - >>> class A: - ... pass - >>> a = A() - >>> a.abcde = 1 - >>> del a.abcdf # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - AttributeError: 'A' object has no attribute 'abcdf'. Did you mean: 'abcde'? +* The interpreter help (such as ``python --help``) is now in color. + This can be controlled by :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148766`.) - (Contributed by Nikita Sobolev and Pranjal Prajapati in :gh:`136588`.) + .. _whatsnew315-color-exceptions: * Unraisable exceptions are now highlighted with color by default. This can be controlled by :ref:`environment variables `. (Contributed by Peter Bierma in :gh:`134170`.) + .. _whatsnew315-more-color: + +* More color in + :ref:`argparse `, + :ref:`ast `, + :ref:`calendar `, + :ref:`difflib `, + :ref:`http.server `, + :ref:`pickletools `, + :ref:`PyREPL tab completion `, + :ref:`python --help `, + :ref:`sqlite3 `, + :ref:`timeit `, + :ref:`tokenize `, + :ref:`unraisable exceptions ` and + :term:`stdlib` (ast, compileall, doctest, gzip, inspect, json.tool, pdb, + profiling.sampling, random, regrtest, sqlite3, timeit, tokenize, trace, + unittest, uuid, zipapp, zipfile) CLI help. + * The :meth:`~object.__repr__` of :class:`ImportError` and :class:`ModuleNotFoundError` now shows "name" and "path" as ``name=`` and ``path=`` if they were given as keyword arguments at construction time. @@ -553,7 +807,7 @@ Other language changes n = buffer.find(b'\n') data = bytes(buffer[:n + 1]) del buffer[:n + 1] - assert data == b'abc' + assert data == b'abc\n' assert buffer == bytearray(b'def') - .. code:: python @@ -601,13 +855,53 @@ Other language changes (Contributed by James Hilton-Balfe in :gh:`128335`.) * The class :class:`memoryview` now supports the :c:expr:`float complex` and - :c:expr:`double complex` C types: formatting characters ``'F'`` and ``'D'`` + :c:expr:`double complex` C types: formatting characters ``'Zf'`` and ``'Zd'`` respectively. - (Contributed by Sergey B Kirpichev in :gh:`146151`.) + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) * Allow the *count* argument of :meth:`bytes.replace` to be a keyword. (Contributed by Stan Ulbrych in :gh:`147856`.) +* Unary plus is now accepted in :keyword:`match` literal patterns, mirroring + the existing support for unary minus. + (Contributed by Bartosz Sławecki in :gh:`145239`.) + +* The import system now acquires per-module locks in hierarchical order + (parent packages before their submodules). This fixes a long-standing + deadlock where one thread importing ``pkg.sub`` and another importing + ``pkg.sub.mod`` could each block the other when ``pkg/sub/__init__.py`` + imports ``pkg.sub.mod``. + (Contributed by Gregory P. Smith in :gh:`83065`.) + +* File names of Stable ABI extensions that use the ``.so`` suffix may now + include a multiarch tuple, for example, ``foo.abi3-x86-64-linux-gnu.so``. + This permits stable ABI extensions for multiple architectures to be + co-installed into the same directory, without clashing with each + other, as regular dynamic extensions do. + (Contributed by Stefano Rivera in :gh:`122931`.) + +* The ``__cached__`` attribute on modules, which was deprecated since version + 3.13, is no longer set or taken into consideration by the import system or + standard library. + Use :attr:`__spec__.cached ` instead. + (Contributed by Brett Cannon in :gh:`97879`) + + Note that the :attr:`~module.__loader__` and :attr:`~module.__package__` + attributes are also deprecated and scheduled for removal. + + +Default interactive shell +========================= + +.. _whatsnew315-pyrepl-completion: + +* Tab completions are now colored by object kind, based on + :pypi:`fancycompleter`. + Set :envvar:`PYTHON_BASIC_COMPLETER` to fall back to :mod:`rlcompleter`. + Color can also be controlled by :ref:`environment variables + `. + (Contributed by Antonio Cuni and Pablo Galindo in :gh:`130472`.) + New modules =========== @@ -626,7 +920,7 @@ Improved modules argparse -------- -* The :class:`~argparse.BooleanOptionalAction` action supports now single-dash +* The :class:`~argparse.BooleanOptionalAction` action now supports single-dash long options and alternate prefix characters. (Contributed by Serhiy Storchaka in :gh:`138525`.) @@ -634,22 +928,59 @@ argparse default to ``True``. This enables suggestions for mistyped arguments by default. (Contributed by Jakob Schluse in :gh:`140450`.) -* Added backtick markup support in description and epilog text to highlight - inline code when color output is enabled. + .. _whatsnew315-color-argparse: + +* Added backtick markup support in :class:`~argparse.ArgumentParser` description + and epilog text to highlight inline code when color output is enabled. (Contributed by Savannah Ostrowski in :gh:`142390`.) +* Extended backtick markup to argument ``help`` text and added support for + double backticks (RST inline-literal style). + (Contributed by Hugo van Kemenade in :gh:`149375`.) + array ----- * Support the :c:expr:`float complex` and :c:expr:`double complex` C types: - formatting characters ``'F'`` and ``'D'`` respectively. - (Contributed by Sergey B Kirpichev in :gh:`146151`.) + formatting characters ``'Zf'`` and ``'Zd'`` respectively. + (Contributed by Victor Stinner in :gh:`146151` and :gh:`148675`.) * Support half-floats (16-bit IEEE 754 binary interchange format): formatting character ``'e'``. (Contributed by Sergey B Kirpichev in :gh:`146238`.) +* The :data:`array.typecodes` type changed from :class:`str` to :class:`tuple` + to support type codes longer than 1 character (``Zf`` and ``Zd``). + (Contributed by Victor Stinner in :gh:`148675`.) + + +ast +--- + + .. _whatsnew315-color-ast: + +* Add *color* parameter to :func:`~ast.dump`. + If ``True``, the returned string is syntax highlighted using ANSI escape + sequences. + If ``False`` (the default), colored output is always disabled. + (Contributed by Stan Ulbrych in :gh:`148981`.) + +* The :ref:`command-line ` output is now syntax highlighted by default. + This can be :ref:`controlled using environment variables `. + (Contributed by Stan Ulbrych in :gh:`148981`.) + + +asyncio +------- + +* Added :meth:`TaskGroup.cancel ` to allow early + termination of a task group, for instance, when the goal of the tasks has + been achieved or their services are no longer needed. + Previously this would involve unintuitive boilerplate such as an extra task + raising a custom exception which is then suppressed as it exits the task group. + (Contributed by John Belmonte in :gh:`127214`.) + base64 ------ @@ -671,11 +1002,20 @@ base64 (Contributed by Serhiy Storchaka in :gh:`143214` and :gh:`146431`.) * Added the *ignorechars* parameter in :func:`~base64.b16decode`, - :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, :func:`~base64.b64decode`, :func:`~base64.b85decode`, and :func:`~base64.z85decode`. (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) +* Added the *canonical* parameter in + :func:`~base64.b32decode`, :func:`~base64.b32hexdecode`, + :func:`~base64.b64decode`, + :func:`~base64.a85decode`, :func:`~base64.b85decode`, and + :func:`~base64.z85decode`, + to reject encodings with non-zero padding bits or other non-canonical + forms. + (Contributed by Gregory P. Smith in :gh:`146311`.) + binascii -------- @@ -709,18 +1049,29 @@ binascii :func:`~binascii.unhexlify`, and :func:`~binascii.a2b_base64`. (Contributed by Serhiy Storchaka in :gh:`144001` and :gh:`146431`.) +* Added the *canonical* parameter in :func:`~binascii.a2b_base64`, + to reject encodings with non-zero padding bits. + (Contributed by Gregory P. Smith in :gh:`146311`.) + calendar -------- -* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support - dark mode and have been migrated to the HTML5 standard for improved accessibility. - (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + .. _whatsnew315-color-calendar: + +* :mod:`calendar`'s :ref:`command-line ` text output has more + color. This can be controlled with :ref:`environment variables + `. + (Contributed by Hugo van Kemenade in :gh:`148352`.) * The :mod:`calendar`'s :ref:`command-line ` HTML output now accepts the year-month option: ``python -m calendar -t html 2009 06``. (Contributed by Pål Grønås Drange in :gh:`140212`.) +* Calendar pages generated by the :class:`calendar.HTMLCalendar` class now support + dark mode and have been migrated to the HTML5 standard for improved accessibility. + (Contributed by Jiahao Li and Hugo van Kemenade in :gh:`137634`.) + collections ----------- @@ -750,6 +1101,25 @@ contextlib consistency with the :keyword:`with` and :keyword:`async with` statements. (Contributed by Serhiy Storchaka in :gh:`144386`.) +* :class:`~contextlib.ContextDecorator` and + :class:`~contextlib.AsyncContextDecorator` (and therefore + :func:`~contextlib.contextmanager` and :func:`~contextlib.asynccontextmanager` + used as decorators) now detect generator functions, coroutine functions, and + asynchronous generator functions and keep the context manager open across + iteration or await. Previously the context manager exited as soon as the + generator or coroutine object was created. + (Contributed by Alex Grönholm & Gregory P. Smith in :gh:`125862`.) + + +ctypes +------ + +* Change the :py:attr:`~ctypes._SimpleCData._type_` of + :class:`~ctypes.c_float_complex`, :class:`~ctypes.c_double_complex` and + :class:`~ctypes.c_longdouble_complex` from ``F``, ``D`` and ``G`` to ``Zf``, + ``Zd`` and ``Zg`` for compatibility with NumPy. + (Contributed by Victor Stinner in :gh:`148675`.) + dataclasses ----------- @@ -762,7 +1132,7 @@ dbm --- * Added new :meth:`!reorganize` methods to :mod:`dbm.dumb` and :mod:`dbm.sqlite3` - which allow to recover unused free space previously occupied by deleted entries. + to recover unused free space previously occupied by deleted entries. (Contributed by Andrea Oliveri in :gh:`134004`.) @@ -782,6 +1152,25 @@ difflib (Contributed by Jiahao Li in :gh:`134580`.) +email +----- + +* Email generators now raise an error when an :class:`.EmailMessage` cannot be + accurately flattened due to a non-ASCII email address (mailbox) in an address + header. Options for supporting Email Address Internationalization (EAI) are + discussed in :attr:`.EmailPolicy.utf8`. + (Contributed by R David Murray and Mike Edmunds in :gh:`122540`.) + + +faulthandler +------------ + +* Added the *max_threads* parameter in :func:`faulthandler.enable`, + :func:`faulthandler.dump_traceback`, :func:`faulthandler.dump_traceback_later`, + and :func:`faulthandler.register`. + (Contributed by Eric Froemling in :gh:`149085`.) + + functools --------- @@ -794,6 +1183,17 @@ functools (Contributed by Bartosz Sławecki in :gh:`143535`.) +gc +-- + +* Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later and Python 3.15. + + hashlib ------- @@ -815,24 +1215,47 @@ http.client (Contributed by Alexander Enrique Urieles Nieto in :gh:`131724`.) -http.cookies ------------- - -* Allow '``"``' double quotes in cookie values. - (Contributed by Nick Burns and Senthil Kumaran in :gh:`92936`.) - - http.server ----------- + .. _whatsnew315-color-http.server: + * The logging of :mod:`~http.server.BaseHTTPRequestHandler`, as used by the :ref:`command-line interface `, is colored by default. This can be controlled with :ref:`environment variables `. - (Contributed by Hugo van Kemenade in :gh:`146292`.) - - + (Contributed by Hugo van Kemenade in :gh:`146292`.) + +* Added :attr:`~http.server.SimpleHTTPRequestHandler.default_content_type` + and the :option:`--content-type ` command-line + option to allow customizing the default ``Content-Type`` header + for files with unknown extensions. + (Contributed by John Comeau and Hugo van Kemenade in :gh:`113471`.) + +* Add a new ``extra_response_headers`` keyword argument to + :class:`~http.server.SimpleHTTPRequestHandler` to support custom headers in + HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + +* Add a ``-H/--header`` option to the :program:`python -m http.server` + command-line interface to support custom headers in HTTP responses. + (Contributed by Anton I. Sipos in :gh:`135057`.) + + +importlib.metadata +------------------ + +* Previously, when accessing a distribution metadata directory not + containing a metadata file, ``metadata()`` and ``Distribution.metadata`` + would return an empty ``PackageMetadata`` object as if the file + was present but empty. Now, a ``MetadataNotFound`` exception is raised. + See `importlib_metadata#493 `_ + for background and rationale and :gh:`143387` for rationale on the + compatibility concerns. + (Contributed by Jason R. Coombs.) + + inspect ------- @@ -840,16 +1263,17 @@ inspect for :func:`~inspect.getdoc`. (Contributed by Serhiy Storchaka in :gh:`132686`.) + json ---- -* Add the *array_hook* parameter to :func:`~json.load` and +* Add the *array_hook* parameter to :func:`~json.load` and :func:`~json.loads` functions: allow a callback for JSON literal array types to customize Python lists in the resulting decoded object. Passing combined :class:`frozendict` to *object_pairs_hook* param and :class:`tuple` to ``array_hook`` will yield a deeply nested immutable Python structure representing the JSON data. - (Contributed by Joao S. O. Bueno in :gh:`146440`) + (Contributed by Joao S. O. Bueno in :gh:`146440`.) locale @@ -877,23 +1301,11 @@ math mimetypes --------- -* Add ``application/dicom`` MIME type for ``.dcm`` extension. - (Contributed by Benedikt Johannes in :gh:`144217`.) -* Add ``application/efi``. (Contributed by Charlie Lin in :gh:`145720`.) -* Add ``application/node`` MIME type for ``.cjs`` extension. - (Contributed by John Franey in :gh:`140937`.) -* Add ``application/toml``. (Contributed by Gil Forcada in :gh:`139959`.) -* Add ``application/sql`` and ``application/vnd.sqlite3``. - (Contributed by Charlie Lin in :gh:`145698`.) -* Add the following MIME types: - - - ``application/vnd.ms-cab-compressed`` for ``.cab`` extension - - ``application/vnd.ms-htmlhelp`` for ``.chm`` extension - - ``application/vnd.ms-officetheme`` for ``.thmx`` extension - - (Contributed by Charlie Lin in :gh:`145718`.) - -* Add ``image/jxl``. (Contributed by Foolbar in :gh:`144213`.) +* Add more MIME types. + (Contributed by Benedikt Johannes, Charlie Lin, Foolbar, Gil Forcada and + John Franey + in :gh:`144217`, :gh:`145720`, :gh:`140937`, :gh:`139959`, :gh:`145698`, + :gh:`145718`, :gh:`145918`, and :gh:`144213`.) * Rename ``application/x-texinfo`` to ``application/texinfo``. (Contributed by Charlie Lin in :gh:`140165`.) * Changed the MIME type for ``.ai`` files to ``application/pdf``. @@ -921,6 +1333,10 @@ os glibc versions 2.28 and later. (Contributed by Jeffrey Bosboom and Victor Stinner in :gh:`83714`.) +* :func:`os.makedirs` function now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories. This can be used to match + the behavior from Python 3.6 and earlier by passing ``parent_mode=mode``. + (Contributed by Zackery Spytz and Gregory P. Smith in :gh:`86533`.) os.path ------- @@ -935,6 +1351,21 @@ os.path (Contributed by Petr Viktorin for :cve:`2025-4517`.) +pathlib +------- + +* :meth:`pathlib.Path.mkdir` now has a *parent_mode* parameter that allows + specifying the mode for intermediate directories when ``parents=True``. + (Contributed by Gregory P. Smith in :gh:`86533`.) + + +pdb +--- + +* Use the new interactive shell as the default input shell for :mod:`pdb`. + (Contributed by Tian Gao in :gh:`145379`.) + + pickle ------ @@ -942,12 +1373,38 @@ pickle (Contributed by Zackery Spytz and Serhiy Storchaka in :gh:`77188`.) +pickletools +----------- + + .. _whatsnew315-color-pickletools: + +* The output of the :mod:`pickletools` command-line interface is colored by + default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`149026`.) + + +pprint +------ + +* Add an *expand* keyword argument for :func:`pprint.pprint`, + :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be + formatted similar to pretty-printed :func:`json.dumps` when + *indent* is supplied. + (Contributed by Stefan Todoran, Semyon Moroz and Hugo van Kemenade in + :gh:`112632`.) + +* Add t-string support to :mod:`pprint`. + (Contributed by Loïc Simon and Hugo van Kemenade in :gh:`134551`.) + + re -- -* :func:`re.prefixmatch` and a corresponding :meth:`~re.Pattern.prefixmatch` - have been added as alternate more explicit names for the existing - :func:`re.match` and :meth:`~re.Pattern.match` APIs. These are intended +* :func:`re.prefixmatch` and a corresponding :meth:`re.Pattern.prefixmatch` + have been added as alternate, more explicit names for the existing + and now :term:`soft deprecated` + :func:`re.match` and :meth:`re.Pattern.match` APIs. These are intended to be used to alleviate confusion around what *match* means by following the Zen of Python's *"Explicit is better than implicit"* mantra. Most other language regular expression libraries use an API named *match* to mean what @@ -975,6 +1432,14 @@ shelve (Contributed by Furkan Onder in :gh:`99631`.) +shutil +------ + +* :func:`shutil.copyfile` now also raises :exc:`~shutil.SpecialFileError` for + sockets and device files. + (Contributed by Savannah Ostrowski in :gh:`142693`.) + + socket ------ @@ -990,6 +1455,8 @@ sqlite3 * SQL keyword completion on . (Contributed by Long Tan in :gh:`133393`.) + .. _whatsnew315-color-sqlite3: + * Prompts, error messages, and help text are now colored. This is enabled by default, see :ref:`using-on-controlling-color` for details. @@ -1006,7 +1473,7 @@ ssl supports "External PSKs" in TLSv1.3, as described in :rfc:`9258`. (Contributed by Will Childs-Klein in :gh:`133624`.) -* Added new methods for managing groups used for SSL key agreement +* Added new methods for managing groups used for SSL key agreement: * :meth:`ssl.SSLContext.set_groups` sets the groups allowed for doing key agreement, extending the previous @@ -1062,7 +1529,7 @@ subprocess If none of these mechanisms are available, the function falls back to the traditional busy loop (non-blocking call and short sleeps). - (Contributed by Giampaolo Rodola in :gh:`83069`). + (Contributed by Giampaolo Rodola in :gh:`83069`.) symtable @@ -1079,6 +1546,19 @@ sys (Contributed by Klaus Zimmermann in :gh:`137476`.) +sys.monitoring +-------------- + +* The :ref:`other events ` + (:monitoring-event:`PY_THROW`, :monitoring-event:`PY_UNWIND`, + :monitoring-event:`RAISE`, :monitoring-event:`EXCEPTION_HANDLED`, and + :monitoring-event:`RERAISE`) can now be turned on and disabled on a per code + object basis. Returning :data:`~sys.monitoring.DISABLE` from a callback for + one of these events disables the event for the entire code object (for the + current tool), rather than raising :exc:`ValueError` as in prior versions. + (Contributed by Gabriele N. Tornetta in :gh:`146182`.) + + tarfile ------- @@ -1100,18 +1580,34 @@ tarfile (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` and :cve:`2025-4435`.) * :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` - now replace slashes by backslashes in symlink targets on Windows to prevent + now replace slashes with backslashes in symlink targets on Windows to prevent creation of corrupted links. (Contributed by Christoph Walcher in :gh:`57911`.) +* :func:`~tarfile.TarFile.gettarinfo` now replaces backslashes with slashes in + symlink targets on Windows to conform to the tar format standard. (Contributed + by Daniele Nicolodi in :gh:`151669`.) + + +threading +--------- + +* Added :class:`~threading.serialize_iterator`, + :func:`~threading.synchronized_iterator`, + and :func:`~threading.concurrent_tee` to support concurrent access to + generators and iterators. + (Contributed by Raymond Hettinger in :gh:`124397`.) timeit ------ + .. _whatsnew315-color-timeit: + * The output of the :mod:`timeit` command-line interface is colored by default. This can be controlled with :ref:`environment variables `. (Contributed by Hugo van Kemenade in :gh:`146609`.) + * The command-line interface now colorizes error tracebacks by default. This can be controlled with :ref:`environment variables `. @@ -1119,7 +1615,7 @@ timeit * Make the target time of :meth:`timeit.Timer.autorange` configurable and add ``--target-time`` option to the command-line interface. - (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`140283`.) + (Contributed by Alessandro Cucci and Miikka Koskinen in :gh:`80642`.) tkinter @@ -1148,6 +1644,17 @@ tkinter (Contributed by Matthias Kievernagel and Serhiy Storchaka in :gh:`47655`.) +tokenize +-------- + + .. _whatsnew315-color-tokenize: + +* The output of the :mod:`tokenize` :ref:`command-line interface + ` is colored by default. This can be controlled with + :ref:`environment variables `. + (Contributed by Hugo van Kemenade in :gh:`148991`.) + + .. _whatsnew315-tomllib-1-1-0: tomllib @@ -1164,10 +1671,7 @@ tomllib Previously an inline table had to be on a single line and couldn't end with a trailing comma. This is now relaxed so that the following is valid: - .. syntax highlighting needs TOML 1.1.0 support in Pygments, - see https://github.com/pygments/pygments/issues/3026 - - .. code-block:: text + .. code-block:: toml tbl = { key = "a string", @@ -1179,7 +1683,7 @@ tomllib - Add ``\xHH`` notation to basic strings for codepoints under 255, and the ``\e`` escape for the escape character: - .. code-block:: text + .. code-block:: toml null = "null byte: \x00; letter a: \x61" csi = "\e[" @@ -1187,7 +1691,7 @@ tomllib - Seconds in datetime and time values are now optional. The following are now valid: - .. code-block:: text + .. code-block:: toml dt = 2010-02-03 14:15 t = 14:15 @@ -1201,7 +1705,7 @@ types ----- * Expose the write-through :func:`locals` proxy type - as :data:`types.FrameLocalsProxyType`. + as :class:`types.FrameLocalsProxyType`. This represents the type of the :attr:`frame.f_locals` attribute, as described in :pep:`667`. @@ -1230,15 +1734,38 @@ typing (Contributed by Jelle Zijlstra in :gh:`145033`.) +.. _whatsnew315-typeddict: + +* :pep:`728`: Add support in :class:`~typing.TypedDict` for the *closed* + and *extra_items* class arguments. A closed :class:`~typing.TypedDict` + does not allow extra keys beyond those specified in the class body, while + a :class:`~typing.TypedDict` with ``extra_items`` allows arbitrary extra + items where the values are of the specified type. (Contributed by Angela + Liss in :gh:`137840`.) + * Code like ``class ExtraTypeVars(P1[S], Protocol[T, T2]): ...`` now raises a :exc:`TypeError`, because ``S`` is not listed in ``Protocol`` parameters. (Contributed by Nikita Sobolev in :gh:`137191`.) * Code like ``class B2(A[T2], Protocol[T1, T2]): ...`` now correctly handles type parameters order: it is ``(T1, T2)``, not ``(T2, T1)`` - as it was incorrectly inferred in runtime before. + as it was incorrectly inferred at runtime before. (Contributed by Nikita Sobolev in :gh:`137191`.) + .. _whatsnew315-disjoint-base: + +* :pep:`800`: Add :deco:`typing.disjoint_base`, a new decorator marking a class + as a disjoint base. This is an advanced feature primarily intended to allow + type checkers to faithfully reflect the runtime semantics of types defined + as builtins or in compiled extensions. If a class ``C`` is a disjoint base, then + child classes of that class cannot inherit from other disjoint bases that are + not parent or child classes of ``C``. (Contributed by Jelle Zijlstra in :gh:`148639`.) + +* :class:`~typing.TypeVarTuple` now accepts ``bound``, ``covariant``, + ``contravariant``, and ``infer_variance`` keyword arguments, matching the + interface of :class:`~typing.TypeVar` and :class:`~typing.ParamSpec`. + ``bound`` semantics remain undefined in the specification. + unicodedata ----------- @@ -1269,10 +1796,16 @@ unicodedata unittest -------- -* :func:`unittest.TestCase.assertLogs` will now accept a formatter +* :meth:`unittest.TestCase.assertLogs` will now accept a formatter to control how messages are formatted. (Contributed by Garry Cairns in :gh:`134567`.) +* :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex. + Nested context managers are now supported. + (Contributed by Serhiy Storchaka in :gh:`143231`.) + urllib.parse ------------ @@ -1281,8 +1814,8 @@ urllib.parse :func:`~urllib.parse.urlparse` and :func:`~urllib.parse.urldefrag` functions. Add the *keep_empty* parameter to :func:`~urllib.parse.urlunsplit` and :func:`~urllib.parse.urlunparse` functions. - This allows to distinguish between empty and not defined URI components - and preserve empty components. + This allows distinguishing between empty and undefined URI components + and preserving empty components. (Contributed by Serhiy Storchaka in :gh:`67041`.) @@ -1290,7 +1823,7 @@ venv ---- * On POSIX platforms, platlib directories will be created if needed when - creating virtual environments, instead of using ``lib64 -> lib`` symlink. + creating virtual environments, instead of using a ``lib64 -> lib`` symlink. This means purelib and platlib of virtual environments no longer share the same ``lib`` directory on platforms where :data:`sys.platlibdir` is not equal to ``lib``. @@ -1328,6 +1861,32 @@ wave (Contributed by Lionel Koenig and Michiel W. Beijen in :gh:`60729`.) +webbrowser +---------- + +* On macOS, the new :class:`!webbrowser.MacOS` class opens URLs via + :program:`/usr/bin/open` instead of constructing and executing AppleScript + via :program:`osascript`. The default browser is detected from the + LaunchServices preferences file using :mod:`plistlib`, with + :class:`!com.apple.Safari` as the fallback on fresh installations. + For non-HTTP(S) URLs, :program:`open -b ` is used to route the + URL through a browser rather than the OS file handler, preventing + file injection attacks. + (Contributed by Jeff Lyon in :gh:`137586`.) + + +xml +--- + +* Add the :func:`xml.is_valid_name` function to check + whether a string can be used as an element or attribute name in XML. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + +* Add the :func:`xml.is_valid_text` function to check + whether a string can be used in an XML document. + (Contributed by Serhiy Storchaka in :gh:`139489`.) + + xml.parsers.expat ----------------- @@ -1362,7 +1921,7 @@ Optimizations ============= * ``mimalloc`` is now used as the default allocator for - for raw memory allocations such as via :c:func:`PyMem_RawMalloc` + raw memory allocations such as via :c:func:`PyMem_RawMalloc` for better performance on :term:`free-threaded builds `. (Contributed by Kumar Aditya in :gh:`144914`.) @@ -1381,7 +1940,7 @@ base64 & binascii * Implementation for Base32 has been rewritten in C. Encoding and decoding is now two orders of magnitude faster. - (Contributed by James Seo in :gh:`146192`) + (Contributed by James Seo in :gh:`146192`.) csv @@ -1398,11 +1957,11 @@ Upgraded JIT compiler Results from the `pyperformance `__ benchmark suite report -`6-7% `__ +`8-9% `__ geometric mean performance improvement for the JIT over the standard CPython interpreter built with all optimizations enabled on x86-64 Linux. On AArch64 macOS, the JIT has a -`12-13% `__ +`12-13% `__ speedup over the :ref:`tail calling interpreter ` with all optimizations enabled. The speedups for JIT builds versus no JIT builds range from roughly 15% slowdown to over @@ -1418,6 +1977,7 @@ The major upgrades to the JIT are: * New tracing frontend * Basic register allocation in the JIT * More JIT optimizations +* GDB and GNU ``backtrace()`` unwinding support * Better machine code generation .. rubric:: LLVM 21 build-time dependency @@ -1428,7 +1988,6 @@ end users running Python do not need LLVM installed. Instructions for installing LLVM can be found in the `JIT compiler documentation `__ for all supported platforms. - (Contributed by Savannah Ostrowski in :gh:`140973`.) .. rubric:: A new tracing frontend @@ -1440,7 +1999,6 @@ code. For example, simple Python object creation is now understood by the supported. This was made possible by an overhauled JIT tracing frontend that records actual execution paths through code, rather than estimating them as the previous implementation did. - (Contributed by Ken Jin in :gh:`139109`. Support for Windows added by Mark Shannon in :gh:`141703`.) @@ -1450,7 +2008,6 @@ A basic form of register allocation has been added to the JIT compiler's optimizer. This allows the JIT compiler to avoid certain stack operations altogether and instead operate on registers. This allows the JIT to produce more efficient traces by avoiding reads and writes to memory. - (Contributed by Mark Shannon in :gh:`135379`.) .. rubric:: More JIT optimizations @@ -1458,31 +2015,62 @@ more efficient traces by avoiding reads and writes to memory. More `constant-propagation `__ is now performed. This means when the JIT compiler detects that certain user code results in constants, the code can be simplified by the JIT. - (Contributed by Ken Jin and Savannah Ostrowski in :gh:`132732`.) -The JIT avoids :term:`reference count`\ s where possible. This generally -reduces the cost of most operations in Python. - +:term:`Reference counts ` are avoided whenever it is safe to +do so. This generally reduces the cost of most operations in Python. (Contributed by Ken Jin, Donghee Na, Zheao Li, Hai Zhu, Savannah Ostrowski, Reiden Ong, Noam Cohen, Tomas Roun, PuQing, Cajetan Rodrigues, and Sacul in :gh:`134584`.) +By tracking unique references to objects, the JIT optimizer can now eliminate +reference count updates and perform in-place operations on ints and floats. +(Contributed by Reiden Ong, and Pieter Eendebak in :gh:`143414` and :gh:`146306`.) + +The JIT optimizer now supports significantly more operations than in 3.14. +(Contributed by Kumar Aditya, Ken Jin, Jiahao Li, and Sacul in :gh:`131798`.) + +.. rubric:: GDB and GNU ``backtrace()`` unwinding support + +The JIT compiler now publishes unwind information for generated machine code to the +GDB interface on supported Linux ELF platforms. When libgcc frame +registration is available, the same unwind information is also registered for +GNU ``backtrace()`` stack walkers. This allows native debuggers, crash +handlers, and diagnostic tools using these mechanisms to unwind through JIT +frames instead of stopping at generated code. +(Contributed by Diego Russo and Pablo Galindo Salgado in :gh:`146071` and +:gh:`149104`.) + .. rubric:: Better machine code generation The JIT compiler's machine code generator now produces better machine code for x86-64 and AArch64 macOS and Linux targets. In general, users should experience lower memory usage for generated machine code and more efficient -machine code versus the old JIT. - -(Contributed by Brandt Bucher in :gh:`136528` and :gh:`136528`. +machine code versus 3.14. +(Contributed by Brandt Bucher in :gh:`136528` and :gh:`135905`. Implementation for AArch64 contributed by Mark Shannon in :gh:`139855`. Additional optimizations for AArch64 contributed by Mark Shannon and Diego Russo in :gh:`140683` and :gh:`142305`.) +.. rubric:: Maintainability + +The JIT optimizer's operations have been simplified. +This was made possible by a refactoring of JIT data structures. +(Contributed by Zhongtian Zheng in :gh:`148211` and Hai Zhu in :gh:`143421`.) + Removed ======== +ast +--- + +* The constructors of :ref:`AST nodes ` now raise a :exc:`TypeError` + when a required argument is omitted or when a keyword argument that does not + map to a field on the AST node is passed. These cases had previously raised a + :exc:`DeprecationWarning` since Python 3.13. + (Contributed by Brian Schubert and Jelle Zijlstra in :gh:`137600` and :gh:`105858`.) + + collections.abc --------------- @@ -1499,6 +2087,15 @@ ctypes (Contributed by Bénédikt Tran in :gh:`133866`.) +datetime +-------- + +* :meth:`~datetime.datetime.strptime` now raises :exc:`ValueError` when the + format string contains ``%d`` (day of month) without a year directive. + This has been deprecated since Python 3.13. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`70647`.) + + glob ---- @@ -1541,16 +2138,6 @@ platform (Contributed by Alexey Makridenko in :gh:`133604`.) -pprint ------- - -* Add an *expand* keyword argument for :func:`pprint.pprint`, - :func:`pprint.pformat`, :func:`pprint.pp`. If true, the output will be - formatted similar to pretty-printed :func:`json.dumps` when - *indent* is supplied. - (Contributed by Stefan Todoran and Semyon Moroz in :gh:`112632`.) - - sre_* ----- @@ -1574,6 +2161,14 @@ threading (Contributed by Bénédikt Tran in :gh:`134087`.) +types +----- + +* Removed :attr:`!codeobject.co_lnotab` from :class:`types.CodeType`, + deprecated since Python 3.12 by :pep:`626`. + (Contributed by Nikita Sobolev in :gh:`134690`.) + + typing ------ @@ -1593,6 +2188,9 @@ typing or ``TD = TypedDict("TD", {})`` instead. (Contributed by Bénédikt Tran in :gh:`133823`.) +* Deprecated :deco:`!typing.no_type_check_decorator` has been removed. + (Contributed by Nikita Sobolev in :gh:`133601`.) + wave ---- @@ -1617,6 +2215,13 @@ Deprecated New deprecations ---------------- +* :mod:`ast` + + * Creating instances of abstract AST nodes (such as :class:`ast.AST` + or :class:`!ast.expr`) is deprecated and will raise an error in Python 3.20. + + (Contributed by Brian Schubert in :gh:`116021`.) + * :mod:`base64`: * Accepting the ``+`` and ``/`` characters with an alternative alphabet in @@ -1665,15 +2270,66 @@ New deprecations (Contributed by Bénédikt Tran in :gh:`134978`.) + +* :mod:`http.cookies`: + + * :meth:`Morsel.js_output ` and + :meth:`BaseCookie.js_output ` are + deprecated and will be removed in Python 3.19. Use + :meth:`Morsel.output ` or + :meth:`BaseCookie.output ` instead. + (Contributed by kishorhange111 in :gh:`148849`.) + + +* :mod:`imaplib`: + + * Altering :attr:`IMAP4.file ` is now deprecated + and slated for removal in Python 3.19. This property is now unused + and changing its value does *not* explicitly close the current file. + + +* :mod:`profile`: + + * The :mod:`!profile` module is deprecated and will be removed in + Python 3.17. Use :mod:`profiling.tracing` instead, which provides a + compatible API. See :pep:`799` for details. + (Contributed by Pablo Galindo and László Kiss Kollár in :gh:`138122`.) + + +* :mod:`re`: + + * :func:`re.match` and :meth:`re.Pattern.match` are now + :term:`soft deprecated` in favor of the new :func:`re.prefixmatch` and + :meth:`re.Pattern.prefixmatch` APIs, which have been added as alternate, + more explicit names. These are intended to be used to alleviate confusion + around what *match* means by following the Zen of Python's *"Explicit is + better than implicit"* mantra. Most other language regular expression + libraries use an API named *match* to mean what Python has always called + *search*. + + We **do not** plan to remove the older :func:`!match` name, as it has been + used in code for over 30 years. Code supporting older versions of Python + should continue to use :func:`!match`, while new code should prefer + :func:`!prefixmatch`. See :ref:`prefixmatch-vs-match`. + + (Contributed by Gregory P. Smith in :gh:`86519` and + Hugo van Kemenade in :gh:`148100`.) + + * :mod:`struct`: - * Calling the ``Struct.__new__()`` without required argument now is - deprecated and will be removed in Python 3.20. Calling - :meth:`~object.__init__` method on initialized :class:`~struct.Struct` - objects is deprecated and will be removed in Python 3.20. + * Calling ``Struct.__new__()`` without a required argument is now + deprecated and will be removed in Python 3.20. Calling the + :meth:`~object.__init__` method on an initialized :class:`~struct.Struct` + object is deprecated and will be removed in Python 3.20. (Contributed by Sergey B Kirpichev and Serhiy Storchaka in :gh:`143715`.) + * The ``'F'`` and ``'D'`` type codes are now :term:`soft deprecated` + in favor of the two-letter forms ``'Zf'`` and ``'Zd'``. + (Contributed by Sergey B Kirpichev in :gh:`121249`.) + + * :mod:`typing`: * The following statements now cause ``DeprecationWarning``\ s to be emitted @@ -1687,8 +2343,19 @@ New deprecations :func:`issubclass`, but warnings were not previously emitted if it was merely imported or accessed from the :mod:`!typing` module. - * Deprecated :func:`!typing.no_type_check_decorator` has been removed. - (Contributed by Nikita Sobolev in :gh:`133601`.) + * It is deprecated to call :func:`isinstance` and :func:`issubclass` checks on + protocol classes that were not explicitly decorated with :func:`!runtime_checkable` + but that inherit from a runtime-checkable protocol class. + This will raise a :exc:`TypeError` in Python 3.20. + + (Contributed by Bartosz Sławecki in :gh:`132604`.) + + +* :mod:`webbrowser`: + + * :class:`!webbrowser.MacOSXOSAScript` is deprecated in favour of + :class:`!webbrowser.MacOS` and scheduled for removal in Python 3.17. + (Contributed by Jeff Lyon in :gh:`137586`.) * ``__version__`` @@ -1735,8 +2402,12 @@ New deprecations .. include:: ../deprecations/pending-removal-in-3.20.rst +.. include:: ../deprecations/pending-removal-in-3.21.rst + .. include:: ../deprecations/pending-removal-in-future.rst +.. include:: ../deprecations/soft-deprecations.rst + C API changes ============= @@ -1759,18 +2430,22 @@ New features (Contributed by Victor Stinner in :gh:`141510`.) +* Add :c:func:`PyObject_CallFinalizerFromDealloc` function to the limited C + API. + (Contributed by Victor Stinner in :gh:`146063`.) + * Add :c:func:`PySys_GetAttr`, :c:func:`PySys_GetAttrString`, :c:func:`PySys_GetOptionalAttr`, and :c:func:`PySys_GetOptionalAttrString` functions as replacements for :c:func:`PySys_GetObject`. (Contributed by Serhiy Storchaka in :gh:`108512`.) -* Add :c:type:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of +* Add :c:func:`PyUnstable_Unicode_GET_CACHED_HASH` to get the cached hash of a string. See the documentation for caveats. (Contributed by Petr Viktorin in :gh:`131510`.) * Add API for checking an extension module's ABI compatibility: - :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, :c:macro:`PyABIInfo_VAR` - and :c:data:`Py_mod_abi`. + :c:data:`Py_mod_abi`, :c:func:`PyABIInfo_Check`, + and :c:macro:`PyABIInfo_VAR`. (Contributed by Petr Viktorin in :gh:`137210`.) .. _whatsnew315-pybyteswriter: @@ -1780,26 +2455,36 @@ New features * :c:func:`PyBytesWriter_Create` * :c:func:`PyBytesWriter_Discard` + * :c:func:`PyBytesWriter_Finish` * :c:func:`PyBytesWriter_FinishWithPointer` * :c:func:`PyBytesWriter_FinishWithSize` - * :c:func:`PyBytesWriter_Finish` * :c:func:`PyBytesWriter_Format` * :c:func:`PyBytesWriter_GetData` * :c:func:`PyBytesWriter_GetSize` - * :c:func:`PyBytesWriter_GrowAndUpdatePointer` * :c:func:`PyBytesWriter_Grow` + * :c:func:`PyBytesWriter_GrowAndUpdatePointer` * :c:func:`PyBytesWriter_Resize` * :c:func:`PyBytesWriter_WriteBytes` (Contributed by Victor Stinner in :gh:`129813`.) -* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating +* :c:type:`PyCriticalSection` and related functions are added to the Stable + ABI. + + (Contributed in :gh:`149227`.) + +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C API for creating a module from a *spec* and *initfunc*. (Contributed by Itamar Oren in :gh:`116146`.) * Add :c:func:`PyTuple_FromArray` to create a :class:`tuple` from an array. (Contributed by Victor Stinner in :gh:`111489`.) +* Add a new module export hook, + :c:func:`PyModExport_* `. + + (Contributed by Petr Viktorin in :pep:`793` and :gh:`140550`.) + * Add functions that are guaranteed to be safe for use in :c:member:`~PyTypeObject.tp_traverse` handlers: :c:func:`PyObject_GetTypeData_DuringGC`, @@ -1815,19 +2500,65 @@ New features It should only be used for debugging. (Contributed by Victor Stinner in :gh:`141070`.) +* Implement :pep:`820`: ``PySlot`` -- Unified slot system for the C API. + See :ref:`capi-slots` for documentation. + + This adds: + + * The :c:type:`PySlot` struct; + * the :c:func:`PyType_FromSlots` function; + * new slot IDs: :c:macro:`Py_slot_end`, :c:macro:`Py_slot_invalid`; + :c:macro:`Py_slot_subslots`, :c:macro:`Py_tp_slots`, + :c:macro:`Py_mod_slots`; + :c:macro:`Py_tp_name`, :c:macro:`Py_tp_basicsize`, + :c:macro:`Py_tp_extra_basicsize`, :c:macro:`Py_tp_itemsize`, + :c:macro:`Py_tp_flags`, :c:macro:`Py_tp_metaclass`, + :c:macro:`Py_tp_module`; + * convenience macros: :c:macro:`PySlot_DATA`, :c:macro:`PySlot_FUNC`, + :c:macro:`PySlot_SIZE`, :c:macro:`PySlot_INT64`, :c:macro:`PySlot_UINT64`, + :c:macro:`PySlot_STATIC_DATA`, :c:macro:`PySlot_END`, + :c:macro:`PySlot_PTR`, :c:macro:`PySlot_PTR_STATIC`. + + The :c:func:`PyModule_FromSlotsAndSpec` function and + ``PyModExport`` :ref:`module export hook ` also + use the new :c:type:`!PySlot` struct. + + These following functions are :term:`soft deprecated`: + + * :c:func:`PyType_FromSpec` + * :c:func:`PyType_FromSpecWithBases` + * :c:func:`PyType_FromModuleAndSpec` + * :c:func:`PyType_FromMetaclass` + * :c:func:`PyModule_FromDefAndSpec` + * :c:func:`PyModule_FromDefAndSpec2` + * :c:func:`PyModule_ExecDef` + + + The slots :c:macro:`Py_tp_bases` and :c:macro:`Py_tp_base` are now + equivalent: they can be set either to a single type or a tuple of types. + The :c:macro:`Py_tp_bases` slot is preferred; the other is ignored if both + are specified. + + (Contributed by Petr Viktorin in :gh:`149044`.) + * Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the stack protection base address and stack protection size of a Python thread state. (Contributed by Victor Stinner in :gh:`139653`.) -* Add :c:func:`PyUnstable_SetImmortal` C-API function to mark objects as :term:`immortal`. +* Add :c:func:`PyUnstable_SetImmortal` C API function to mark objects as :term:`immortal`. (Contributed by Kumar Aditya in :gh:`143300`.) * Restore private provisional ``_Py_InitializeMain()`` function removed in Python 3.14. (Contributed by Victor Stinner in :gh:`142417`.) +* Add :c:func:`PyUnstable_DumpTraceback` and + :c:func:`PyUnstable_DumpTracebackThreads` functions to output Python + stacktraces. + (Contributed by Alex Malyshev in :gh:`145559`.) + Changed C APIs -------------- @@ -1839,14 +2570,6 @@ Changed C APIs for ``NULL`` should be updated to call :c:macro:`PyDateTime_IMPORT` instead. (Contributed by Kumar Aditya in :gh:`141563`.) -Porting to Python 3.15 ----------------------- - -* Private functions promoted to public C APIs: - - The |pythoncapi_compat_project| can be used to get most of these new - functions on Python 3.14 and older. - Removed C APIs -------------- @@ -1856,12 +2579,12 @@ Removed C APIs * :c:func:`!PyUnicode_AsDecodedObject`: Use :c:func:`PyCodec_Decode` instead. * :c:func:`!PyUnicode_AsDecodedUnicode`: - Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Decode` instead; note that some codecs (for example, "base64") may return a type other than :class:`str`, such as :class:`bytes`. * :c:func:`!PyUnicode_AsEncodedObject`: Use :c:func:`PyCodec_Encode` instead. * :c:func:`!PyUnicode_AsEncodedUnicode`: - Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + Use :c:func:`PyCodec_Encode` instead; note that some codecs (for example, "base64") may return a type other than :class:`bytes`, such as :class:`str`. (Contributed by Stan Ulbrych in :gh:`133612`.) @@ -1973,8 +2696,8 @@ Deprecated C APIs - :c:macro:`Py_ALIGNED`: Prefer ``alignas`` instead. - :c:macro:`PY_FORMAT_SIZE_T`: Use ``"z"`` directly. - - :c:macro:`Py_LL` & :c:macro:`Py_ULL`: - Use standard suffixes, ``LL`` & ``ULL``. + - :c:macro:`Py_LL` and :c:macro:`Py_ULL`: + Use standard suffixes, ``LL`` and ``ULL``. - :c:macro:`PY_LONG_LONG`, :c:macro:`PY_LLONG_MIN`, :c:macro:`PY_LLONG_MAX`, :c:macro:`PY_ULLONG_MAX`, :c:macro:`PY_INT32_T`, :c:macro:`PY_UINT32_T`, :c:macro:`PY_INT64_T`, :c:macro:`PY_UINT64_T`, :c:macro:`PY_SIZE_MAX`: @@ -1999,7 +2722,7 @@ Build changes ============= * Removed implicit fallback to the bundled copy of the ``libmpdec`` library. - Now this should be explicitly enabled with :option:`--with-system-libmpdec` + Now this should be explicitly enabled with :option:`!--with-system-libmpdec` set to ``no`` or with :option:`!--without-system-libmpdec`. (Contributed by Sergey B Kirpichev in :gh:`115119`.) @@ -2017,12 +2740,24 @@ Build changes At runtime, huge pages must be explicitly enabled by setting the :envvar:`PYTHON_PYMALLOC_HUGEPAGES` environment variable to ``1``. -* Annotating anonymous mmap usage is now supported if Linux kernel supports +* Annotating anonymous mmap usage is now supported if the Linux kernel supports :manpage:`PR_SET_VMA_ANON_NAME ` (Linux 5.17 or newer). Annotations are visible in ``/proc//maps`` if the kernel supports the feature - and :option:`-X dev <-X>` is passed to the Python or Python is built in :ref:`debug mode `. + and :option:`-X dev <-X>` is passed to Python, + or Python is built in :ref:`debug mode `. (Contributed by Donghee Na in :gh:`141770`.) +* CPython is now built with frame pointers enabled by default + (:pep:`831`). Pass :option:`--without-frame-pointers` to opt out. + + Authors of C extensions and native libraries built with custom build + systems should ensure the unwind chain is intact. + This is usually done by adding ``-fno-omit-frame-pointer`` and + similar flags to ``CFLAGS``. See :option:`--without-frame-pointers` + documentation for the specific flags Python uses. + + (Contributed by Pablo Galindo Salgado and Savannah Ostrowski in :gh:`149201`.) + .. _whatsnew315-windows-tail-calling-interpreter: * 64-bit builds using Visual Studio 2026 (MSVC 18) may now use the new @@ -2085,3 +2820,11 @@ that may require changes to your code. with argument ``altchars=b'-_'`` (this works with older Python versions) to make padding required. (Contributed by Serhiy Storchaka in :gh:`73613`.) + +* Since :meth:`unittest.TestCase.assertWarns` and + :meth:`unittest.TestCase.assertWarnsRegex` no longer swallow warnings that + do not match the specified category or regex, your tests may start leaking + some warnings that were previously masked. + Use warning filters to silence them or additional :meth:`!assertWarns*` + to catch and check them. + (Contributed by Serhiy Storchaka in :gh:`143231`.) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst new file mode 100644 index 000000000000000..3262acd87d6d49f --- /dev/null +++ b/Doc/whatsnew/3.16.rst @@ -0,0 +1,986 @@ + +**************************** + What's new in Python 3.16 +**************************** + +:Editor: TBD + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the issue number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :gh:`12345`.) + + This saves the maintainer the effort of going through the VCS log + when researching a change. + +This article explains the new features in Python 3.16, compared to 3.15. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.16 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary --- release highlights +============================== + +.. This section singles out the most important changes in Python 3.16. + Brevity is key. + + +.. PEP-sized items next. + + + +New features +============ + + + +Other language changes +====================== + +* The :func:`iter` function now accepts the *stop_exception* parameter. + The created iterator stops when the callable raises the specified exception. + The second parameter is now named *stop_value* and can be passed by keyword. + :func:`aiter` now accepts the same *stop_value* and *stop_exception* + parameters, calling an asynchronous callable and awaiting the result. + (Contributed by Serhiy Storchaka in :gh:`64862`.) + +* :meth:`memoryview.cast` now allows casting a multidimensional + F-contiguous view to a one-dimensional view. + (Contributed by Jaemin Park in :gh:`91484`.) + +* :meth:`memoryview.cast` now accepts an *order* parameter. With + ``order='F'`` it returns a zero-copy Fortran-contiguous (column-major) + view of a flat buffer, which is useful for interfacing with column-major + libraries. + (Contributed by Serhiy Storchaka in :gh:`78959`.) + +* :ref:`Frame objects ` now support :mod:`weak references + `. This allows associating extra data with active frames, + for example in debuggers, without keeping the frames (and everything + they reference) alive indefinitely. + (Contributed by Łukasz Langa in :gh:`102960`.) + + +New modules +=========== + +* None yet. + + +Improved modules +================ + +asyncio +------- + +* Add the *mode* parameter to :meth:`asyncio.loop.create_unix_server` and + :func:`asyncio.start_unix_server` to set the permissions of the Unix + socket file created for *path*. + (Contributed by Sam Bull in :gh:`94984`.) + + +codecs +------ + +* On platforms that provide the C library's :manpage:`iconv(3)` function, + every encoding known to ``iconv`` for which Python has no built-in codec + is now available (for example ``cp1133``). + Prefixing an encoding name with ``iconv:`` forces the ``iconv``-based codec + even when a built-in codec of the same name exists. + (Contributed by Serhiy Storchaka in :gh:`152997`.) + + +concurrent.futures +------------------ + +* The iterator returned by :meth:`concurrent.futures.Executor.map` is no longer + automatically closed if a function call raises an exception. + Use method :meth:`!close` to explicitly close the iterator. + (Contributed by xzmeng and Serhiy Storchaka in :gh:`108518`.) + + +csv +--- + +* :meth:`csv.Sniffer.sniff` now deduces the dialect by trial parsing + instead of heuristics based on matching isolated fragments + and on character frequencies, + so the result is consistent with how :func:`csv.reader` will parse the sample. + It can now detect the *escapechar* parameter, + accepts non-ASCII delimiters in the *delimiters* argument, + no longer misdetects the delimiter + when the sample contains delimiter characters inside quoted fields, + and no longer takes quadratic time on quoted samples. + The results may differ from those of earlier Python versions. + (Contributed by Serhiy Storchaka in :gh:`83273`.) + +* :meth:`csv.Sniffer.sniff` now detects the *lineterminator* parameter + by a majority vote among the line endings of the sample, + instead of always returning ``'\r\n'``. + (Contributed by Serhiy Storchaka in :gh:`75008`.) + + +ctypes +------ + +* Add :func:`ctypes.util.struct` for generating :class:`~ctypes.Structure` types + from an annotation-based syntax, similar to how the :mod:`dataclasses` module + is used. + (Contributed by Peter Bierma in :gh:`104533`.) +* Add :func:`ctypes.util.wrap_dll_function` for generating function pointers + through a function signature. + (Contributed by Peter Bierma in :gh:`153903`.) + + +curses +------ + +* The :mod:`curses` character-cell window methods now accept a full character + cell --- a spacing character optionally followed by combining characters --- + in addition to a single integer or byte character. This affects + :meth:`~curses.window.addch`, :meth:`~curses.window.bkgd`, + :meth:`~curses.window.bkgdset`, :meth:`~curses.window.border`, + :meth:`~curses.window.box`, :meth:`~curses.window.echochar`, + :meth:`~curses.window.hline`, :meth:`~curses.window.insch` and + :meth:`~curses.window.vline`. + Also add the wide-character read methods :meth:`~curses.window.get_wstr` and + :meth:`~curses.window.in_wstr`, the counterparts of + :meth:`~curses.window.getstr` and :meth:`~curses.window.instr` that return a + :class:`str` rather than :class:`bytes`, + and the module functions :func:`curses.erasewchar`, :func:`curses.killwchar` + and :func:`curses.wunctrl`, the wide-character counterparts of + :func:`curses.erasechar`, :func:`curses.killchar` and :func:`curses.unctrl`. + On a narrow (non-ncursesw) build the character cell holds a single character + without combining marks, representable as one byte in the window's encoding, + and :meth:`~curses.window.in_wstr` returns its decoded text. + (Contributed by Serhiy Storchaka in :gh:`151757`.) + +* Add the :class:`curses.complexchar` type, representing a styled + character cell (its text, attributes and color pair), and the window + methods :meth:`~curses.window.in_wch` and :meth:`~curses.window.getbkgrnd` + that return one --- the counterparts of + :meth:`~curses.window.inch` and :meth:`~curses.window.getbkgd`. The + character-cell methods, such as :meth:`~curses.window.addch` and + :meth:`~curses.window.border`, now also accept a + :class:`~curses.complexchar`. These work whether or not Python was built + against a wide-character-aware curses library; on a narrow build a cell holds a + single character representable as one byte in the window's encoding (so only + 8-bit locales are supported). + (Contributed by Serhiy Storchaka in :gh:`152233`.) + +* Add the :class:`curses.complexstr` type, an immutable run of styled cells + (the string counterpart of :class:`~curses.complexchar`), and the window + method :meth:`~curses.window.in_wchstr` that returns one. The string-cell + methods :meth:`~curses.window.addstr`, :meth:`~curses.window.addnstr`, + :meth:`~curses.window.insstr` and :meth:`~curses.window.insnstr` now also + accept a :class:`~curses.complexstr`. Like :class:`~curses.complexchar`, it + works whether or not Python was built against a wide-character-aware curses + library. + (Contributed by Serhiy Storchaka in :gh:`152233`.) + +* The wide-character :mod:`curses` functions and methods + :meth:`~curses.window.get_wch`, :meth:`~curses.window.get_wstr`, + :func:`curses.unget_wch`, :func:`curses.erasewchar`, + :func:`curses.killwchar` and :func:`curses.wunctrl` now also work when Python + is not built against a wide-character-aware curses library, on an 8-bit + locale, where each character is a single byte in the relevant encoding. + :func:`curses.ungetch` now also accepts a one-character string, like + :func:`curses.unget_wch`; on a wide-character build it can be any character + (previously a multibyte character raised :exc:`OverflowError`). + (Contributed by Serhiy Storchaka in :gh:`152470`.) + +* Add support for multiple terminals to the :mod:`curses` module: + the new functions :func:`curses.newterm`, :func:`curses.set_term` + and :func:`curses.new_prescr`, + the corresponding :ref:`screen ` object, + and the :meth:`window.use() ` method. + (Contributed by Serhiy Storchaka in :gh:`90092`.) + +* Add the :mod:`curses` window methods :meth:`~curses.window.attr_get`, + :meth:`~curses.window.attr_set`, :meth:`~curses.window.attr_on`, + :meth:`~curses.window.attr_off` and :meth:`~curses.window.color_set`, which + pass the color pair as a separate argument instead of packing it into the + attribute value, and the corresponding ``WA_*`` attribute constants. + (Contributed by Serhiy Storchaka in :gh:`152219`.) + +* Add the :func:`curses.term_attrs` function, which returns the supported + video attributes as :ref:`WA_* ` values, the + counterpart of :func:`curses.termattrs`. + (Contributed by Serhiy Storchaka in :gh:`152332`.) + +* Add the ``WACS_*`` constants to the :mod:`curses` module, the counterparts of + the :ref:`ACS_* ` line-drawing codes as + :class:`curses.complexchar` cells. + (Contributed by Serhiy Storchaka in :gh:`155863`.) + +* Add the :mod:`curses` functions :func:`curses.alloc_pair`, + :func:`curses.find_pair`, :func:`curses.free_pair` and + :func:`curses.reset_color_pairs` for dynamic color-pair management, + available when built against a wide-character ncurses with extended-color + support. + (Contributed by Serhiy Storchaka in :gh:`151774`.) + +* Add the :mod:`curses` functions :func:`~curses.scr_dump`, + :func:`~curses.scr_restore`, :func:`~curses.scr_init` and + :func:`~curses.scr_set`, which dump the whole screen to a file and restore it. + (Contributed by Serhiy Storchaka in :gh:`152260`.) + +* Add the :mod:`curses` window method :meth:`~curses.window.dupwin`, which + returns a new window that is an independent duplicate of an existing one. + (Contributed by Serhiy Storchaka in :gh:`152258`.) + +* Add the soft-label-key functions to the :mod:`curses` module, which manage a + row of labels along the bottom line of the screen: + :func:`~curses.slk_init`, :func:`~curses.slk_set`, :func:`~curses.slk_label`, + :func:`~curses.slk_refresh`, :func:`~curses.slk_noutrefresh`, + :func:`~curses.slk_clear`, :func:`~curses.slk_restore`, + :func:`~curses.slk_touch`, the attribute functions + :func:`~curses.slk_attron`, :func:`~curses.slk_attroff`, + :func:`~curses.slk_attrset`, :func:`~curses.slk_attr`, + :func:`~curses.slk_attr_on`, :func:`~curses.slk_attr_off`, + :func:`~curses.slk_attr_set`, and :func:`~curses.slk_color`. + (Contributed by Serhiy Storchaka in :gh:`152263`.) + +* Add the :mod:`curses` key-management functions :func:`~curses.define_key`, + :func:`~curses.key_defined` and :func:`~curses.keyok`, available when built + against an ncurses with ``NCURSES_EXT_FUNCS``. + (Contributed by Serhiy Storchaka in :gh:`152334`.) + +* Add the :func:`curses.has_mouse` function and the + :meth:`curses.window.mouse_trafo` method, completing the :mod:`curses` + mouse interface. + (Contributed by Serhiy Storchaka in :gh:`152325`.) + +* Add :mod:`curses` functions and window methods that report state which could + previously only be set, such as :meth:`curses.window.is_keypad`, + :meth:`curses.window.getparent` and :func:`curses.is_cbreak`, + available when built against an ncurses with ``NCURSES_EXT_FUNCS``. + (Contributed by Serhiy Storchaka in :gh:`151776`.) + +* Add :func:`curses.nofilter`, which undoes the effect of :func:`curses.filter`. + (Contributed by Serhiy Storchaka in :gh:`151744`.) + +* :class:`curses.textpad.Textbox` now supports entering and reading back the + full Unicode range, including combining characters, when curses is built with + wide-character support. + (Contributed by Serhiy Storchaka in :gh:`133031`.) + + +encodings +--------- + +* Add the ``utf-7-imap`` codec, implementing the modified UTF-7 encoding + used for international IMAP4 mailbox names (:rfc:`3501`). + (Contributed by Serhiy Storchaka in :gh:`66788`.) + + +gzip +---- + +* :func:`gzip.open` now accepts an optional argument ``mtime`` + which is passed on to the constructor of the :class:`~gzip.GzipFile` class. + (Contributed by Marin Misur in :gh:`91372`.) + + +imaplib +------- + +* Add the :meth:`~imaplib.IMAP4.id` method, + a wrapper for the ``ID`` command (:rfc:`2971`). + (Contributed by Serhiy Storchaka in :gh:`98092`.) + +* Add the :meth:`~imaplib.IMAP4.move` method, + a wrapper for the ``MOVE`` command (:rfc:`6851`). + (Contributed by Serhiy Storchaka in :gh:`77508`.) + +* Add the :meth:`~imaplib.IMAP4.login_plain` method, which authenticates + using the ``PLAIN`` SASL mechanism (:rfc:`4616`) and supports non-ASCII + user names and passwords. + (Contributed by Przemysław Buczkowski and Serhiy Storchaka in :gh:`89869`.) + +* Non-ASCII mailbox names are now automatically encoded as modified UTF-7 + (:rfc:`3501`, section 5.1.3) in the default mode, so international mailbox + names can be passed as ordinary :class:`str` without enabling ``UTF8=ACCEPT`` + (under which they are sent as UTF-8). + (Contributed by Serhiy Storchaka in :gh:`49555`.) + +* The :meth:`~imaplib.IMAP4.copy`, :meth:`~imaplib.IMAP4.move`, + :meth:`~imaplib.IMAP4.fetch`, :meth:`~imaplib.IMAP4.store`, + :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort`, + :meth:`~imaplib.IMAP4.thread` and :meth:`~imaplib.IMAP4.expunge` methods + now accept a keyword-only *uid* argument that selects the corresponding + ``UID`` command, as a more convenient alternative to + :meth:`~imaplib.IMAP4.uid`. + (Contributed by Serhiy Storchaka in :gh:`153502`.) + +* :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` + and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) + now encode :class:`str` search criteria to the declared *charset*, + so international search text can be passed as an ordinary :class:`str`. + When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), + the criteria are sent using the connection encoding instead. + (Contributed by Serhiy Storchaka in :gh:`153494`.) + +* Command methods now accept structured arguments, + so the module takes care of quoting instead of the caller. + A *message_set* and lists of flags or other atoms + can be passed as sequences instead of preformatted strings, + and the :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.fetch`, + :meth:`~imaplib.IMAP4.sort`, :meth:`~imaplib.IMAP4.thread` and + :meth:`~imaplib.IMAP4.uid` methods accept a *params* keyword argument + that substitutes and quotes ``?`` placeholders, + in the manner of :mod:`sqlite3` parameter substitution. + (Contributed by Serhiy Storchaka in :gh:`153521`.) + + +io +-- + +* Add :meth:`io.BytesIO.peek` method to read without advancing position. + (Contributed by Marcel Martin in :gh:`90533`.) + + +ipaddress +--------- + +* Add :meth:`~ipaddress.IPv4Network.next_network` and + :meth:`~ipaddress.IPv6Network.next_network` methods to find the next nearest + network with a specific prefix size. + (Contributed by Faisal Mahmood in :gh:`87027`.) + + +logging +------- + +* :class:`~logging.handlers.TimedRotatingFileHandler` now uses the creation + time instead of the last modification time of an existing log file as + the basis for the first rotation after handler creation, if supported by + the OS and file system. + This allows it to be used in short-running programs that start and end + before the rotation interval expires. + (Contributed by Iván Márton and Serhiy Storchaka in :gh:`84649`.) + + +lzma +---- + +* Add support of new BCJ filters ARM64 and RISC-V via + :const:`!lzma.FILTER_ARM64` and :const:`!lzma.FILTER_RISCV`. Note that the + new filters will work only if the runtime library supports them. ARM64 filter + requires ``lzma`` 5.4.0 or newer while RISC-V requires 5.6.0 or newer. + (Contributed by Chien Wong in :gh:`115988`.) + + +math +---- + +* Added trigonometric functions that work in units of half turns, rather than + radians. The new functions :func:`math.acospi`, :func:`math.asinpi`, + :func:`math.atanpi`, and :func:`math.atan2pi` return half-turn angles. The + new functions :func:`math.cospi`, :func:`math.sinpi`, and :func:`math.tanpi` + take half-turn angle arguments. These functions are recommended by IEEE + 754-2019 and standardized in C23. + (Contributed by Jeff Epler in :gh:`150534`.) + + +multiprocessing +--------------- + +* Add the optional *buffersize* parameter to + :meth:`multiprocessing.pool.Pool.imap` and + :meth:`multiprocessing.pool.Pool.imap_unordered` to limit the number of + submitted tasks whose results have not yet been yielded. If the buffer is + full, iteration over the *iterable* pauses until a result is yielded from + the buffer. To fully utilize pool's capacity when using this feature, set + *buffersize* at least to the number of processes in pool (to consume + *iterable* as you go), or even higher (to prefetch the next + ``N=buffersize-processes`` arguments). + (Contributed by Oleksandr Baltian in :gh:`136871`.) + + +os +-- + +* Add :func:`os.pidfd_getfd` for duplicating a file descriptor from another + process via a pidfd. Available on Linux 5.6+. + (Contributed by Maurycy Pawłowski-Wieroński in :gh:`149464`.) + + +pydoc +----- + +* Modernize the HTML output of :mod:`pydoc`. The pages generated by the + :mod:`pydoc` HTTP server now use semantic HTML5 markup and a new style + sheet based on the python-docs-theme used by `docs.python.org + `__, with dark mode support. Class members + inherited from other classes are collapsed by default. + (Contributed by Serhiy Storchaka in :gh:`153906`.) + + +re +-- + +* :mod:`re` now supports set operations and nested sets in character classes, + as described in `Unicode Technical Standard #18 + `__: set difference (``[A--B]``), + intersection (``[A&&B]``) and union (``[A||B]``), where an operand may be a + nested set written in square brackets. For example, ``[a-z--[aeiou]]`` + matches an ASCII lowercase consonant. + (Contributed by Serhiy Storchaka in :gh:`152100`.) + +* Regular expressions now support Unicode property escapes ``\p{...}`` and + ``\P{...}``, which match a character by a Unicode property -- for example + ``\p{Lu}`` (an uppercase letter), ``\p{Cased}`` or ``\p{ASCII}``. See + :ref:`the regular expression syntax ` for the supported + properties. + (Contributed by Serhiy Storchaka in :gh:`95555`.) + + +shlex +----- + +* Add keyword-only parameter *force* to :func:`shlex.quote` to force quoting + a string, even if it is already safe for a shell without being quoted. + (Contributed by Jay Berry in :gh:`148846`.) + + +sqlite3 +------- + +* :class:`sqlite3.Blob` now supports negative-step slices for reading and + writing (e.g. ``blob[9:0:-2]``). Previously, such slices would raise + :exc:`SystemError` or :exc:`ValueError`. + (Contributed by Jiseok CHOI in :gh:`150449`.) + + +symtable +-------- + +* :func:`symtable.symtable` now accepts an AST object, + like the builtin :func:`compile`. + (Contributed by Serhiy Storchaka in :gh:`153844`.) + + +tkinter +------- + +* Added many :class:`tkinter.ttk.Treeview` methods wrapping the enhanced + ``ttk::treeview`` widget commands from Tk 9.1, such as + :meth:`~tkinter.ttk.Treeview.sort`, :meth:`~tkinter.ttk.Treeview.search`, + :meth:`~tkinter.ttk.Treeview.expand`, :meth:`~tkinter.ttk.Treeview.collapse`, + :meth:`~tkinter.ttk.Treeview.hide`, :meth:`~tkinter.ttk.Treeview.unhide`, and + methods for cell focus, selection and tagging. The + :meth:`~tkinter.ttk.Treeview.expand` and :meth:`~tkinter.ttk.Treeview.collapse` + methods (without recursion) also work on Tk older than 9.1. + (Contributed by Serhiy Storchaka in :gh:`151910`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.edit_canundo` + and :meth:`~tkinter.Text.edit_canredo` which return whether an undo or redo + is possible. + (Contributed by Serhiy Storchaka in :gh:`151674`.) + +* Added new :class:`!tkinter.Text` methods :meth:`~tkinter.Text.sync` and + :meth:`~tkinter.Text.pendingsync` which control and report the + synchronization of the displayed view with the underlying text. + (Contributed by Serhiy Storchaka in :gh:`151675`.) + +* Added the :meth:`ttk.Style.theme_styles + ` method which returns the list of styles + defined in a theme. + (Contributed by Serhiy Storchaka in :gh:`151920`.) + +* Added new :class:`!tkinter.Canvas` methods :meth:`~tkinter.Canvas.rchars` + which replaces the text or coordinates of canvas items, and + :meth:`~tkinter.Canvas.rotate` which rotates the coordinates of canvas items. + (Contributed by Serhiy Storchaka in :gh:`151876`.) + +* Added a :meth:`!validate` method to the :class:`!tkinter.Entry` and + :class:`!tkinter.Spinbox` widgets, which forces an evaluation of the + validation command. + (Contributed by Serhiy Storchaka in :gh:`151878`.) + +* Added the :meth:`tkinter.Menu.postcascade` method, and the + :meth:`~tkinter.Misc.tk_scaling` and :meth:`~tkinter.Misc.tk_inactive` + methods which respectively query or set the display scaling factor and + report the user idle time. + (Contributed by Serhiy Storchaka in :gh:`151881`.) + +* Added the :meth:`!tk_print` method to :class:`tkinter.Canvas` and + :class:`tkinter.Text` which prints the contents of the widget using the + native print dialog. It requires Tk 8.7/9.0 or newer. + (Contributed by Serhiy Storchaka in :gh:`153256`.) + +* Added new window-management methods :meth:`~tkinter.Misc.winfo_isdark` + (dark mode detection), :meth:`~tkinter.Wm.wm_iconbadge` (application icon + badge) and :meth:`~tkinter.Wm.wm_stackorder` (toplevel stacking order). + (Contributed by Serhiy Storchaka in :gh:`151874`.) + +* Added the :meth:`~tkinter.Misc.tk_appname`, + :meth:`~tkinter.Misc.tk_useinputmethods` and :meth:`~tkinter.Misc.tk_caret` + methods, exposing the application send name, the X Input Methods state and + the input method caret location. + (Contributed by Serhiy Storchaka in :gh:`151886`.) + +* Added support for more options in :class:`!tkinter.PhotoImage` methods: the + *format* parameter of :meth:`~tkinter.PhotoImage.put`, the *metadata* + parameter of :meth:`~tkinter.PhotoImage.put`, :meth:`~tkinter.PhotoImage.read`, + :meth:`~tkinter.PhotoImage.write` and :meth:`~tkinter.PhotoImage.data`, and + the *withalpha* parameter of :meth:`~tkinter.PhotoImage.get`. + (Contributed by Serhiy Storchaka in :gh:`151890`.) + +* Added the :meth:`~tkinter.PhotoImage.redither` method which recalculates the + dithered image when its data was supplied in pieces. + (Contributed by Serhiy Storchaka in :gh:`151888`.) + +* :class:`tkinter.OptionMenu` now accepts arbitrary :class:`!tkinter.Menubutton` + options as keyword arguments, which can also override its default appearance. + (Contributed by Serhiy Storchaka in :gh:`101284`.) + +* The :mod:`tkinter.simpledialog` dialogs were modernized to match the look + and feel of the native Tk dialogs. + :class:`!tkinter.simpledialog.SimpleDialog` and the + :func:`~tkinter.simpledialog.askinteger`, + :func:`~tkinter.simpledialog.askfloat` and + :func:`~tkinter.simpledialog.askstring` dialogs are now built from the themed + :mod:`tkinter.ttk` widgets instead of the classic :mod:`tkinter` widgets; + the :class:`!tkinter.simpledialog.Dialog` base class still defaults to the + classic widgets for compatibility. Both :class:`!Dialog` and + :class:`!SimpleDialog` gained a *use_ttk* parameter that selects between the + classic Tk widgets and the themed ttk widgets. :class:`!SimpleDialog` also + gained *bitmap* and + *detail* parameters, draws the standard icons with themed images in the + ttk version, and accepts mappings of button options as *buttons* entries. + (Contributed by Serhiy Storchaka in :gh:`59396`.) + +* The :class:`!tkinter.filedialog.FileDialog` dialog and its + :class:`!tkinter.filedialog.LoadFileDialog` and + :class:`!tkinter.filedialog.SaveFileDialog` subclasses are now built from the + themed :mod:`tkinter.ttk` widgets by default instead of the classic + :mod:`tkinter` widgets, and gained a *use_ttk* parameter to select between + them. + (Contributed by Serhiy Storchaka in :gh:`59396`.) + +* Added the :mod:`tkinter.systray` module which provides the + :class:`~tkinter.systray.SysTrayIcon` class as an interface to the system + tray icon and the :func:`~tkinter.systray.notify` function which sends a + desktop notification. They require Tk 8.7/9.0 or newer. + (Contributed by Serhiy Storchaka in :gh:`153259`.) + +* :class:`tkinter.scrolledtext.ScrolledText` gained a *use_ttk* parameter to use + the themed :mod:`tkinter.ttk` frame and scroll bar instead of the classic + :mod:`tkinter` widgets. + (Contributed by Serhiy Storchaka in :gh:`59396`.) + +* :class:`tkinter.font.Font` can now wrap a font description without creating a + new named font, by passing it as *font* with ``exists=True`` and no *name*. + This avoids a loss of precision in :meth:`~tkinter.font.Font.actual`, + :meth:`~tkinter.font.Font.measure` and :meth:`~tkinter.font.Font.metrics`. + (Contributed by Serhiy Storchaka in :gh:`143990`.) + +* Added the :mod:`tkinter.fontchooser` module which provides the + :class:`~tkinter.fontchooser.FontChooser` class as an interface to the + native font selection dialog. + (Contributed by Serhiy Storchaka in :gh:`72880`.) + +* Values of several Tcl object types returned by :mod:`tkinter` are now + converted to the corresponding Python type instead of being wrapped in a + :class:`!_tkinter.Tcl_Obj`: ``index``, ``window``, ``nsName`` and + ``parsedVarName`` objects to :class:`str`, and ``pixel`` screen distances + with no unit suffix to :class:`int` or :class:`float`. + (Contributed by Serhiy Storchaka in :gh:`153513`.) + +xml +--- + +* Add support for multiple multi-byte encodings in the :mod:`XML parser + `: "cp932", "cp949", "cp950", "Big5", "EUC-JP", + "GB2312", "GBK", "johab", and "Shift_JIS". + Add partial support (only BMP characters) for multi-byte encodings + "Big5-HKSCS", "EUC_JIS-2004", "EUC_JISX0213", "Shift_JIS-2004", + "Shift_JISX0213", "utf-8-sig" and non-standard aliases like "UTF8" + (without hyphen). + The parser now raises :exc:`ValueError` for known unsupported + multi-byte encodings such as "ISO-2022-JP" or "raw-unicode-escape" + instead of failing later, when encountering non-ASCII data. + (Contributed by Serhiy Storchaka in :gh:`62259`.) + +* :mod:`xml.dom.minidom` now conforms closer to the DOM Level 1 specification. + It checks names passed to the factory methods, + rejects inserting or setting a node created by another document + or making a node a descendant of itself, + rejects a second element or document type child of a document + and a notation child of a document fragment, + reports attributes defaulted in the DTD + and whether an attribute was given in the start tag, + and implements :class:`!EntityReference` nodes + and :meth:`!Document.createEntityReference`. + (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) + +* Add :meth:`!GetSpecifiedAttributeCount` method + to the :mod:`XML parser ` objects. + It tells how many of the reported attributes were given in the start tag + rather than defaulted from the DTD. + (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) + +zipfile +------- + +* Add :meth:`ZipFile.remove() ` to remove a member + from an archive's central directory, and + :meth:`ZipFile.repack() ` to reclaim the space used + by the local file entries of removed members. + (Contributed by Danny Lin in :gh:`51067`.) + +.. Add improved modules above alphabetically, not here at the end. + +Optimizations +============= + +re +-- + +* Character class escapes (``\d``, ``\D``, ``\s``, ``\S``, ``\w`` and ``\W``) + outside a character set, and character sets containing a single such escape + (such as ``[\d]`` or ``[^\s]``), are now compiled to a single ``CATEGORY`` + opcode instead of being wrapped in an ``IN`` block. This speeds up matching + of patterns such as ``\d+`` and reduces the size of the compiled byte code. + (Contributed by Serhiy Storchaka in :gh:`152033` and Pieter Eendebak in + :gh:`152056`.) + + +Removed +======== + +annotationlib +------------- + +* The :meth:`!annotationlib.ForwardRef._evaluate` method + which has been deprecated since Python 3.14. + Use :meth:`annotationlib.ForwardRef.evaluate` + or :func:`typing.evaluate_forward_ref` instead. + +array +----- + +* The ``'u'`` format code (:c:type:`wchar_t`) which has been deprecated in + documentation since Python 3.3 and at runtime since Python 3.13. + Use ``'w'`` format code instead (:c:type:`Py_UCS4`, always 4 bytes). + +asyncio +------- + +* The :func:`!asyncio.iscoroutinefunction` + which has been deprecated since Python 3.14. + Use :func:`inspect.iscoroutinefunction` instead. + +* The event loop policy system, including the + :class:`!asyncio.AbstractEventLoopPolicy`, + :class:`!asyncio.DefaultEventLoopPolicy`, + :class:`!asyncio.WindowsSelectorEventLoopPolicy` and + :class:`!asyncio.WindowsProactorEventLoopPolicy` classes and the + :func:`!asyncio.get_event_loop_policy` and + :func:`!asyncio.set_event_loop_policy` functions, + which have been deprecated since Python 3.14. + Use :func:`asyncio.run` or :class:`asyncio.Runner` with a custom + *loop_factory* instead. + +functools +--------- + +* Remove support for calling the Python implementation of + :func:`functools.reduce` with *function* or *sequence* as keyword arguments, + deprecated since Python 3.14. These parameters are now positional-only. + (Contributed by Nikita Sobolev in :gh:`149537`.) + +logging +------- + +* Remove support for custom logging handlers with the *strm* argument, + deprecated since Python 3.14. + Define handlers with the *stream* argument instead. + (Contributed by Nikita Sobolev in :gh:`149598`.) + +mimetypes +--------- + +* Remove support for undotted extensions in + :meth:`mimetypes.MimeTypes.add_type`, deprecated since Python 3.14. + Valid extensions must now start with a ``'.'`` or be empty; + undotted extensions raise a :exc:`ValueError`. + (Contributed by Nikita Sobolev in :gh:`149720`.) + +shutil +------ + +* The :exc:`!ExecError` exception which has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4. + (Contributed by Stan Ulbrych in :gh:`149567`.) + +symtable +-------- + +* The :meth:`!symtable.Class.get_methods` method + which has been deprecated since Python 3.14. + +sys +--- + +* The :func:`!_enablelegacywindowsfsencoding` function + which has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + (Contributed by Stan Ulbrych in :gh:`149595`.) + +sysconfig +--------- + +* The :func:`!sysconfig.expand_makefile_vars` function + which has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + (Contributed by Stan Ulbrych in :gh:`149499`.) + +tarfile +------- + +* The undocumented and unused :attr:`!tarfile.TarFile.tarfile` attribute + has been deprecated since Python 3.13. + +xml.etree.ElementTree +--------------------- + +* The undocumented :meth:`!xml.etree.ElementTree.ElementTree.write_c14n` method + and the support of ``method="c14n"`` in + :meth:`~xml.etree.ElementTree.ElementTree.write` and + :func:`~xml.etree.ElementTree.tostring`. + They never worked and always raised :exc:`ValueError`. + Use :func:`xml.etree.ElementTree.canonicalize` instead. + (Contributed by Daniel Hillier in :gh:`118318`.) + +.. Add removals above alphabetically, not here at the end. + + +Deprecated +========== + +New deprecations +---------------- + +* :mod:`abc`: + + * :class:`abc.abstractclassmethod`, :class:`abc.abstractstaticmethod`, + and :class:`abc.abstractproperty`, soft-deprecated since Python 3.3, + now raise a :exc:`DeprecationWarning`. + These classes will be removed in Python 3.21; + use :func:`abc.abstractmethod` with :func:`classmethod`, + :func:`staticmethod`, and :class:`property` respectively instead. + +* :mod:`ast`: + + * Classes ``slice``, ``Index``, ``ExtSlice``, ``Suite``, ``Param``, + ``AugLoad`` and ``AugStore``, deprecated since Python 3.9, are no longer + imported by ``from ast import *`` and issue a deprecation warning on + use. The classes are slated for removal in Python 3.21. These types are not + generated by the parser or accepted by the code generator. + * The ``dims`` property of ``ast.Tuple`` objects, deprecated since Python + 3.9, now issues a deprecation warning on use. This property is slated for + removal in 3.21. Use ``ast.Tuple.elts`` instead. + +* :mod:`struct`: + + * The ``'F'`` and ``'D'`` type codes, soft-deprecated since Python 3.15, + are now deprecated. These codes will be removed in Python 3.21. + Use the two-letter forms ``'Zf'`` and ``'Zd'`` instead. + (Contributed by Sergey B Kirpichev in :gh:`121249`.) + +* :mod:`tempfile`: + + * The private ``tempfile._TemporaryFileWrapper`` name is deprecated + and is slated for removal in Python 3.21. + Use the new public :class:`tempfile.TemporaryFileWrapper` instead, + which is the return type of :func:`tempfile.NamedTemporaryFile`. + +* :mod:`tkinter`: + + * :func:`tkinter.filedialog.askopenfiles` is deprecated and slated for + removal in Python 3.19. Opening several files at once is error-prone, and + the returned list cannot be used in a :keyword:`with` statement. Iterate + over the names returned by :func:`~tkinter.filedialog.askopenfilenames` and + open them one by one instead. + (Contributed by Serhiy Storchaka in :gh:`152638`.) + +.. Add deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/pending-removal-in-3.17.rst + +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + +.. include:: ../deprecations/pending-removal-in-3.20.rst + +.. include:: ../deprecations/pending-removal-in-3.21.rst + +.. include:: ../deprecations/pending-removal-in-future.rst + + +Porting to Python 3.16 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +* In :mod:`tkinter`, the *name* parameter of the + :meth:`~tkinter.Misc.wait_variable`, :meth:`~tkinter.Misc.setvar` and + :meth:`~tkinter.Misc.getvar` methods and the *value* parameter of + :meth:`!setvar` are now required. Calling these methods without + them, which formerly defaulted to ``'PY_VAR'`` and ``'1'``, now raises + :exc:`TypeError`. + (Contributed by Serhiy Storchaka in :gh:`152587`.) + +* :mod:`xml.dom.minidom` now raises :exc:`~xml.dom.InvalidCharacterErr` + for a name which is not a valid XML name, + :exc:`~xml.dom.WrongDocumentErr` + for inserting or setting a node created by another document, + :exc:`~xml.dom.InuseAttributeErr` for setting an attribute node + which belongs to another element, + and :exc:`~xml.dom.HierarchyRequestErr` for inserting a node into itself + or its descendant. + It also rejects a second element or document type child of a document + and a notation child of a document fragment. + Such operations formerly succeeded + and produced an invalid document or an endless loop. + On the other hand, :meth:`!Element.removeAttribute` and + :meth:`!Element.removeAttributeNS` no longer raise + :exc:`~xml.dom.NotFoundErr` if there is no matching attribute. + Attributes defaulted in the DTD are no longer omitted when parsing. + (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.) + +* On Windows, seeking a pipe now fails instead of silently appearing to + succeed: :func:`os.lseek` and :meth:`~io.IOBase.seek` raise :exc:`OSError`, + and :meth:`~io.IOBase.seekable` returns ``False``. As a consequence, + opening a pipe in a read-write binary mode (``'r+b'`` or ``'w+b'``) now + raises :exc:`io.UnsupportedOperation` unless buffering is disabled. + (Contributed by An Long in :gh:`86768`.) + + +Build changes +============= + +* Remove the bundled copy of the libmpdec_ decimal library from the CPython source tree + to simplify maintenance and updates. The :mod:`decimal` module will now + unconditionally use the system's libmpdec decimal library. Also remove the + now unused :option:`!--with-system-libmpdec` :program:`configure` flag. + This change has no impact on binary releases of Python, which have been + built against a separate copy of libmpdec for the past several releases. + + (Contributed by Sergey B Kirpichev in :gh:`115119`.) + + .. _libmpdec: https://www.bytereef.org/mpdecimal/ + +* Add a :option:`--with-build-details-suffix` configure flag to allow + Linux distributions that co-install multiple versions of Python in the + same tree to avoid ``build-details.json`` clashes. + + (Contributed by Stefano Rivera in :gh:`131372`.) + +* Add the :option:`--with-curses` :program:`configure` option to select the + curses backend for the :mod:`curses` and :mod:`curses.panel` modules. + In addition to ``ncursesw`` and ``ncurses``, it can now build against the + system's native ``curses`` library (for example on NetBSD or Solaris), with + wide-character support when the library provides it. :option:`--without-curses` + excludes the modules from the build. + + (Contributed by Serhiy Storchaka in :gh:`136687`.) + +* Add the :option:`--with-zlib` and :option:`--with-bzip2` :program:`configure` + options to select the zlib and bzip2 implementations used to build the + :mod:`zlib` and :mod:`bz2` modules. In addition to the default system + libraries, they can build against `zlib-rs + `__ and `libbzip2-rs + `__, and + ``--with-zlib=zlib-ng`` verifies that the detected zlib is `zlib-ng + `__. :option:`--without-zlib` and + :option:`--without-bzip2` exclude the modules from the build. + + (Contributed by Stan Ulbrych in :gh:`139314`.) + + +C API changes +============= + +New features +------------ + +* TODO + +Porting to Python 3.16 +---------------------- + +* :c:func:`PyType_ClearCache` is now a no-op as the type cache is now + implemented per-type. It still returns the current version tag. + +Deprecated C APIs +----------------- + +* :c:func:`PyGen_New`, :c:func:`PyGen_NewWithQualName`, :c:func:`PyCoro_New`, + and :c:func:`PyAsyncGen_New` are deprecated. + They are scheduled for removal in 3.18. + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.19.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.20.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + +Removed C APIs +-------------- + +* The :c:var:`!PyUnstable_ExecutableKinds` array, as well as the macros + :c:macro:`!PyUnstable_EXECUTABLE_KIND_SKIP`, + :c:macro:`!PyUnstable_EXECUTABLE_KIND_PY_FUNCTION`, + :c:macro:`!PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION`, + :c:macro:`!PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR` and + :c:macro:`!PyUnstable_EXECUTABLE_KINDS`, were removed. + + (Contributed by Peters Bierma and Viktorin in :gh:`152105`.) diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 48c461d891e8614..c61eb659ccc0f62 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -756,7 +756,7 @@ functools --------- * The :mod:`functools` module includes a new decorator for caching function - calls. :func:`functools.lru_cache` can save repeated queries to an external + calls. :deco:`functools.lru_cache` can save repeated queries to an external resource whenever the results are expected to be the same. For example, adding a caching decorator to a database query function can save @@ -789,7 +789,7 @@ functools `_\, :issue:`10586`, and :issue:`10593`.) -* The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute +* The :deco:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute pointing to the original callable function. This allows wrapped functions to be introspected. It also copies :attr:`~function.__annotations__` if defined. And now it also gracefully skips over missing attributes such as @@ -805,11 +805,11 @@ functools :issue:`8814`.) * To help write classes with rich comparison methods, a new decorator - :func:`functools.total_ordering` will use existing equality and inequality + :deco:`functools.total_ordering` will use existing equality and inequality methods to fill in the remaining methods. For example, supplying *__eq__* and *__lt__* will enable - :func:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: + :deco:`~functools.total_ordering` to fill-in *__le__*, *__gt__* and *__ge__*:: @total_ordering class Student: @@ -1097,11 +1097,11 @@ logarithm of the gamma function: abc --- -The :mod:`abc` module now supports :func:`~abc.abstractclassmethod` and -:func:`~abc.abstractstaticmethod`. +The :mod:`abc` module now supports :deco:`~abc.abstractclassmethod` and +:deco:`~abc.abstractstaticmethod`. These tools make it possible to define an :term:`abstract base class` that -requires a particular :func:`classmethod` or :func:`staticmethod` to be +requires a particular :deco:`classmethod` or :deco:`staticmethod` to be implemented:: class Temperature(metaclass=abc.ABCMeta): @@ -1155,7 +1155,7 @@ self-reference by displaying "..." in the recursive part of the representation string. To help write such :meth:`~object.__repr__` methods, the :mod:`reprlib` module has a new -decorator, :func:`~reprlib.recursive_repr`, for detecting recursive calls to +decorator, :deco:`~reprlib.recursive_repr`, for detecting recursive calls to :meth:`!__repr__` and substituting a placeholder string instead:: >>> class MyList(list): @@ -1241,7 +1241,7 @@ There is a new and slightly mind-blowing tool :term:`context manager` that does double duty as a function decorator. As a convenience, this new functionality is used by -:func:`~contextlib.contextmanager` so that no extra effort is needed to support +:deco:`~contextlib.contextmanager` so that no extra effort is needed to support both roles. The basic idea is that both context managers and function decorators can be used @@ -1253,7 +1253,7 @@ write a pre-action or post-action wrapper that can be used in either role. For example, it is sometimes useful to wrap functions or groups of statements with a logger that can track the time of entry and time of exit. Rather than writing both a function decorator and a context manager for the task, the -:func:`~contextlib.contextmanager` provides both capabilities in a single +:deco:`~contextlib.contextmanager` provides both capabilities in a single definition:: from contextlib import contextmanager diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 1bb79bce2c3e972..02fd264e53e1b06 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -919,12 +919,12 @@ abstract methods. The recommended approach to declaring abstract descriptors is now to provide :attr:`!__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. -* :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. -* :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. +* :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. +* :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. (Contributed by Darren Dale in :issue:`11610`.) @@ -1390,7 +1390,7 @@ ftplib functools --------- -The :func:`functools.lru_cache` decorator now accepts a ``typed`` keyword +The :deco:`functools.lru_cache` decorator now accepts a ``typed`` keyword argument (that defaults to ``False`` to ensure that it caches values of different types that compare equal in separate cache slots. (Contributed by Raymond Hettinger in :issue:`13227`.) @@ -2244,12 +2244,12 @@ Deprecated Python modules, functions and methods * The :func:`!os.stat_float_times` function is deprecated. * :mod:`abc` module: - * :class:`abc.abstractproperty` has been deprecated, use :class:`property` - with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractclassmethod` has been deprecated, use - :class:`classmethod` with :func:`abc.abstractmethod` instead. - * :class:`abc.abstractstaticmethod` has been deprecated, use - :class:`staticmethod` with :func:`abc.abstractmethod` instead. + * :deco:`abc.abstractproperty` has been deprecated, use :deco:`property` + with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractclassmethod` has been deprecated, use + :deco:`classmethod` with :deco:`abc.abstractmethod` instead. + * :deco:`abc.abstractstaticmethod` has been deprecated, use + :deco:`staticmethod` with :deco:`abc.abstractmethod` instead. * :mod:`importlib` package: diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index a390211ddb50215..0067491c569cc05 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -858,7 +858,7 @@ to behave like normal instance methods when included in a class definition. .. _whatsnew-singledispatch: -The new :func:`~functools.singledispatch` decorator brings support for +The new :deco:`~functools.singledispatch` decorator brings support for single-dispatch generic functions to the Python standard library. Where object oriented programming focuses on grouping multiple operations on a common set of data into a class, a generic function focuses on grouping @@ -870,7 +870,7 @@ multiple implementations of an operation that allows it to work with :pep:`443` -- Single-dispatch generic functions PEP written and implemented by Łukasz Langa. -:func:`~functools.total_ordering` now supports a return value of +:deco:`~functools.total_ordering` now supports a return value of :data:`NotImplemented` from the underlying comparison function. (Contributed by Katie Miller in :issue:`10042`.) @@ -1029,7 +1029,7 @@ information for modules, classes and functions. (Contributed by Claudiu Popa and Nick Coghlan in :issue:`18626`.) :func:`~inspect.unwrap` makes it easy to unravel wrapper function chains -created by :func:`functools.wraps` (and any other API that sets the +created by :deco:`functools.wraps` (and any other API that sets the ``__wrapped__`` attribute on a wrapper function). (Contributed by Daniel Urban, Aaron Iles and Nick Coghlan in :issue:`13266`.) @@ -2317,7 +2317,7 @@ Changes in the Python API wish to continue to ignore syntax or decoding issues, catch all three exceptions now. -* :func:`functools.update_wrapper` and :func:`functools.wraps` now correctly +* :func:`functools.update_wrapper` and :deco:`functools.wraps` now correctly set the ``__wrapped__`` attribute to the function being wrapped, even if that function also had its ``__wrapped__`` attribute set. This means ``__wrapped__`` attributes now correctly link a stack of decorated diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index 6009dd8a71eea52..24a2ffbefd0cfba 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -121,7 +121,7 @@ Significant improvements in the standard library: :ref:`better and significantly faster way ` of directory traversal. -* :func:`functools.lru_cache` has been mostly +* :deco:`functools.lru_cache` has been mostly :ref:`reimplemented in C `, yielding much better performance. @@ -1148,7 +1148,7 @@ functools .. _whatsnew-lrucache: -Most of the :func:`~functools.lru_cache` machinery is now implemented in C, making +Most of the :deco:`~functools.lru_cache` machinery is now implemented in C, making it significantly faster. (Contributed by Matt Joiner, Alexey Kachayev, and Serhiy Storchaka in :issue:`14373`.) @@ -2158,7 +2158,7 @@ improvement in some benchmarks. Objects from the :mod:`random` module now use 50% less memory on 64-bit builds. (Contributed by Serhiy Storchaka in :issue:`23488`.) -The :func:`property` getter calls are up to 25% faster. +The :deco:`property` getter calls are up to 25% faster. (Contributed by Joe Jevnik in :issue:`23910`.) Instantiation of :class:`fractions.Fraction` is now up to 30% faster. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 9eafc09dbee5f43..b0382dd8215bc00 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -517,7 +517,7 @@ Applications that do not use str to represent paths should use :func:`os.fsencode` and :func:`os.fsdecode` to ensure their bytes are correctly encoded. To revert to the previous behaviour, set :envvar:`PYTHONLEGACYWINDOWSFSENCODING` or call -:func:`sys._enablelegacywindowsfsencoding`. +:func:`!sys._enablelegacywindowsfsencoding`. See :pep:`529` for more information and discussion of code modifications that may be required. @@ -2173,7 +2173,7 @@ Changes in the Python API * :c:func:`PyErr_SetImportError` now sets :exc:`TypeError` when its **msg** argument is not set. Previously only ``NULL`` was returned. -* The format of the :attr:`~codeobject.co_lnotab` attribute of code objects +* The format of the :attr:`!codeobject.co_lnotab` attribute of code objects changed to support a negative line number delta. By default, Python does not emit bytecode with a negative line number delta. Functions using :attr:`frame.f_lineno`, diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 5dd47cdac96a5c0..3af3e6ec9cac08f 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -571,7 +571,7 @@ decimal operations to work with the correct context in asynchronous code. dataclasses ----------- -The new :func:`~dataclasses.dataclass` decorator provides a way to declare +The new :deco:`~dataclasses.dataclass` decorator provides a way to declare *data classes*. A data class describes its attributes using class variable annotations. Its constructor and other magic methods, such as :meth:`~object.__repr__`, :meth:`~object.__eq__`, and @@ -830,7 +830,7 @@ The new :func:`~contextlib.nullcontext` is a simpler and faster no-op context manager than :class:`~contextlib.ExitStack`. (Contributed by Jesse-Bakker in :issue:`10049`.) -The new :func:`~contextlib.asynccontextmanager`, +The new :deco:`~contextlib.asynccontextmanager`, :class:`~contextlib.AbstractAsyncContextManager`, and :class:`~contextlib.AsyncExitStack` have been added to complement their synchronous counterparts. (Contributed @@ -921,7 +921,7 @@ return :const:`False` instead and are deprecated. functools --------- -:func:`functools.singledispatch` now supports registering implementations +:deco:`functools.singledispatch` now supports registering implementations using type annotations. (Contributed by Łukasz Langa in :issue:`32227`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 5078fc30ac111e4..42d98ff94ec0d1b 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -764,7 +764,7 @@ these are the inverse of each class's ``isocalendar`` method. functools --------- -:func:`functools.lru_cache` can now be used as a straight decorator rather +:deco:`functools.lru_cache` can now be used as a straight decorator rather than as a function returning a decorator. So both of these are now supported:: @lru_cache @@ -777,7 +777,7 @@ than as a function returning a decorator. So both of these are now supported:: (Contributed by Raymond Hettinger in :issue:`36772`.) -Added a new :func:`functools.cached_property` decorator, for computed properties +Added a new :deco:`functools.cached_property` decorator, for computed properties cached for the life of the instance. :: import functools @@ -893,7 +893,7 @@ inspect The :func:`inspect.getdoc` function can now find docstrings for ``__slots__`` if that attribute is a :class:`dict` where the values are docstrings. This provides documentation options similar to what we already have -for :func:`property`, :func:`classmethod`, and :func:`staticmethod`:: +for :deco:`property`, :deco:`classmethod`, and :deco:`staticmethod`:: class AudioClip: __slots__ = {'bit_rate': 'expressed in kilohertz to one decimal place', @@ -1333,14 +1333,14 @@ The :mod:`typing` module incorporates several new features: ... * "Final" variables, functions, methods and classes. See :pep:`591`, - :class:`typing.Final` and :func:`typing.final`. + :class:`typing.Final` and :deco:`typing.final`. The final qualifier instructs a static type checker to restrict subclassing, overriding, or reassignment:: pi: Final[float] = 3.1415926536 * Protocol definitions. See :pep:`544`, :class:`typing.Protocol` and - :func:`typing.runtime_checkable`. Simple ABCs like + :deco:`typing.runtime_checkable`. Simple ABCs like :class:`typing.SupportsInt` are now ``Protocol`` subclasses. * New protocol class :class:`typing.SupportsIndex`. diff --git a/Doc/whatsnew/index.rst b/Doc/whatsnew/index.rst index 38194db670b8390..420876a9b3ecd00 100644 --- a/Doc/whatsnew/index.rst +++ b/Doc/whatsnew/index.rst @@ -11,6 +11,7 @@ anyone wishing to stay up-to-date after a new release. .. toctree:: :maxdepth: 2 + 3.16.rst 3.15.rst 3.14.rst 3.13.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 3a91d426c365019..2713ba9466a0b1d 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -229,8 +229,9 @@ import_name[stmt_ty]: # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS import_from[stmt_ty]: + | invalid_import_from | lazy="lazy"? 'from' a=('.' | '...')* b=dotted_name 'import' c=import_from_targets { - _PyPegen_checked_future_import(p, b->v.Name.id, c, _PyPegen_seq_count_dots(a), lazy, EXTRA) } + _PyPegen_checked_from_import(p, a, b, c, lazy, EXTRA) } | lazy="lazy"? 'from' a=('.' | '...')+ 'import' b=import_from_targets { _PyAST_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), lazy ? 1 : 0, EXTRA) } import_from_targets[asdl_alias_seq*]: @@ -554,10 +555,12 @@ complex_number[expr_ty]: signed_number[expr_ty]: | NUMBER + | '+' number=NUMBER { number } | '-' number=NUMBER { _PyAST_UnaryOp(USub, number, EXTRA) } signed_real_number[expr_ty]: | real_number + | '+' real=real_number { real } | '-' real=real_number { _PyAST_UnaryOp(USub, real, EXTRA) } real_number[expr_ty]: @@ -816,6 +819,7 @@ is_bitwise_or[CmpopExprPair*]: 'is' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, I bitwise_or[expr_ty]: | a=bitwise_or '|' b=bitwise_xor { _PyAST_BinOp(a, BitOr, b, EXTRA) } + | invalid_bitwise_or | bitwise_xor bitwise_xor[expr_ty]: @@ -824,12 +828,12 @@ bitwise_xor[expr_ty]: bitwise_and[expr_ty]: | a=bitwise_and '&' b=shift_expr { _PyAST_BinOp(a, BitAnd, b, EXTRA) } + | invalid_bitwise_and | shift_expr shift_expr[expr_ty]: | a=shift_expr '<<' b=sum { _PyAST_BinOp(a, LShift, b, EXTRA) } | a=shift_expr '>>' b=sum { _PyAST_BinOp(a, RShift, b, EXTRA) } - | invalid_arithmetic | sum # Arithmetic operators @@ -838,6 +842,7 @@ shift_expr[expr_ty]: sum[expr_ty]: | a=sum '+' b=term { _PyAST_BinOp(a, Add, b, EXTRA) } | a=sum '-' b=term { _PyAST_BinOp(a, Sub, b, EXTRA) } + | invalid_arithmetic | term term[expr_ty]: @@ -846,7 +851,6 @@ term[expr_ty]: | a=term '//' b=factor { _PyAST_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _PyAST_BinOp(a, Mod, b, EXTRA) } | a=term '@' b=factor { CHECK_VERSION(expr_ty, 5, "The '@' operator is", _PyAST_BinOp(a, MatMult, b, EXTRA)) } - | invalid_factor | factor factor[expr_ty] (memo): @@ -854,6 +858,7 @@ factor[expr_ty] (memo): | '-' a=factor { _PyAST_UnaryOp(USub, a, EXTRA) } | '~' a=factor { _PyAST_UnaryOp(Invert, a, EXTRA) } | power + | invalid_factor power[expr_ty]: | a=await_primary '**' b=factor { _PyAST_BinOp(a, Pow, b, EXTRA) } @@ -1440,6 +1445,11 @@ invalid_import_from_as_name: RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } +invalid_import_from: + | 'from' ('.' | '...')* dotted_name a="lazy" 'import' import_from_targets { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, + "use 'lazy from ... ' instead of 'from ... lazy import'") } + invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } @@ -1635,3 +1645,17 @@ invalid_type_params: RAISE_SYNTAX_ERROR_STARTING_FROM( token, "Type parameter list cannot be empty")} + +invalid_bitwise_and: + | a=bitwise_and b='&' c='&' { + _PyPegen_tokens_are_adjacent(b, c) + ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'and' or '&' instead of '&&'?") + : NULL + } + +invalid_bitwise_or: + | a=bitwise_or b='|' c='|' { + _PyPegen_tokens_are_adjacent(b, c) + ? RAISE_SYNTAX_ERROR_KNOWN_RANGE(b, c, "invalid syntax. Maybe you meant 'or' or '|' instead of '||'?") + : NULL + } diff --git a/Include/Python.h b/Include/Python.h index e6e5cab67e2045b..337119c15fe8b6f 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -9,10 +9,11 @@ // is not needed. -// Include Python header files -#include "patchlevel.h" -#include "pyconfig.h" -#include "pymacconfig.h" +// Include Python configuration headers +#include "patchlevel.h" // the Python version +#include "pyconfig.h" // information from configure +#include "pymacconfig.h" // overrides for pyconfig +#include "pyabi.h" // feature/ABI selection // Include standard header files @@ -46,13 +47,11 @@ # endif #endif -#if defined(Py_GIL_DISABLED) -# if defined(_MSC_VER) -# include // __readgsqword() -# endif - -# if defined(__MINGW32__) -# include // __readgsqword() +#if !defined(Py_LIMITED_API) +# if defined(Py_GIL_DISABLED) +# if defined(_MSC_VER) || defined(__MINGW32__) +# include // __readgsqword() +# endif # endif #endif // Py_GIL_DISABLED @@ -67,6 +66,7 @@ __pragma(warning(disable: 4201)) // Include Python header files #include "pyport.h" +#include "exports.h" #include "pymacro.h" #include "pymath.h" #include "pymem.h" @@ -79,7 +79,8 @@ __pragma(warning(disable: 4201)) #include "object.h" #include "refcount.h" #include "objimpl.h" -#include "typeslots.h" +#include "slots.h" +#include "slots_generated.h" #include "pyhash.h" #include "cpython/pydebug.h" #include "bytearrayobject.h" @@ -117,6 +118,7 @@ __pragma(warning(disable: 4201)) #include "cpython/genobject.h" #include "descrobject.h" #include "genericaliasobject.h" +#include "cpython/sentinelobject.h" #include "warnings.h" #include "weakrefobject.h" #include "structseq.h" diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index bbab8d35b75cb2b..5b66fa1040d738f 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -38,7 +38,7 @@ typedef struct { PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void); PyAPI_FUNC(int) PyUnstable_WritePerfMapEntry( const void *code_addr, - unsigned int code_size, + size_t code_size, const char *entry_name); PyAPI_FUNC(void) PyUnstable_PerfMapState_Fini(void); PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index 84456a709a6abe7..1be47e42ed62bfb 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -34,9 +34,13 @@ typedef struct { char *entries[1]; } _PyCodeArray; +#define _PyCode_DEF_UNIQUE_ID() \ + Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ + #define _PyCode_DEF_THREAD_LOCAL_BYTECODE() \ _PyCodeArray *co_tlbc; #else +#define _PyCode_DEF_UNIQUE_ID() #define _PyCode_DEF_THREAD_LOCAL_BYTECODE() #endif @@ -101,7 +105,7 @@ typedef struct { _PyCoCached *_co_cached; /* cached co_* attributes */ \ uintptr_t _co_instrumentation_version; /* current instrumentation version */ \ struct _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ - Py_ssize_t _co_unique_id; /* ID used for per-thread refcounting */ \ + _PyCode_DEF_UNIQUE_ID() \ int _co_firsttraceable; /* index of first traceable instruction */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 4fc46fefb93a24a..bcba32da412f326 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -2,15 +2,6 @@ # error "this header file must not be included directly" #endif -// Python critical sections -// -// Conceptually, critical sections are a deadlock avoidance layer on top of -// per-object locks. These helpers, in combination with those locks, replace -// our usage of the global interpreter lock to provide thread-safety for -// otherwise thread-unsafe objects, such as dict. -// -// NOTE: These APIs are no-ops in non-free-threaded builds. -// // Straightforward per-object locking could introduce deadlocks that were not // present when running with the GIL. Threads may hold locks for multiple // objects simultaneously because Python operations can nest. If threads were @@ -43,52 +34,19 @@ // `_PyThreadState_Attach()`, it resumes the top-most (i.e., most recent) // critical section by reacquiring the associated lock or locks. See // `_PyCriticalSection_Resume()`. -// -// NOTE: Only the top-most critical section is guaranteed to be active. -// Operations that need to lock two objects at once must use -// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections -// to lock more than one object at once, because the inner critical section -// may suspend the outer critical sections. This API does not provide a way -// to lock more than two objects at once (though it could be added later -// if actually needed). -// -// NOTE: Critical sections implicitly behave like reentrant locks because -// attempting to acquire the same lock will suspend any outer (earlier) -// critical sections. However, they are less efficient for this use case than -// purposefully designed reentrant locks. -// -// Example usage: -// Py_BEGIN_CRITICAL_SECTION(op); -// ... -// Py_END_CRITICAL_SECTION(); -// -// To lock two objects at once: -// Py_BEGIN_CRITICAL_SECTION2(op1, op2); -// ... -// Py_END_CRITICAL_SECTION2(); - -typedef struct PyCriticalSection PyCriticalSection; -typedef struct PyCriticalSection2 PyCriticalSection2; - -PyAPI_FUNC(void) -PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); PyAPI_FUNC(void) PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); -PyAPI_FUNC(void) -PyCriticalSection_End(PyCriticalSection *c); - -PyAPI_FUNC(void) -PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); - PyAPI_FUNC(void) PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); -PyAPI_FUNC(void) -PyCriticalSection2_End(PyCriticalSection2 *c); - #ifndef Py_GIL_DISABLED +#undef Py_BEGIN_CRITICAL_SECTION +#undef Py_END_CRITICAL_SECTION +#undef Py_BEGIN_CRITICAL_SECTION2 +#undef Py_END_CRITICAL_SECTION2 + # define Py_BEGIN_CRITICAL_SECTION(op) \ { # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ @@ -101,54 +59,17 @@ PyCriticalSection2_End(PyCriticalSection2 *c); { # define Py_END_CRITICAL_SECTION2() \ } -#else /* !Py_GIL_DISABLED */ - -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection { - // Tagged pointer to an outer active critical section (or 0). - uintptr_t _cs_prev; - - // Mutex used to protect critical section - PyMutex *_cs_mutex; -}; - -// A critical section protected by two mutexes. Use -// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. -// NOTE: the contents of this struct are private and may change betweeen -// Python releases without a deprecation period. -struct PyCriticalSection2 { - PyCriticalSection _cs_base; - PyMutex *_cs_mutex2; -}; - -# define Py_BEGIN_CRITICAL_SECTION(op) \ - { \ - PyCriticalSection _py_cs; \ - PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +#else /* !Py_GIL_DISABLED */ # define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ { \ PyCriticalSection _py_cs; \ PyCriticalSection_BeginMutex(&_py_cs, mutex) -# define Py_END_CRITICAL_SECTION() \ - PyCriticalSection_End(&_py_cs); \ - } - -# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ - { \ - PyCriticalSection2 _py_cs2; \ - PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) - # define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ { \ PyCriticalSection2 _py_cs2; \ PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) -# define Py_END_CRITICAL_SECTION2() \ - PyCriticalSection2_End(&_py_cs2); \ - } - -#endif +#endif /* !Py_GIL_DISABLED */ diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index f75884e597e2c24..e14facd77edec53 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -16,8 +16,8 @@ PyAPI_DATA(PyTypeObject) PyGen_Type; #define PyGen_Check(op) PyObject_TypeCheck((op), &PyGen_Type) #define PyGen_CheckExact(op) Py_IS_TYPE((op), &PyGen_Type) -PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); -PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_New(PyFrameObject *); +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyGen_NewWithQualName(PyFrameObject *, PyObject *name, PyObject *qualname); PyAPI_FUNC(PyCodeObject *) PyGen_GetCode(PyGenObject *gen); @@ -29,7 +29,7 @@ typedef struct _PyCoroObject PyCoroObject; PyAPI_DATA(PyTypeObject) PyCoro_Type; #define PyCoro_CheckExact(op) Py_IS_TYPE((op), &PyCoro_Type) -PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyCoro_New(PyFrameObject *, PyObject *name, PyObject *qualname); @@ -40,7 +40,7 @@ typedef struct _PyAsyncGenObject PyAsyncGenObject; PyAPI_DATA(PyTypeObject) PyAsyncGen_Type; PyAPI_DATA(PyTypeObject) _PyAsyncGenASend_Type; -PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, +Py_DEPRECATED(3.16) PyAPI_FUNC(PyObject *) PyAsyncGen_New(PyFrameObject *, PyObject *name, PyObject *qualname); #define PyAsyncGen_CheckExact(op) Py_IS_TYPE((op), &PyAsyncGen_Type) diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h index 5094c8c23ae32bb..c93271f6ca95f5b 100644 --- a/Include/cpython/monitoring.h +++ b/Include/cpython/monitoring.h @@ -24,9 +24,10 @@ extern "C" { #define PY_MONITORING_EVENT_STOP_ITERATION 10 #define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) +((ev) <= PY_MONITORING_EVENT_STOP_ITERATION) -/* Other events, mainly exceptions */ +/* Other events, mainly exceptions. + * These can now be turned on and disabled on a per code object basis. */ #define PY_MONITORING_EVENT_RAISE 11 #define PY_MONITORING_EVENT_EXCEPTION_HANDLED 12 @@ -34,6 +35,9 @@ extern "C" { #define PY_MONITORING_EVENT_PY_THROW 14 #define PY_MONITORING_EVENT_RERAISE 15 +#define _PY_MONITORING_IS_UNGROUPED_EVENT(ev) \ +((ev) < _PY_MONITORING_UNGROUPED_EVENTS) + /* Ancillary events */ diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 80d30c7765fdb00..4c5a677e5543ece 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -230,6 +230,8 @@ struct _typeobject { destructor tp_finalize; vectorcallfunc tp_vectorcall; + /* Below here all fields are internal to the VM */ + /* bitset of which type-watchers care about this type */ unsigned char tp_watched; @@ -239,6 +241,13 @@ struct _typeobject { * Otherwise, limited to MAX_VERSIONS_PER_CLASS (defined elsewhere). */ uint16_t tp_versions_used; + + /* Virtual iterator next function. + * This function must escape to any code that can result in + * the GC being run, such as Py_DECREF. */ + _Py_iteritemfunc _tp_iteritem; + + void *_tp_cache; }; #define _Py_ATTR_CACHE_UNUSED (30000) // (see tp_versions_used) @@ -304,7 +313,6 @@ Py_DEPRECATED(3.15) PyAPI_FUNC(PyObject*) _PyObject_GetAttrId(PyObject *, _Py_Id PyAPI_FUNC(PyObject **) _PyObject_GetDictPtr(PyObject *); PyAPI_FUNC(void) PyObject_CallFinalizer(PyObject *); -PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); PyAPI_FUNC(void) PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *); diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index 51529763923ec3a..24a947de1ede1e1 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -35,11 +35,3 @@ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame /* Returns the currently executing line number, or -1 if there is no line number. * Does not raise an exception. */ PyAPI_FUNC(int) PyUnstable_InterpreterFrame_GetLine(struct _PyInterpreterFrame *frame); - -#define PyUnstable_EXECUTABLE_KIND_SKIP 0 -#define PyUnstable_EXECUTABLE_KIND_PY_FUNCTION 1 -#define PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION 3 -#define PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR 4 -#define PyUnstable_EXECUTABLE_KINDS 5 - -PyAPI_DATA(const PyTypeObject *) const PyUnstable_ExecutableKinds[PyUnstable_EXECUTABLE_KINDS+1]; diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 1c56ad5af8072f3..f367146e262bfe8 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -105,7 +105,7 @@ struct _ts { # define _PyThreadState_WHENCE_INIT 1 # define _PyThreadState_WHENCE_FINI 2 # define _PyThreadState_WHENCE_THREADING 3 -# define _PyThreadState_WHENCE_GILSTATE 4 +# define _PyThreadState_WHENCE_C_API 4 # define _PyThreadState_WHENCE_EXEC 5 # define _PyThreadState_WHENCE_THREADING_DAEMON 6 #endif @@ -143,6 +143,7 @@ struct _ts { struct _PyInterpreterFrame *base_frame; struct _PyInterpreterFrame *last_profiled_frame; + uintptr_t last_profiled_frame_seq; Py_tracefunc c_profilefunc; Py_tracefunc c_tracefunc; @@ -239,6 +240,20 @@ struct _ts { // structure and all share the same per-interpreter structure). PyStats *pystats; #endif + + struct { + /* Number of nested PyThreadState_Ensure() calls on this thread state */ + Py_ssize_t counter; + + /* Should this thread state be deleted upon calling + PyThreadState_Release() (with the counter at 1)? + + This is only true for thread states created by PyThreadState_Ensure() */ + int delete_on_release; + + /* The interpreter guard owned by PyThreadState_EnsureFromView(), if any. */ + PyInterpreterGuard *owned_guard; + } ensure; }; /* other API */ @@ -319,3 +334,8 @@ PyAPI_FUNC(_PyFrameEvalFunction) _PyInterpreterState_GetEvalFrameFunc( PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameAllowSpecialization( + PyInterpreterState *interp, + int allow_specialization); +PyAPI_FUNC(int) _PyInterpreterState_IsSpecializationEnabled( + PyInterpreterState *interp); diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index e473110eca74156..738e342d1407686 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -96,7 +96,9 @@ typedef struct _object_stats { uint64_t type_cache_misses; uint64_t type_cache_dunder_hits; uint64_t type_cache_dunder_misses; - uint64_t type_cache_collisions; + uint64_t type_cache_too_big; + uint64_t type_cache_invalidations; + uint64_t type_cache_resizes; /* Temporary value used during GC */ uint64_t object_visits; } ObjectStats; @@ -144,6 +146,7 @@ typedef struct _optimization_stats { uint64_t unknown_callee; uint64_t trace_immediately_deopts; uint64_t executors_invalidated; + uint64_t fitness_terminated_traces; UOpStats opcode[PYSTATS_MAX_UOP_ID + 1]; uint64_t unsupported_opcode[256]; uint64_t trace_length_hist[_Py_UOP_HIST_SIZE]; @@ -162,6 +165,7 @@ typedef struct _optimization_stats { uint64_t jit_code_size; uint64_t jit_trampoline_size; uint64_t jit_data_size; + uint64_t jit_got_size; uint64_t jit_padding_size; uint64_t jit_freed_memory_size; uint64_t trace_total_memory_hist[_Py_UOP_HIST_SIZE]; diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index edc40952254029f..b0bf527f4f0ec19 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -40,6 +40,11 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags( PyCompilerFlags *flags); +PyAPI_FUNC(PyObject *) Py_CompileStringFlags( + const char *str, + const char *filename, + int start, + PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( const char *str, const char *filename, /* decoded from the filesystem encoding */ diff --git a/Include/cpython/sentinelobject.h b/Include/cpython/sentinelobject.h new file mode 100644 index 000000000000000..e621d6abbfed8aa --- /dev/null +++ b/Include/cpython/sentinelobject.h @@ -0,0 +1,26 @@ +/* Sentinel object interface */ + +#ifndef Py_LIMITED_API +#ifndef _Py_SENTINELOBJECT_H +#define _Py_SENTINELOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(PyTypeObject) PySentinel_Type; + +#define PySentinel_CheckExact(op) Py_IS_TYPE((op), &PySentinel_Type) + +/* Alias as long as subclasses are not allowed. */ +#define PySentinel_Check(op) PySentinel_CheckExact(op) + +PyAPI_FUNC(PyObject *) PySentinel_New( + const char *name, + const char *module_name, + const char *repr); + +#ifdef __cplusplus +} +#endif +#endif /* !_Py_SENTINELOBJECT_H */ +#endif /* !Py_LIMITED_API */ diff --git a/Include/cpython/sliceobject.h b/Include/cpython/sliceobject.h index 4c3ea1facebc4ed..137206eff15b33f 100644 --- a/Include/cpython/sliceobject.h +++ b/Include/cpython/sliceobject.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_SLICEOBJECT_H +#ifndef _Py_CPYTHON_SLICEOBJECT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/structseq.h b/Include/cpython/structseq.h index 328fbe86143b024..83a1abcd6f3b347 100644 --- a/Include/cpython/structseq.h +++ b/Include/cpython/structseq.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_STRUCTSEQ_H +#ifndef _Py_CPYTHON_STRUCTSEQ_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/traceback.h b/Include/cpython/traceback.h index 81c51944f136f29..7f42730f1b09194 100644 --- a/Include/cpython/traceback.h +++ b/Include/cpython/traceback.h @@ -11,3 +11,11 @@ struct _traceback { int tb_lasti; int tb_lineno; }; + +PyAPI_FUNC(const char*) PyUnstable_DumpTraceback(int fd, PyThreadState *tstate); + +PyAPI_FUNC(const char*) PyUnstable_DumpTracebackThreads( + int fd, + PyInterpreterState *interp, + PyThreadState *current_tstate, + Py_ssize_t max_threads); diff --git a/Include/critical_section.h b/Include/critical_section.h index 3b37615a8b17e2a..732bfab7ecf2349 100644 --- a/Include/critical_section.h +++ b/Include/critical_section.h @@ -4,6 +4,91 @@ extern "C" { #endif +// Python critical sections +// +// Conceptually, critical sections are a deadlock avoidance layer on top of +// per-object locks. These helpers, in combination with those locks, replace +// our usage of the global interpreter lock to provide thread-safety for +// otherwise thread-unsafe objects, such as dict. +// +// NOTE: These APIs are no-ops in non-free-threaded builds. +// +// NOTE: Only the top-most critical section is guaranteed to be active. +// Operations that need to lock two objects at once must use +// `Py_BEGIN_CRITICAL_SECTION2()`. You *CANNOT* use nested critical sections +// to lock more than one object at once, because the inner critical section +// may suspend the outer critical sections. This API does not provide a way +// to lock more than two objects at once (though it could be added later +// if actually needed). +// +// NOTE: Critical sections implicitly behave like reentrant locks because +// attempting to acquire the same lock will suspend any outer (earlier) +// critical sections. However, they are less efficient for this use case than +// purposefully designed reentrant locks. +// +// Example usage: +// Py_BEGIN_CRITICAL_SECTION(op); +// ... +// Py_END_CRITICAL_SECTION(); +// +// To lock two objects at once: +// Py_BEGIN_CRITICAL_SECTION2(op1, op2); +// ... +// Py_END_CRITICAL_SECTION2(); + +// NOTE: the contents of this struct are private and their meaning may +// change betweeen Python releases without a deprecation period. +typedef struct PyCriticalSection { + // Tagged pointer to an outer active critical section (or 0). + uintptr_t _cs_prev; + + // Mutex used to protect critical section + struct PyMutex *_cs_mutex; +} PyCriticalSection; + +// A critical section protected by two mutexes. Use +// Py_BEGIN_CRITICAL_SECTION2 and Py_END_CRITICAL_SECTION2. +// NOTE: the contents of this struct are private and may change betweeen +// Python releases without a deprecation period. +typedef struct PyCriticalSection2 { + PyCriticalSection _cs_base; + + struct PyMutex *_cs_mutex2; +} PyCriticalSection2; + +PyAPI_FUNC(void) +PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); + +PyAPI_FUNC(void) +PyCriticalSection_End(PyCriticalSection *c); + +PyAPI_FUNC(void) +PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); + +PyAPI_FUNC(void) +PyCriticalSection2_End(PyCriticalSection2 *c); + +// These are definitions for the stable ABI. For GIL-ful builds they're +// conditionally redefined as no-ops in cpython/critical_section.h. + +# define Py_BEGIN_CRITICAL_SECTION(op) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) + +# define Py_END_CRITICAL_SECTION() \ + PyCriticalSection_End(&_py_cs); \ + } + +# define Py_BEGIN_CRITICAL_SECTION2(a, b) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) + +# define Py_END_CRITICAL_SECTION2() \ + PyCriticalSection2_End(&_py_cs2); \ + } + #ifndef Py_LIMITED_API # define Py_CPYTHON_CRITICAL_SECTION_H # include "cpython/critical_section.h" diff --git a/Include/dynamic_annotations.h b/Include/dynamic_annotations.h index 4d4def9bf8983e2..5290319dd762c55 100644 --- a/Include/dynamic_annotations.h +++ b/Include/dynamic_annotations.h @@ -461,6 +461,7 @@ int RunningOnValgrind(void); #if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) +extern "C++" { /* _Py_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. Instead of doing @@ -476,6 +477,8 @@ int RunningOnValgrind(void); _Py_ANNOTATE_IGNORE_READS_END(); return res; } +} + /* Apply _Py_ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ #define _Py_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ namespace { \ diff --git a/Include/exports.h b/Include/exports.h index 97a674ec2403a4e..18692283005e596 100644 --- a/Include/exports.h +++ b/Include/exports.h @@ -36,7 +36,7 @@ #define Py_LOCAL_SYMBOL #endif /* module init functions outside the core must be exported */ - #if defined(Py_BUILD_CORE) + #if defined(_PyEXPORTS_CORE) #define _PyINIT_EXPORTED_SYMBOL Py_EXPORTED_SYMBOL #else #define _PyINIT_EXPORTED_SYMBOL __declspec(dllexport) @@ -64,13 +64,13 @@ /* only get special linkage if built as shared or platform is Cygwin */ #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__) # if defined(HAVE_DECLSPEC_DLL) -# if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# if defined(_PyEXPORTS_CORE) && !defined(_PyEXPORTS_CORE_MODULE) /* module init functions inside the core need no external linkage */ /* except for Cygwin to handle embedding */ # if !defined(__CYGWIN__) # define _PyINIT_FUNC_DECLSPEC # endif /* __CYGWIN__ */ -# else /* Py_BUILD_CORE */ +# else /* _PyEXPORTS_CORE */ /* Building an extension module, or an embedded situation */ /* public Python functions and data are imported */ /* Under Cygwin, auto-import functions to prevent compilation */ @@ -80,7 +80,7 @@ # define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE # endif /* !__CYGWIN__ */ # define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE -# endif /* Py_BUILD_CORE */ +# endif /* _PyEXPORTS_CORE */ # endif /* HAVE_DECLSPEC_DLL */ #endif /* Py_ENABLE_SHARED */ @@ -103,7 +103,7 @@ #define PyMODINIT_FUNC _PyINIT_FUNC_DECLSPEC PyObject* #endif #ifndef PyMODEXPORT_FUNC - #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PyModuleDef_Slot* + #define PyMODEXPORT_FUNC _PyINIT_FUNC_DECLSPEC PySlot* #endif #endif /* Py_EXPORTS_H */ diff --git a/Include/import.h b/Include/import.h index 6f1c13787b8569a..c062e46bff46bf7 100644 --- a/Include/import.h +++ b/Include/import.h @@ -90,8 +90,7 @@ PyAPI_FUNC(int) PyImport_AppendInittab( typedef enum { PyImport_LAZY_NORMAL, - PyImport_LAZY_ALL, - PyImport_LAZY_NONE + PyImport_LAZY_ALL } PyImport_LazyImportsMode; #ifndef Py_LIMITED_API diff --git a/Include/internal/pycore_abstract.h b/Include/internal/pycore_abstract.h index 30809e097002ddf..67c6fa7c0c4ed5b 100644 --- a/Include/internal/pycore_abstract.h +++ b/Include/internal/pycore_abstract.h @@ -16,10 +16,11 @@ _PyIndex_Check(PyObject *obj) return (tp_as_number != NULL && tp_as_number->nb_index != NULL); } -PyObject *_PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); -PyObject *_PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyNumber_PowerNoMod(PyObject *lhs, PyObject *rhs); +PyAPI_FUNC(PyObject *) _PyNumber_InPlacePowerNoMod(PyObject *lhs, PyObject *rhs); -extern int _PyObject_HasLen(PyObject *o); +PyAPI_FUNC(int) _PyObject_HasLen(PyObject *o); /* === Sequence protocol ================================================ */ @@ -59,6 +60,14 @@ PyAPI_FUNC(int) _Py_convert_optional_to_non_negative_ssize_t(PyObject *, void *) // Export for 'math' shared extension. PyAPI_FUNC(PyObject*) _PyNumber_Index(PyObject *o); +typedef struct { + PyObject *object; + PySendResult kind; +} PySendResultPair; + +// Same as PyIter_Send but returns a struct for MSVC tailcall support +PyAPI_FUNC(PySendResultPair) _PyIter_Send(PyObject *iter, PyObject *arg); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index 1caf200ee34b2a4..32c12fb5875e8ea 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -161,6 +161,7 @@ struct ast_state { PyObject *__module__; PyObject *_attributes; PyObject *_fields; + PyObject *abstract_types; PyObject *alias_type; PyObject *annotation; PyObject *arg; diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 38dd82f6fc8a140..9d1912cc28921c3 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -108,6 +108,12 @@ backoff_counter_triggers(_Py_BackoffCounter counter) return counter.value_and_backoff < UNREACHABLE_BACKOFF; } +static inline bool +backoff_counter_is_unreachable(_Py_BackoffCounter counter) +{ + return (counter.value_and_backoff & BACKOFF_MASK) == UNREACHABLE_BACKOFF; +} + static inline _Py_BackoffCounter trigger_backoff_counter(void) { diff --git a/Include/internal/pycore_blocks_output_buffer.h b/Include/internal/pycore_blocks_output_buffer.h index 016e7a18665859f..322c1e93344ba30 100644 --- a/Include/internal/pycore_blocks_output_buffer.h +++ b/Include/internal/pycore_blocks_output_buffer.h @@ -242,9 +242,12 @@ static inline PyObject * _BlocksOutputBuffer_Finish(_BlocksOutputBuffer *buffer, const Py_ssize_t avail_out) { + PyObject *obj; assert(buffer->writer != NULL); - return PyBytesWriter_FinishWithSize(buffer->writer, - buffer->allocated - avail_out); + obj = PyBytesWriter_FinishWithSize(buffer->writer, + buffer->allocated - avail_out); + buffer->writer = NULL; + return obj; } /* Clean up the buffer when an error occurred. */ diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 8e8fa696ee0350b..27a7a46152f57b8 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -14,6 +14,11 @@ extern PyObject* _PyBytes_FormatEx( PyObject *args, int use_bytearray); +/* Concatenate two bytes objects. Used as the sq_concat slot and by the + * specializing interpreter. Unlike PyBytes_Concat(), this returns a new + * reference rather than modifying its first argument in place. */ +extern PyObject* _PyBytes_Concat(PyObject *a, PyObject *b); + extern PyObject* _PyBytes_FromHex( PyObject *string, int use_bytearray); @@ -57,9 +62,11 @@ _PyBytes_ReverseFind(const char *haystack, Py_ssize_t len_haystack, // // Export for 'array' shared extension. PyAPI_FUNC(void) -_PyBytes_Repeat(char* dest, Py_ssize_t len_dest, +_PyBytes_RepeatBuffer(char* dest, Py_ssize_t len_dest, const char* src, Py_ssize_t len_src); +PyAPI_FUNC(PyObject *) _PyBytes_Repeat(PyObject *self, Py_ssize_t n); + /* _PyBytesObject_SIZE gives the basic size of a bytes object; any memory allocation for a bytes object of length n should request PyBytesObject_SIZE + n bytes. diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index e544b4cf49d1fbc..a9db8860e91c06c 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -166,7 +166,8 @@ _PyStack_UnpackDict(PyThreadState *tstate, PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, PyObject **p_kwnames); -extern void _PyStack_UnpackDict_Free( +// Exported for external JIT support +PyAPI_FUNC(void) _PyStack_UnpackDict_Free( PyObject *const *stack, Py_ssize_t nargs, PyObject *kwnames); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 2479630021bfe37..19b5473271fffb5 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -94,7 +94,7 @@ typedef struct { void* (*init_state)(void); // Callback to register every trampoline being created void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); // Callback to free the trampoline state int (*free_state)(void* state); } _PyPerf_Callbacks; @@ -108,6 +108,10 @@ extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; +extern void _PyPerfJit_WriteNamedCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); #endif static inline PyObject* @@ -121,18 +125,11 @@ _PyEval_EvalFrame(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwfl } #ifdef _Py_TIER2 -#ifdef _Py_JIT -_Py_CODEUNIT *_Py_LazyJitShim( - struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, - _PyStackRef *stack_pointer, PyThreadState *tstate -); -#else _Py_CODEUNIT *_PyTier2Interpreter( struct _PyExecutorObject *current_executor, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate ); #endif -#endif extern _PyJitEntryFuncPtr _Py_jit_entry; @@ -211,16 +208,16 @@ extern void _PyEval_DeactivateOpCache(void); /* --- _Py_EnterRecursiveCall() ----------------------------------------- */ -static inline int _Py_MakeRecCheck(PyThreadState *tstate) { +static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - // Overflow if stack pointer is between soft limit and the base of the hardware stack. - // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. - // We could have the wrong stack limits because of limited platform support, or user-space threads. + // Possible overflow if stack pointer is beyond the soft limit. + // _Py_CheckRecursiveCall will check for corner cases and + // report an error if there is an overflow. #if _Py_STACK_GROWS_DOWN - return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; + return here_addr < _tstate->c_stack_soft_limit; #else - return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; + return here_addr > _tstate->c_stack_soft_limit; #endif } @@ -235,7 +232,7 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCallPy( static inline int _Py_EnterRecursiveCallTstate(PyThreadState *tstate, const char *where) { - return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); + return (_Py_ReachedRecursionLimit(tstate) && _Py_CheckRecursiveCall(tstate, where)); } static inline int _Py_EnterRecursiveCall(const char *where) { @@ -249,8 +246,6 @@ static inline void _Py_LeaveRecursiveCallTstate(PyThreadState *tstate) { PyAPI_FUNC(void) _Py_InitializeRecursionLimits(PyThreadState *tstate); -PyAPI_FUNC(int) _Py_ReachedRecursionLimit(PyThreadState *tstate); - // Export for test_peg_generator PyAPI_FUNC(int) _Py_ReachedRecursionLimitWithMargin( PyThreadState *tstate, @@ -320,7 +315,7 @@ PyObject * _PyEval_ImportNameWithImport( PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); -PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); @@ -334,6 +329,7 @@ PyAPI_FUNC(PyObject *) _PyEval_LoadName(PyThreadState *tstate, _PyInterpreterFra PyAPI_FUNC(int) _Py_Check_ArgsIterable(PyThreadState *tstate, PyObject *func, PyObject *args); PyAPI_FUNC(_PyStackRef) _PyEval_GetIter(_PyStackRef iterable, _PyStackRef *null_or_index, int yield_from); +PyAPI_FUNC(int) _PyEval_StoreName(PyThreadState *tstate, _PyStackRef v, PyObject *name, PyObject* ns); /* * Indicate whether a special method of given 'oparg' can use the (improved) @@ -433,19 +429,19 @@ _Py_VectorCallInstrumentation_StackRefSteal( PyThreadState* tstate); PyAPI_FUNC(PyObject *) -_Py_BuiltinCallFast_StackRefSteal( +_Py_BuiltinCallFast_StackRef( _PyStackRef callable, _PyStackRef *arguments, int total_args); PyAPI_FUNC(PyObject *) -_Py_BuiltinCallFastWithKeywords_StackRefSteal( +_Py_BuiltinCallFastWithKeywords_StackRef( _PyStackRef callable, _PyStackRef *arguments, int total_args); PyAPI_FUNC(PyObject *) -_PyCallMethodDescriptorFast_StackRefSteal( +_PyCallMethodDescriptorFast_StackRef( _PyStackRef callable, PyCFunctionFast cfunc, PyObject *self, @@ -453,7 +449,7 @@ _PyCallMethodDescriptorFast_StackRefSteal( int total_args); PyAPI_FUNC(PyObject *) -_PyCallMethodDescriptorFastWithKeywords_StackRefSteal( +_PyCallMethodDescriptorFastWithKeywords_StackRef( _PyStackRef callable, PyCFunctionFastWithKeywords cfunc, PyObject *self, @@ -461,7 +457,7 @@ _PyCallMethodDescriptorFastWithKeywords_StackRefSteal( int total_args); PyAPI_FUNC(PyObject *) -_Py_CallBuiltinClass_StackRefSteal( +_Py_CallBuiltinClass_StackRef( _PyStackRef callable, _PyStackRef *arguments, int total_args); diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index fe8d0a54f2af1af..293c1ea4414e23e 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -141,6 +141,12 @@ typedef struct { #define INLINE_CACHE_ENTRIES_FOR_ITER CACHE_ENTRIES(_PyForIterCache) +typedef struct { + _Py_BackoffCounter counter; +} _PyGetIterCache; + +#define INLINE_CACHE_ENTRIES_GET_ITER CACHE_ENTRIES(_PyGetIterCache) + typedef struct { _Py_BackoffCounter counter; } _PySendCache; @@ -199,7 +205,7 @@ _PyLocals_GetKind(PyObject *kinds, int i) { assert(PyBytes_Check(kinds)); assert(0 <= i && i < PyBytes_GET_SIZE(kinds)); - char *ptr = PyBytes_AS_STRING(kinds); + const char *ptr = PyBytes_AS_STRING(kinds); return (_PyLocals_Kind)(ptr[i]); } @@ -324,6 +330,7 @@ PyAPI_FUNC(void) _Py_Specialize_ContainsOp(_PyStackRef value, _Py_CODEUNIT *inst PyAPI_FUNC(void) _Py_GatherStats_GetIter(_PyStackRef iterable); PyAPI_FUNC(void) _Py_Specialize_CallFunctionEx(_PyStackRef func_st, _Py_CODEUNIT *instr); PyAPI_FUNC(void) _Py_Specialize_Resume(_Py_CODEUNIT *instr, PyThreadState *tstate, _PyInterpreterFrame *frame); +PyAPI_FUNC(void) _Py_Specialize_GetIter(_PyStackRef iterable, _Py_CODEUNIT *instr); // Utility functions for reading/writing 32/64-bit values in the inline caches. // Great care should be taken to ensure that these functions remain correct and @@ -503,6 +510,11 @@ typedef struct { aliased to either operand). Used by the tier 2 optimizer to enable inplace follow-up ops. */ int result_unique; + /* Expected types of the left and right operands. Used by the tier 2 + optimizer to eliminate _GUARD_BINARY_OP_EXTEND when the operand + types are already known. NULL means unknown/don't eliminate. */ + PyTypeObject *lhs_type; + PyTypeObject *rhs_type; } _PyBinaryOpSpecializationDescr; /* Comparison bit masks. */ @@ -527,7 +539,8 @@ typedef struct { PyAPI_FUNC(int) _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); -extern _Py_CODEUNIT _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); +// Export for '_testinternalcapi' shared extension +PyAPI_FUNC(_Py_CODEUNIT) _Py_GetBaseCodeUnit(PyCodeObject *code, int offset); extern int _PyInstruction_GetLength(PyCodeObject *code, int offset); @@ -569,6 +582,10 @@ PyAPI_FUNC(_Py_CODEUNIT *) _PyCode_GetTLBC(PyCodeObject *co); // Returns the reserved index or -1 on error. extern int32_t _Py_ReserveTLBCIndex(PyInterpreterState *interp); +// Release an index returned by _Py_ReserveTLBCIndex() that was never stored +// in a PyThreadState. +extern void _Py_UnreserveTLBCIndex(PyInterpreterState *interp, int32_t index); + // Release the current thread's index into thread-local bytecode arrays extern void _Py_ClearTLBCIndex(_PyThreadStateImpl *tstate); diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index 52dca1362592d69..bfa10eadf735731 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name); in Python 3.5+? */ -extern PyObject* _PyCodec_LookupTextEncoding( +PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding( const char *encoding, const char *alternate_command); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 911cc1f10f15131..7e248429af8eb8a 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -137,6 +137,7 @@ int _PyCompile_EnterScope(struct _PyCompiler *c, identifier name, int scope_type void *key, int lineno, PyObject *private, _PyCompile_CodeUnitMetadata *umd); void _PyCompile_ExitScope(struct _PyCompiler *c); +int _PyCompile_SetQualname(struct _PyCompiler *c); Py_ssize_t _PyCompile_AddConst(struct _PyCompiler *c, PyObject *o); _PyInstructionSequence *_PyCompile_InstrSequence(struct _PyCompiler *c); int _PyCompile_StartAnnotationSetup(struct _PyCompiler *c); diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 2a2846b1296b901..51d99d74ca159f6 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -196,10 +196,9 @@ _PyCriticalSection2_Begin(PyThreadState *tstate, PyCriticalSection2 *c, PyObject static inline void _PyCriticalSection2_End(PyThreadState *tstate, PyCriticalSection2 *c) { - // if mutex1 is NULL, we used the fast path in - // _PyCriticalSection_BeginSlow for mutexes that are already held, - // which should only happen when mutex1 and mutex2 were the same mutex, - // and mutex2 should also be NULL. + // if mutex1 is NULL, we used the fast path in either + // _PyCriticalSection_BeginSlow or _PyCriticalSection2_BeginSlow for mutexes + // that are already held, and mutex2 should also be NULL. if (c->_cs_base._cs_mutex == NULL) { assert(c->_cs_mutex2 == NULL); return; diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index c166f963da4f669..6e1eb573c8c2c83 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -104,6 +104,7 @@ typedef struct _Py_DebugOffsets { uint64_t current_frame; uint64_t base_frame; uint64_t last_profiled_frame; + uint64_t last_profiled_frame_seq; uint64_t thread_id; uint64_t native_thread_id; uint64_t datastack_chunk; @@ -158,8 +159,16 @@ typedef struct _Py_DebugOffsets { uint64_t tp_name; uint64_t tp_repr; uint64_t tp_flags; + uint64_t tp_basicsize; + uint64_t tp_dictoffset; } type_object; + // PyHeapTypeObject offset; + struct _heap_type_object { + uint64_t size; + uint64_t ht_cached_keys; + } heap_type_object; + // PyTuple object offset; struct _tuple_object { uint64_t size; @@ -215,6 +224,7 @@ typedef struct _Py_DebugOffsets { uint64_t state; uint64_t length; uint64_t asciiobject_size; + uint64_t compactunicodeobject_size; } unicode_object; // GC runtime state offset; @@ -285,6 +295,7 @@ typedef struct _Py_DebugOffsets { .current_frame = offsetof(PyThreadState, current_frame), \ .base_frame = offsetof(PyThreadState, base_frame), \ .last_profiled_frame = offsetof(PyThreadState, last_profiled_frame), \ + .last_profiled_frame_seq = offsetof(PyThreadState, last_profiled_frame_seq), \ .thread_id = offsetof(PyThreadState, thread_id), \ .native_thread_id = offsetof(PyThreadState, native_thread_id), \ .datastack_chunk = offsetof(PyThreadState, datastack_chunk), \ @@ -329,6 +340,12 @@ typedef struct _Py_DebugOffsets { .tp_name = offsetof(PyTypeObject, tp_name), \ .tp_repr = offsetof(PyTypeObject, tp_repr), \ .tp_flags = offsetof(PyTypeObject, tp_flags), \ + .tp_basicsize = offsetof(PyTypeObject, tp_basicsize), \ + .tp_dictoffset = offsetof(PyTypeObject, tp_dictoffset), \ + }, \ + .heap_type_object = { \ + .size = sizeof(PyHeapTypeObject), \ + .ht_cached_keys = offsetof(PyHeapTypeObject, ht_cached_keys), \ }, \ .tuple_object = { \ .size = sizeof(PyTupleObject), \ @@ -370,6 +387,7 @@ typedef struct _Py_DebugOffsets { .state = offsetof(PyUnicodeObject, _base._base.state), \ .length = offsetof(PyUnicodeObject, _base._base.length), \ .asciiobject_size = sizeof(PyASCIIObject), \ + .compactunicodeobject_size = sizeof(PyCompactUnicodeObject), \ }, \ .gc = { \ .size = sizeof(struct _gc_runtime_state), \ diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index d58539fa8465636..7032b61d7654b89 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -31,7 +31,8 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); -extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, Py_hash_t hash); extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); @@ -43,6 +44,10 @@ extern int _PyDict_Next( extern int _PyDict_HasOnlyStringKeys(PyObject *mp); +PyAPI_FUNC(PyObject *) _PyDict_Subscript(PyObject *self, PyObject *key); +PyAPI_FUNC(PyObject *) _PyDict_SubscriptKnownHash(PyObject *self, PyObject *key, Py_hash_t hash); +PyAPI_FUNC(int) _PyDict_StoreSubscript(PyObject *self, PyObject *key, PyObject *value); + // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); @@ -85,11 +90,18 @@ typedef struct { } PyDictUnicodeEntry; extern PyDictKeysObject *_PyDict_NewKeysForClass(PyHeapTypeObject *); +extern void _PyDict_RemoveKeysForClass(PyHeapTypeObject *); extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *); +/* Implementations of the `|` and `|=` operators for dict, used by the + * specializing interpreter. */ +extern PyObject *_PyDict_Or(PyObject *self, PyObject *other); +extern PyObject *_PyDict_IOr(PyObject *self, PyObject *other); + /* Gets a version number unique to the current state of the keys of dict, if possible. - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDictKeys_GetVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDictKeys_GetVersionForCurrentState( PyInterpreterState *interp, PyDictKeysObject *dictkeys); /* Gets a version number unique to the current state of the keys of dict, if possible. @@ -99,8 +111,9 @@ extern uint32_t _PyDictKeys_GetVersionForCurrentState( * * The caller must hold the per-object lock on dict. * - * Returns the version number, or zero if it was not possible to get a version number. */ -extern uint32_t _PyDict_GetKeysVersionForCurrentState( + * Returns the version number, or zero if it was not possible to get a version number. + * Exported for external JIT support */ +PyAPI_FUNC(uint32_t) _PyDict_GetKeysVersionForCurrentState( PyInterpreterState *interp, PyDictObject *dict); extern size_t _PyDict_KeysSize(PyDictKeysObject *keys); @@ -109,16 +122,18 @@ extern void _PyDictKeys_DecRef(PyDictKeysObject *keys); /* _Py_dict_lookup() returns index of entry which can be used like DK_ENTRIES(dk)[index]. * -1 when no entry found, -3 when compare raises error. + * Exported for external JIT support */ -extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); +PyAPI_FUNC(Py_ssize_t) _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method); -extern Py_ssize_t _PyDict_LookupIndexAndValue(PyDictObject *, PyObject *, PyObject **); -extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); -extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); +// Exported for external JIT support +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndexAndValue(PyDictObject *, PyObject *, PyObject **); +PyAPI_FUNC(Py_ssize_t) _PyDict_LookupIndex(PyDictObject *, PyObject *); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); /* Look up a string key in an all unicode dict keys, assign the keys object a version, and * store it in version. @@ -127,9 +142,11 @@ extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject * strings. * * Returns DKIX_EMPTY if the key is not present. + * + * Exported for external JIT support */ -extern Py_ssize_t _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); -extern Py_ssize_t _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupAndVersion(PyDictKeysObject* dictkeys, PyObject *key, uint32_t *version); +PyAPI_FUNC(Py_ssize_t) _PyDictKeys_StringLookupSplit(PyDictKeysObject* dictkeys, PyObject *key); PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); PyAPI_FUNC(void) _PyDict_LoadGlobalStackRef(PyDictObject *, PyDictObject *, PyObject *, _PyStackRef *); @@ -139,7 +156,8 @@ extern PyObject *_PyDict_LoadBuiltinsFromGlobals(PyObject *globals); /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); PyAPI_FUNC(int) _PyDict_SetItem_Take2_KnownHash(PyDictObject *op, PyObject *key, PyObject *value, Py_hash_t hash); -extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +// Exported for external JIT support +PyAPI_FUNC(int) _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_SetItem_KnownHash_LockHeld(PyDictObject *mp, PyObject *key, PyObject *value, Py_hash_t hash); @@ -222,6 +240,17 @@ struct _dictkeysobject { see the DK_ENTRIES() / DK_UNICODE_ENTRIES() functions below */ }; +struct _instancekeysobject { + PyTypeObject* dsk_owning_type; + struct _dictkeysobject dsk_keys; +}; + +static inline struct _instancekeysobject *_PyDictKeys_AsSharedKeys(struct _dictkeysobject *keys) +{ + assert(keys->dk_kind == DICT_KEYS_SPLIT); + return _Py_CONTAINER_OF(keys, struct _instancekeysobject, dsk_keys); +} + /* This must be no more than 250, for the prefix size to fit in one byte. */ #define SHARED_KEYS_MAX_SIZE 30 #define NEXT_LOG2_SHARED_KEYS_MAX_SIZE 6 @@ -292,7 +321,7 @@ _PyDict_NotifyEvent(PyDict_WatchEvent event, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; + int watcher_bits = FT_ATOMIC_LOAD_UINT64_ACQUIRE(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); @@ -324,6 +353,10 @@ _PyDictValues_AddToInsertionOrder(PyDictValues *values, Py_ssize_t ix) values->size = size+1; } +// Exported for external JIT support +PyAPI_FUNC(void) +_PyDict_InsertSplitValue(PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix); + static inline size_t shared_keys_usable_size(PyDictKeysObject *keys) { @@ -368,7 +401,7 @@ PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); static inline Py_ssize_t _PyDict_UniqueId(PyDictObject *mp) { - return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT); + return (Py_ssize_t)(FT_ATOMIC_LOAD_UINT64_RELAXED(mp->_ma_watcher_tag) >> DICT_UNIQUE_ID_SHIFT); } static inline void diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab60..bb6fe2625975596 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_faulthandler.h b/Include/internal/pycore_faulthandler.h index 78cd657e6ae5aee..9ddd70d39ed0d51 100644 --- a/Include/internal/pycore_faulthandler.h +++ b/Include/internal/pycore_faulthandler.h @@ -42,6 +42,7 @@ struct faulthandler_user_signal { int chain; _Py_sighandler_t previous; PyInterpreterState *interp; + Py_ssize_t max_threads; }; #endif /* FAULTHANDLER_USER */ @@ -57,6 +58,7 @@ struct _faulthandler_runtime_state { void *exc_handler; #endif int c_stack; + Py_ssize_t max_threads; } fatal_error; struct { @@ -68,6 +70,7 @@ struct _faulthandler_runtime_state { int exit; char *header; size_t header_len; + Py_ssize_t max_threads; /* The main thread always holds this lock. It is only released when faulthandler_thread() is interrupted before this thread exits, or at Python exit. */ diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 2c6d6daa01994ee..83cdc4f3dfdd44a 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -302,6 +302,16 @@ extern void _Py_skiproot(const wchar_t *path, Py_ssize_t size, Py_ssize_t *drvsi // Export for 'select' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_FileDescriptor_Converter(PyObject *, void *); +#ifdef MS_WINDOWS +/* Windows uses long long for offsets */ +typedef long long Py_off_t; +#else +typedef off_t Py_off_t; +#endif + +// Export for '_ssl' and 'zlib' shared extensions (Argument Clinic code) +PyAPI_FUNC(int) _Py_Off_t_Converter(PyObject *, void *); + // Export for test_peg_generator PyAPI_FUNC(char*) _Py_UniversalNewlineFgetsWithSize(char *, int, FILE*, PyObject *, size_t*); diff --git a/Include/internal/pycore_fileutils_windows.h b/Include/internal/pycore_fileutils_windows.h index b79aa9fb4653765..4c42f16beb12a4f 100644 --- a/Include/internal/pycore_fileutils_windows.h +++ b/Include/internal/pycore_fileutils_windows.h @@ -55,7 +55,11 @@ static inline BOOL _Py_GetFileInformationByName( static int GetFileInformationByName_init = -1; if (GetFileInformationByName_init < 0) { +#ifdef MS_WINDOWS_DESKTOP HMODULE hMod = LoadLibraryW(L"api-ms-win-core-file-l2-1-4"); +#else + HMODULE hMod = LoadPackagedLibrary(L"api-ms-win-core-file-l2-1-4", 0); +#endif GetFileInformationByName_init = 0; if (hMod) { GetFileInformationByName = (PGetFileInformationByName)GetProcAddress( diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 5e73ae3c8549b84..4eefbf89acb85f7 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -34,11 +34,13 @@ struct _frame { * "support" for the borrowed references, ensuring that they remain valid. */ PyObject *f_overwritten_fast_locals; + PyObject *f_weakreflist; /* List of weak references */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; -extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); +// Exported for external JIT support +PyAPI_FUNC(PyFrameObject *) _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 99dacaf0fe7c545..2184f40956d4e24 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -27,7 +27,8 @@ _PyFunction_IsVersionValid(uint32_t version) return version >= FUNC_VERSION_FIRST_VALID; } -extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); +// Exported for external JIT support +PyAPI_FUNC(uint32_t) _PyFunction_GetVersionForCurrentState(PyFunctionObject *func); PyAPI_FUNC(void) _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version); void _PyFunction_ClearCodeByVersion(uint32_t version); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index fd284d0e4ecc2f5..84cbb56a9192156 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,21 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -/* Bit 0 in _gc_next is the old space bit. - * It is set as follows: - * Young: gcstate->visited_space - * old[0]: 0 - * old[1]: 1 - * permanent: 0 - * - * During a collection all objects handled should have the bit set to - * gcstate->visited_space, as objects are moved from the young gen - * and the increment into old[gcstate->visited_space]. - * When object are moved from the pending space, old[gcstate->visited_space^1] - * into the increment, the old space bit is flipped. -*/ -#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 - #define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) @@ -159,13 +144,11 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; + uintptr_t next = gc->_gc_next; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - uintptr_t unext = (uintptr_t)next; - assert((unext & ~_PyGC_PREV_MASK) == 0); - gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -207,10 +190,6 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { extern void _Py_ScheduleGC(PyThreadState *tstate); -#ifndef Py_GIL_DISABLED -extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); -#endif - /* Tell the GC to track this object. * @@ -220,7 +199,7 @@ extern void _Py_TriggerGC(struct _gc_runtime_state *gcstate); * ob_traverse method. * * Internal note: interp->gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and + * because it's not an object header. So we don't use _PyGCHead_PREV() and * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. * * See also the public PyObject_GC_Track() function. @@ -245,18 +224,13 @@ static inline void _PyObject_GC_TRACK( filename, lineno, __func__); struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; - PyGC_Head *generation0 = &gcstate->young.head; + PyGC_Head *generation0 = gcstate->generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ gcstate->visited_space; - gc->_gc_next = ((uintptr_t)generation0) | not_visited; + _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; - gcstate->young.count++; /* number of tracked GC objects */ gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold) { - _Py_TriggerGC(gcstate); - } #endif } @@ -292,9 +266,6 @@ static inline void _PyObject_GC_UNTRACK( gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; - if (gcstate->young.count > 0) { - gcstate->young.count--; - } gcstate->heap_size--; #endif } @@ -364,8 +335,9 @@ extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; // GC visit callback for tracked interpreter frames -extern int _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); -extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); +// GH-150766: exported for greenlet +PyAPI_FUNC(int) _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg); +PyAPI_FUNC(int) _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); #ifdef Py_GIL_DISABLED extern void _PyGC_VisitObjectsWorldStopped(PyInterpreterState *interp, diff --git a/Include/internal/pycore_genobject.h b/Include/internal/pycore_genobject.h index a3badb59cb771ae..266add0fb7a9c34 100644 --- a/Include/internal/pycore_genobject.h +++ b/Include/internal/pycore_genobject.h @@ -29,14 +29,22 @@ PyAPI_FUNC(int) _PyGen_SetStopIterationValue(PyObject *); // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **); +// Set the exception passed to throw(typ[, val[, tb]]). +// Return 0 on success, -1 on failure. +extern int _PyGen_SetException(PyObject *typ, PyObject *val, PyObject *tb); PyAPI_FUNC(PyObject *)_PyCoro_GetAwaitableIter(PyObject *o); PyAPI_FUNC(PyObject *)_PyAsyncGenValueWrapperNew(PyThreadState *state, PyObject *); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyCoro_ComputeOrigin(int origin_depth, _PyInterpreterFrame *current_frame); + extern PyTypeObject _PyCoroWrapper_Type; extern PyTypeObject _PyAsyncGenWrappedValue_Type; extern PyTypeObject _PyAsyncGenAThrow_Type; +PyAPI_FUNC(PySendResult) _PyAsyncGenASend_Send(PyObject *iter, PyObject *arg, PyObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index beae65213a27b68..4fd6c618fb74404 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -1544,6 +1544,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_filters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_finalizing)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_find_and_load_lazy_submodule)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_fix_up_module)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_flags_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_get_sourcefile)); @@ -1582,6 +1583,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_interpreters)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alphabet)); @@ -1594,8 +1596,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(asend)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(assoclen)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ast)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(athrow)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(autocommit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(backtick)); @@ -1636,6 +1640,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(cancel)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(canonical)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capath)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(capitals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(category)); @@ -1719,6 +1724,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dont_inherit)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dst_dir_fd)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(duration_sec)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eager_start)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(effective_ids)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(element_factory)); @@ -1732,6 +1738,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(endpos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(entrypoint)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(env)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(error_rate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(errors)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(event)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(eventmask)); @@ -1791,7 +1798,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(generation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_debug)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_event_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_loop)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(get_source)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(getattr)); @@ -1808,6 +1814,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(headers)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hi)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hook)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(host)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hour)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(hours)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(id)); @@ -1856,6 +1863,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(iter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(iterable)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(iterations)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(iv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(join)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(jump)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(keepends)); @@ -1896,6 +1904,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mask)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(match)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_length)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(max_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxdigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxevents)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(maxlen)); @@ -1914,6 +1923,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(milliseconds)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(minutes)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(missed_samples)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mod)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(mode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(module)); @@ -1933,6 +1943,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespace_separator)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(namespaces)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(native)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nbytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ndigits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(nested)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(new_file_name)); @@ -1957,6 +1968,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(onceregistry)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_active_thread)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(op)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcodes)); @@ -1976,6 +1988,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pad)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(padded)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pages)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pair)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parameter)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(parent)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(password)); @@ -1987,8 +2000,10 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(person)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pi_factory)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pid)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pidfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pointer_bits)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(policy)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(port)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos1)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(pos2)); @@ -2028,6 +2043,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repeat)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repl)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(replace)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(repr)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reqrefs)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(require_ready)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(reserved)); @@ -2040,6 +2056,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(rounding)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(salt)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_interval_us)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sample_rate)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(sched_priority)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(scheduler)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(script)); @@ -2091,6 +2108,8 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stdout)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(step)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(steps)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stop_exception)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(stop_value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(store_name)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strategy)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(strftime)); @@ -2105,6 +2124,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(take_bytes)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(target_is_directory)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(targetfd)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(task)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_frame)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(tb_lasti)); @@ -2147,6 +2167,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(updates)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(uri)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(usedforsecurity)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(utcoffset)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(value)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(values)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(version)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index bb1c6dbaf039067..5b35c53e0aa03b1 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -267,6 +267,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(_filters) STRUCT_FOR_ID(_finalizing) STRUCT_FOR_ID(_find_and_load) + STRUCT_FOR_ID(_find_and_load_lazy_submodule) STRUCT_FOR_ID(_fix_up_module) STRUCT_FOR_ID(_flags_) STRUCT_FOR_ID(_get_sourcefile) @@ -305,6 +306,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_interpreters) STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(alphabet) @@ -317,8 +319,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(argv) STRUCT_FOR_ID(as_integer_ratio) STRUCT_FOR_ID(asend) + STRUCT_FOR_ID(assoclen) STRUCT_FOR_ID(ast) STRUCT_FOR_ID(athrow) + STRUCT_FOR_ID(attr) STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(autocommit) STRUCT_FOR_ID(backtick) @@ -359,6 +363,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(callable) STRUCT_FOR_ID(callback) STRUCT_FOR_ID(cancel) + STRUCT_FOR_ID(canonical) STRUCT_FOR_ID(capath) STRUCT_FOR_ID(capitals) STRUCT_FOR_ID(category) @@ -442,6 +447,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(dont_inherit) STRUCT_FOR_ID(dst) STRUCT_FOR_ID(dst_dir_fd) + STRUCT_FOR_ID(duration_sec) STRUCT_FOR_ID(eager_start) STRUCT_FOR_ID(effective_ids) STRUCT_FOR_ID(element_factory) @@ -455,6 +461,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(endpos) STRUCT_FOR_ID(entrypoint) STRUCT_FOR_ID(env) + STRUCT_FOR_ID(error_rate) STRUCT_FOR_ID(errors) STRUCT_FOR_ID(event) STRUCT_FOR_ID(eventmask) @@ -514,7 +521,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(generation) STRUCT_FOR_ID(get) STRUCT_FOR_ID(get_debug) - STRUCT_FOR_ID(get_event_loop) STRUCT_FOR_ID(get_loop) STRUCT_FOR_ID(get_source) STRUCT_FOR_ID(getattr) @@ -531,6 +537,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(headers) STRUCT_FOR_ID(hi) STRUCT_FOR_ID(hook) + STRUCT_FOR_ID(host) STRUCT_FOR_ID(hour) STRUCT_FOR_ID(hours) STRUCT_FOR_ID(id) @@ -579,6 +586,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(iter) STRUCT_FOR_ID(iterable) STRUCT_FOR_ID(iterations) + STRUCT_FOR_ID(iv) STRUCT_FOR_ID(join) STRUCT_FOR_ID(jump) STRUCT_FOR_ID(keepends) @@ -619,6 +627,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(mask) STRUCT_FOR_ID(match) STRUCT_FOR_ID(max_length) + STRUCT_FOR_ID(max_threads) STRUCT_FOR_ID(maxdigits) STRUCT_FOR_ID(maxevents) STRUCT_FOR_ID(maxlen) @@ -637,6 +646,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(milliseconds) STRUCT_FOR_ID(minute) STRUCT_FOR_ID(minutes) + STRUCT_FOR_ID(missed_samples) STRUCT_FOR_ID(mod) STRUCT_FOR_ID(mode) STRUCT_FOR_ID(module) @@ -656,6 +666,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(namespace_separator) STRUCT_FOR_ID(namespaces) STRUCT_FOR_ID(native) + STRUCT_FOR_ID(nbytes) STRUCT_FOR_ID(ndigits) STRUCT_FOR_ID(nested) STRUCT_FOR_ID(new_file_name) @@ -680,6 +691,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(onceregistry) STRUCT_FOR_ID(only_active_thread) STRUCT_FOR_ID(only_keys) + STRUCT_FOR_ID(op) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) STRUCT_FOR_ID(opcodes) @@ -699,6 +711,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(pad) STRUCT_FOR_ID(padded) STRUCT_FOR_ID(pages) + STRUCT_FOR_ID(pair) STRUCT_FOR_ID(parameter) STRUCT_FOR_ID(parent) STRUCT_FOR_ID(password) @@ -710,8 +723,10 @@ struct _Py_global_strings { STRUCT_FOR_ID(person) STRUCT_FOR_ID(pi_factory) STRUCT_FOR_ID(pid) + STRUCT_FOR_ID(pidfd) STRUCT_FOR_ID(pointer_bits) STRUCT_FOR_ID(policy) + STRUCT_FOR_ID(port) STRUCT_FOR_ID(pos) STRUCT_FOR_ID(pos1) STRUCT_FOR_ID(pos2) @@ -751,6 +766,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(repeat) STRUCT_FOR_ID(repl) STRUCT_FOR_ID(replace) + STRUCT_FOR_ID(repr) STRUCT_FOR_ID(reqrefs) STRUCT_FOR_ID(require_ready) STRUCT_FOR_ID(reserved) @@ -763,6 +779,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(rounding) STRUCT_FOR_ID(salt) STRUCT_FOR_ID(sample_interval_us) + STRUCT_FOR_ID(sample_rate) STRUCT_FOR_ID(sched_priority) STRUCT_FOR_ID(scheduler) STRUCT_FOR_ID(script) @@ -814,6 +831,8 @@ struct _Py_global_strings { STRUCT_FOR_ID(stdout) STRUCT_FOR_ID(step) STRUCT_FOR_ID(steps) + STRUCT_FOR_ID(stop_exception) + STRUCT_FOR_ID(stop_value) STRUCT_FOR_ID(store_name) STRUCT_FOR_ID(strategy) STRUCT_FOR_ID(strftime) @@ -828,6 +847,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(take_bytes) STRUCT_FOR_ID(target) STRUCT_FOR_ID(target_is_directory) + STRUCT_FOR_ID(targetfd) STRUCT_FOR_ID(task) STRUCT_FOR_ID(tb_frame) STRUCT_FOR_ID(tb_lasti) @@ -870,6 +890,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(updates) STRUCT_FOR_ID(uri) STRUCT_FOR_ID(usedforsecurity) + STRUCT_FOR_ID(utcoffset) STRUCT_FOR_ID(value) STRUCT_FOR_ID(values) STRUCT_FOR_ID(version) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 32ed3a62b2b4a7c..669e328c266d00f 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,6 +39,13 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); +typedef enum { + _Py_LAZY_SUBMODULE_ERROR = -1, + _Py_LAZY_SUBMODULE_NOT_FOUND = 0, + _Py_LAZY_SUBMODULE_LOADED = 1, +} _PyLazySubmoduleImportResult; +extern _PyLazySubmoduleImportResult _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name, PyObject **result); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index f60c5510d200752..9ed87a544234c59 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -124,7 +124,7 @@ extern void _Py_ext_module_loader_result_apply_error( /* The module init function. */ typedef PyObject *(*PyModInitFunction)(void); -typedef PyModuleDef_Slot *(*PyModExportFunction)(void); +typedef PySlot *(*PyModExportFunction)(void); #ifdef HAVE_DYNAMIC_LOADING extern int _PyImport_GetModuleExportHooks( struct _Py_ext_module_loader_info *info, diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 3775b074ecf54cf..56b55e93a014cb5 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -64,25 +64,21 @@ PyAPI_FUNC(void) _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern int -_Py_Instrumentation_GetLine(PyCodeObject *code, int index); - PyAPI_DATA(PyObject) _PyInstrumentation_MISSING; PyAPI_DATA(PyObject) _PyInstrumentation_DISABLE; /* Total tool ids available */ #define PY_MONITORING_TOOL_IDS 8 -/* Count of all local monitoring events */ -#define _PY_MONITORING_LOCAL_EVENTS 11 -/* Count of all "real" monitoring events (not derived from other events) */ +/* Count of all "real" monitoring events (not derived from other events). + * "Other" events can now be turned on/disabled per code object. */ #define _PY_MONITORING_UNGROUPED_EVENTS 16 /* Count of all monitoring events */ #define _PY_MONITORING_EVENTS 19 /* Tables of which tools are active for each monitored event. */ typedef struct _Py_LocalMonitors { - uint8_t tools[_PY_MONITORING_LOCAL_EVENTS]; + uint8_t tools[_PY_MONITORING_UNGROUPED_EVENTS]; } _Py_LocalMonitors; typedef struct _Py_GlobalMonitors { @@ -122,6 +118,17 @@ typedef struct _PyCoMonitoringData { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); + +static inline uint8_t +_PyCode_GetOriginalOpcode(_PyCoLineInstrumentationData *line_data, int index) +{ + return line_data->data[index*line_data->bytes_per_entry]; +} + +// Exported for external JIT support +PyAPI_FUNC(uint8_t) _PyCode_Deinstrument(uint8_t opcode); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c4b084642668a9f..6c907e0cf79894d 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -69,7 +69,7 @@ struct code_arena_st; struct trampoline_api_st { void* (*init_state)(void); void (*write_state)(void* state, const void *code_addr, - unsigned int code_size, PyCodeObject* code); + size_t code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; Py_ssize_t code_padding; @@ -181,31 +181,18 @@ struct gc_generation { struct gc_generation_stats { PyTime_t ts_start; PyTime_t ts_stop; - - /* heap_size on the start of the collection */ - Py_ssize_t heap_size; - - /* work_to_do on the start of the collection */ - Py_ssize_t work_to_do; - /* total number of collections */ Py_ssize_t collections; - - /* total number of visited objects */ - Py_ssize_t object_visits; - /* total number of collected objects */ Py_ssize_t collected; /* total number of uncollectable objects (put into gc.garbage) */ Py_ssize_t uncollectable; // Total number of objects considered for collection and traversed: Py_ssize_t candidates; - - Py_ssize_t objects_transitively_reachable; - Py_ssize_t objects_not_transitively_reachable; - // Total duration of the collection in seconds: double duration; + /* heap_size on the start of the collection */ + Py_ssize_t heap_size; }; #ifdef Py_GIL_DISABLED @@ -225,11 +212,6 @@ struct gc_old_stats_buffer { int8_t index; }; -enum _GCPhase { - GC_PHASE_MARK = 0, - GC_PHASE_COLLECT = 1 -}; - /* If we change this, we need to change the default value in the signature of gc.collect and change the size of PyStats.gc_stats */ #define NUM_GENERATIONS 3 @@ -244,11 +226,21 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ +#ifndef Py_GIL_DISABLED + struct gc_generation generations[NUM_GENERATIONS]; +#else struct gc_generation young; struct gc_generation old[2]; +#endif /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_stats *generation_stats; +#ifdef Py_GIL_DISABLED + /* Protects generation_stats between gc_get_stats_impl() (reader) and + gc_collect_main() (writer); both resolve the stats slot (buffer index) + under this lock so they agree on which slot to touch (gh-151646). */ + PyMutex stats_mutex; +#endif /* true if we are currently running the collector */ int collecting; // The frame that started the current collection. It might be NULL even when @@ -259,13 +251,9 @@ struct _gc_runtime_state { /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* The number of live objects. */ Py_ssize_t heap_size; - Py_ssize_t work_to_do; - /* Which of the old spaces is the visited space */ - int visited_space; - int phase; -#ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -278,21 +266,31 @@ struct _gc_runtime_state { the first time. */ Py_ssize_t long_lived_pending; +#ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; - - /* Memory usage of the process (RSS + swap) after last GC. */ - Py_ssize_t last_mem; - - /* This accumulates the new object count whenever collection is deferred - due to the RSS increase condition not being meet. Reset on collection. */ - Py_ssize_t deferred_count; - - /* Mutex held for gc_should_collect_mem_usage(). */ - PyMutex mutex; +#else + PyGC_Head *generation0; #endif }; +#ifndef Py_GIL_DISABLED +#define GC_GENERATION_INIT \ + .generations = { \ + { .threshold = 2000, }, \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, \ + .heap_size = 0, +#else +#define GC_GENERATION_INIT \ + .young = { .threshold = 2000, }, \ + .old = { \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#endif + #include "pycore_gil.h" // struct _gil_runtime_state /**** Import ********/ @@ -316,8 +314,6 @@ struct _import_runtime_state { Modules are added there and looked up in _imp.find_extension(). */ struct _Py_hashtable_t *hashtable; } extensions; - /* Package context -- the full module name for package imports */ - const char * pkgcontext; }; struct _import_state { @@ -357,7 +353,15 @@ struct _import_state { int lazy_imports_mode; PyObject *lazy_imports_filter; PyObject *lazy_importing_modules; + // The set stored in sys.lazy_modules if values that have been + // lazily imported. This value is only for debugging/introspection + // purposes and is not used by the runtime. PyObject *lazy_modules; + // A dict mapping package names to a set of submodule names that + // have been imported lazily from packages which have been imported + // lazily. When the package is reified we need to add a + // LazyImportObject which refers to the submodule on the module. + PyObject *lazy_pending_submodules; #ifdef Py_GIL_DISABLED PyMutex lazy_mutex; #endif @@ -449,6 +453,9 @@ typedef struct _PyOptimizationConfig { uint16_t side_exit_initial_value; uint16_t side_exit_initial_backoff; + // Trace fitness thresholds + uint16_t fitness_initial; + // Optimization flags bool specialization_enabled; bool uops_optimize_enabled; @@ -527,8 +534,13 @@ struct _py_func_state { /****** type state *********/ /* For now we hard-code this to a value for which we are confident - all the static builtin types will fit (for all builds). */ -#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES 202 + all the static builtin types will fit (for all builds). + If you add a new static type to the standard library, you may have to + update one of these numbers. + */ +#define _Py_NUM_MANAGED_PREINITIALIZED_TYPES 122 +#define _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES \ + (_Py_NUM_MANAGED_PREINITIALIZED_TYPES + 83) #define _Py_MAX_MANAGED_STATIC_EXT_TYPES 10 #define _Py_MAX_MANAGED_STATIC_TYPES \ (_Py_MAX_MANAGED_STATIC_BUILTIN_TYPES + _Py_MAX_MANAGED_STATIC_EXT_TYPES) @@ -548,23 +560,6 @@ struct _types_runtime_state { }; -// Type attribute lookup cache: speed up attribute and method lookups, -// see _PyType_Lookup(). -struct type_cache_entry { - unsigned int version; // initialized from type->tp_version_tag -#ifdef Py_GIL_DISABLED - _PySeqLock sequence; -#endif - PyObject *name; // reference to exactly a str or None - PyObject *value; // borrowed reference or NULL -}; - -#define MCACHE_SIZE_EXP 12 - -struct type_cache { - struct type_cache_entry hashtable[1 << MCACHE_SIZE_EXP]; -}; - typedef struct { PyTypeObject *type; int isbuiltin; @@ -579,6 +574,10 @@ typedef struct { are also some diagnostic uses for the list of weakrefs, so we still keep it. */ PyObject *tp_weaklist; + /* Per-interpreter attribute lookup cache (struct type_cache *). + For static builtin types the cache must be per-interpreter + because tp_dict and the values it stores are per-interpreter. */ + void *_tp_cache; } managed_static_type_state; #define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ @@ -589,8 +588,6 @@ struct types_state { where all those lower numbers are used for core static types. */ unsigned int next_version_tag; - struct type_cache type_cache; - /* Every static builtin type is initialized for each interpreter during its own initialization, including for the main interpreter during global runtime initialization. This is done by calling @@ -824,6 +821,8 @@ struct _Py_unique_id_pool { typedef _Py_CODEUNIT *(*_PyJitEntryFuncPtr)(struct _PyExecutorObject *exec, _PyInterpreterFrame *frame, _PyStackRef *stack_pointer, PyThreadState *tstate); +#define _PyInterpreterGuard_GUARDS_NOT_ALLOWED UINTPTR_MAX + /* PyInterpreterState holds the global state for one of the runtime's interpreters. Typically the initial (main) interpreter is the only one. @@ -927,6 +926,7 @@ struct _is { PyObject *builtins_copy; // Initialized to _PyEval_EvalFrameDefault(). _PyFrameEvalFunction eval_frame; + int eval_frame_allow_specialization; PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS]; // One bit is set for each non-NULL entry in func_watchers @@ -998,7 +998,7 @@ struct _is { struct ast_state ast; struct types_state types; struct callable_cache callable_cache; - PyObject *common_consts[NUM_COMMON_CONSTANTS]; + _PyStackRef common_consts[NUM_COMMON_CONSTANTS]; bool jit; bool compiling; @@ -1049,6 +1049,11 @@ struct _is { #endif #endif + // The number of remaining finalization guards. + // If this is _PyInterpreterGuard_GUARDS_NOT_ALLOWED, then finalization + // guards can no longer be created. + uintptr_t finalization_guards; + /* the initial PyInterpreterState.threads.head */ _PyThreadStateImpl _initial_thread; // _initial_thread should be the last field of PyInterpreterState. diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index d744dd12cd04795..812e1a28debef44 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -7,6 +7,7 @@ #include "pycore_code.h" // _PyCode_CODE() #include "pycore_interpframe_structs.h" // _PyInterpreterFrame +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE() #include "pycore_stackref.h" // PyStackRef_AsPyObjectBorrow() #include "pycore_stats.h" // CALL_STAT_INC() @@ -152,6 +153,7 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * // visited is GC bookkeeping for the current stack walk, not frame state. dest->visited = 0; #ifdef Py_DEBUG + dest->stackpointer_valid = src->stackpointer_valid; dest->lltrace = src->lltrace; #endif for (int i = 0; i < stacktop; i++) { @@ -207,6 +209,7 @@ _PyFrame_Initialize( frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif @@ -230,21 +233,51 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) static inline _PyStackRef* _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - assert(frame->stackpointer != NULL); - _PyStackRef *sp = frame->stackpointer; -#ifndef NDEBUG - frame->stackpointer = NULL; -#endif - return sp; + return frame->stackpointer; } static inline void _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer) { - assert(frame->stackpointer == NULL); frame->stackpointer = stack_pointer; } +static inline void +_PyFrame_StackPointerValidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif + frame->stackpointer_valid = 1; +#endif +} + +static inline void +_PyFrame_StackPointerInvalidate(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 1); +#endif + frame->stackpointer_valid = 0; +#endif +} + +static inline void +_PyFrame_StackAssertInvalid(_PyInterpreterFrame *frame) +{ +#ifdef Py_DEBUG +/* Avoid bloating the JIT code */ +#ifndef _Py_JIT + assert(frame->stackpointer_valid == 0); +#endif +#endif +} + + /* Determine whether a frame is incomplete. * A frame is incomplete if it is part way through * creating cell objects or a generator or coroutine. @@ -282,6 +315,23 @@ _PyThreadState_GetFrame(PyThreadState *tstate) return _PyFrame_GetFirstComplete(tstate->current_frame); } +// Update last_profiled_frame for remote profiler frame caching. +// Only update if we're removing the exact frame that was last profiled. +// This avoids corrupting the cache when transient frames (called and returned +// between profiler samples) update last_profiled_frame to addresses the +// profiler never saw. +// The sequence distinguishes this anchor from a later frame that reuses the +// same _PyInterpreterFrame address. +#define _PyThreadState_UpdateLastProfiledFrame(tstate, frame, previous) \ + do { \ + PyThreadState *tstate_ = (tstate); \ + _PyInterpreterFrame *frame_ = (frame); \ + if (tstate_->last_profiled_frame == frame_) { \ + tstate_->last_profiled_frame = (previous); \ + tstate_->last_profiled_frame_seq++; \ + } \ + } while (0) + /* For use by _PyFrame_GetFrameObject Do not call directly. */ PyAPI_FUNC(PyFrameObject *) @@ -295,14 +345,15 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) { assert(!_PyFrame_IsIncomplete(frame)); - PyFrameObject *res = frame->frame_obj; + PyFrameObject *res = FT_ATOMIC_LOAD_PTR_ACQUIRE(frame->frame_obj); if (res != NULL) { return res; } return _PyFrame_MakeAndSetFrameObject(frame); } -void +// Exported for external JIT support +PyAPI_FUNC(void) _PyFrame_ClearLocals(_PyInterpreterFrame *frame); /* Clears all references in the frame. @@ -313,8 +364,10 @@ _PyFrame_ClearLocals(_PyInterpreterFrame *frame); * in the frame. * take should be set to 1 for heap allocated * frames like the ones in generators and coroutines. + * + * Exported for external JIT support */ -void + PyAPI_FUNC(void) _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int @@ -338,7 +391,8 @@ _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) size < tstate->datastack_limit - tstate->datastack_top; } -extern _PyInterpreterFrame * +// Exported for external JIT support +PyAPI_FUNC(_PyInterpreterFrame *) _PyThreadState_PushFrame(PyThreadState *tstate, size_t size); PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); @@ -388,6 +442,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->owner = FRAME_OWNED_BY_THREAD; frame->visited = 0; #ifdef Py_DEBUG + frame->stackpointer_valid = 1; frame->lltrace = 0; #endif frame->return_offset = 0; diff --git a/Include/internal/pycore_interpframe_structs.h b/Include/internal/pycore_interpframe_structs.h index 38510685f4093cd..560fb66a97fc89d 100644 --- a/Include/internal/pycore_interpframe_structs.h +++ b/Include/internal/pycore_interpframe_structs.h @@ -44,7 +44,8 @@ struct _PyInterpreterFrame { char owner; #ifdef Py_DEBUG uint8_t visited:1; - uint8_t lltrace:7; + uint8_t stackpointer_valid:1; + uint8_t lltrace:6; #else uint8_t visited; #endif @@ -65,9 +66,9 @@ struct _PyInterpreterFrame { PyObject *prefix##_qualname; \ _PyErr_StackItem prefix##_exc_state; \ PyObject *prefix##_origin_or_finalizer; \ - char prefix##_hooks_inited; \ - char prefix##_closed; \ - char prefix##_running_async; \ + int8_t prefix##_hooks_inited; \ + int8_t prefix##_closed; \ + int8_t prefix##_running_async; \ /* The frame */ \ int8_t prefix##_frame_state; \ _PyInterpreterFrame prefix##_iframe; \ diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 39c2a30f6e979de..447cea91716eca9 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -18,8 +18,9 @@ #define INTRINSIC_TYPEVARTUPLE 9 #define INTRINSIC_SUBSCRIPT_GENERIC 10 #define INTRINSIC_TYPEALIAS 11 +#define INTRINSIC_BUILD_FROZENSET 12 -#define MAX_INTRINSIC_1 11 +#define MAX_INTRINSIC_1 12 /* Binary Functions: */ @@ -29,8 +30,9 @@ #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 #define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 +#define INTRINSIC_ADD_CONDITIONAL_ANNOTATION 6 -#define MAX_INTRINSIC_2 5 +#define MAX_INTRINSIC_2 6 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); diff --git a/Include/internal/pycore_iterobject.h b/Include/internal/pycore_iterobject.h new file mode 100644 index 000000000000000..90b444976e2f19c --- /dev/null +++ b/Include/internal/pycore_iterobject.h @@ -0,0 +1,28 @@ +#ifndef Py_INTERNAL_ITEROBJECT_H +#define Py_INTERNAL_ITEROBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +extern PyTypeObject _PyACallIter_Type; +extern PyTypeObject _PyACallIterAwaitable_Type; + +// Like PyCallIter_New(), but the iteration also stops when *callable* raises +// an exception matching *stop_exc* (an exception class or a tuple of exception +// classes). *sentinel* can be NULL; NULL *stop_exc* means StopIteration. +extern PyObject *_PyCallIter_NewEx(PyObject *callable, PyObject *sentinel, + PyObject *stop_exc); + +// The asynchronous counterpart of _PyCallIter_NewEx(): the result of +// *callable* is awaited, and NULL *stop_exc* means StopAsyncIteration. +extern PyObject *_PyACallIter_New(PyObject *callable, PyObject *sentinel, + PyObject *stop_exc); + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_ITEROBJECT_H */ diff --git a/Include/internal/pycore_jit.h b/Include/internal/pycore_jit.h index 70bccce4166c18a..9ce6ec97b6090e4 100644 --- a/Include/internal/pycore_jit.h +++ b/Include/internal/pycore_jit.h @@ -23,10 +23,16 @@ typedef _Py_CODEUNIT *(*jit_func)( _PyStackRef _tos_cache0, _PyStackRef _tos_cache1, _PyStackRef _tos_cache2 ); +_Py_CODEUNIT *_PyJIT_Entry( + _PyExecutorObject *executor, _PyInterpreterFrame *frame, + _PyStackRef *stack_pointer, PyThreadState *tstate +); + int _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length); void _PyJIT_Free(_PyExecutorObject *executor); -void _PyJIT_Fini(void); PyAPI_FUNC(int) _PyJIT_AddressInJitCode(PyInterpreterState *interp, uintptr_t addr); +PyAPI_FUNC(void) _Py_jit_assert_within_stack_bounds(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer, int lineno); +PyAPI_FUNC(int) _Py_jit_assertion_failure(int line); #endif // _Py_JIT diff --git a/Include/internal/pycore_jit_publish.h b/Include/internal/pycore_jit_publish.h new file mode 100644 index 000000000000000..7ee77f5c996fafe --- /dev/null +++ b/Include/internal/pycore_jit_publish.h @@ -0,0 +1,31 @@ +#ifndef Py_INTERNAL_JIT_PUBLISH_H +#define Py_INTERNAL_JIT_PUBLISH_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef struct _PyJitCodeRegistration _PyJitCodeRegistration; + +#ifdef _Py_JIT + +/* Publish JIT code to optional tooling backends. + * + * The return value is a backend-specific deregistration handle, not a + * success/failure indicator. NULL means there is nothing to unregister later: + * perf does not need a handle, and GDB/GNU backtrace registration failures + * are intentionally non-fatal because tooling support must not make JIT + * compilation fail. + */ +_PyJitCodeRegistration *_PyJit_RegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJit_UnregisterCode(_PyJitCodeRegistration *registration); + +#endif // _Py_JIT + +#endif // Py_INTERNAL_JIT_PUBLISH_H diff --git a/Include/internal/pycore_jit_unwind.h b/Include/internal/pycore_jit_unwind.h new file mode 100644 index 000000000000000..7099b88812ce7be --- /dev/null +++ b/Include/internal/pycore_jit_unwind.h @@ -0,0 +1,81 @@ +#ifndef Py_INTERNAL_JIT_UNWIND_H +#define Py_INTERNAL_JIT_UNWIND_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include +#include + +#if defined(_Py_JIT) && defined(__linux__) && defined(__ELF__) +# define PY_HAVE_JIT_GDB_UNWIND +# if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \ + defined(_Py_HAVE_LIBGCC_EH_FRAME_REGISTRATION) +# define PY_HAVE_JIT_GNU_BACKTRACE_UNWIND +# endif +#endif + +#if defined(PY_HAVE_PERF_TRAMPOLINE) \ + || defined(PY_HAVE_JIT_GDB_UNWIND) \ + || defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) + +#if defined(PY_HAVE_JIT_GDB_UNWIND) +extern PyMutex _Py_jit_debug_mutex; +#endif + +/* DWARF exception-handling pointer encodings shared by JIT unwind users. */ +enum { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + DWRF_EH_PE_indirect = 0x80 +}; + +/* Return the size of the generated .eh_frame data for the given encoding. */ +size_t _PyJitUnwind_EhFrameSize(int absolute_addr); + +/* + * Build DWARF .eh_frame data for JIT code; returns size written or 0 on error. + * absolute_addr selects the FDE address encoding: + * - 0: PC-relative offsets (perf jitdump synthesized DSO). + * - nonzero: absolute addresses (GDB JIT in-memory ELF). + */ +size_t _PyJitUnwind_BuildEhFrame(uint8_t *buffer, size_t buffer_size, + const void *code_addr, size_t code_size, + int absolute_addr); + +void *_PyJitUnwind_GdbRegisterCode(const void *code_addr, + size_t code_size, + const char *entry, + const char *filename); + +void _PyJitUnwind_GdbUnregisterCode(void *handle); + +#if defined(PY_HAVE_JIT_GNU_BACKTRACE_UNWIND) +void *_PyJitUnwind_GnuBacktraceRegisterCode(const void *code_addr, + size_t code_size); + +void _PyJitUnwind_GnuBacktraceUnregisterCode(void *handle); +#endif + +#endif // JIT unwind support + +#endif // Py_INTERNAL_JIT_UNWIND_H diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 6b92dc5d111f3be..df0d00f752573b5 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -15,6 +15,7 @@ extern "C" { PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(PyObject) *_PyList_SliceSubscript(PyObject*, PyObject*); PyAPI_FUNC(PyObject *) _PyList_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyList_Concat(PyObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); // _PyList_GetItemRef should be used only when the object is known as a list // because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index e31d8b4e5c68c92..7dd2419ececf299 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -34,6 +34,11 @@ _PyMutex_at_fork_reinit(PyMutex *m) typedef enum _PyLockFlags { // Do not detach/release the GIL when waiting on the lock. + // + // Note that code executed while holding a mutex with this flag must + // not detach, reach a safepoint or initiate a stop-the-world pause. + // Otherwise, a non-detaching waiter may remain waiting for this mutex and + // prevent the pause from completing. _Py_LOCK_DONT_DETACH = 0, // Detach/release the GIL while waiting on the lock. @@ -207,40 +212,6 @@ PyAPI_FUNC(void) _PyRWMutex_RUnlock(_PyRWMutex *rwmutex); PyAPI_FUNC(void) _PyRWMutex_Lock(_PyRWMutex *rwmutex); PyAPI_FUNC(void) _PyRWMutex_Unlock(_PyRWMutex *rwmutex); -// Similar to linux seqlock: https://en.wikipedia.org/wiki/Seqlock -// We use a sequence number to lock the writer, an even sequence means we're unlocked, an odd -// sequence means we're locked. Readers will read the sequence before attempting to read the -// underlying data and then read the sequence number again after reading the data. If the -// sequence has not changed the data is valid. -// -// Differs a little bit in that we use CAS on sequence as the lock, instead of a separate spin lock. -// The writer can also detect that the undelering data has not changed and abandon the write -// and restore the previous sequence. -typedef struct { - uint32_t sequence; -} _PySeqLock; - -// Lock the sequence lock for the writer -PyAPI_FUNC(void) _PySeqLock_LockWrite(_PySeqLock *seqlock); - -// Unlock the sequence lock and move to the next sequence number. -PyAPI_FUNC(void) _PySeqLock_UnlockWrite(_PySeqLock *seqlock); - -// Abandon the current update indicating that no mutations have occurred -// and restore the previous sequence value. -PyAPI_FUNC(void) _PySeqLock_AbandonWrite(_PySeqLock *seqlock); - -// Begin a read operation and return the current sequence number. -PyAPI_FUNC(uint32_t) _PySeqLock_BeginRead(_PySeqLock *seqlock); - -// End the read operation and confirm that the sequence number has not changed. -// Returns 1 if the read was successful or 0 if the read should be retried. -PyAPI_FUNC(int) _PySeqLock_EndRead(_PySeqLock *seqlock, uint32_t previous); - -// Check if the lock was held during a fork and clear the lock. Returns 1 -// if the lock was held and any associated data should be cleared. -PyAPI_FUNC(int) _PySeqLock_AfterFork(_PySeqLock *seqlock); - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 9d36165c8a8ffb7..b6945f2bc5f6e0d 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -294,20 +294,25 @@ Known values: Python 3.15a4 3661 (Lazy imports IMPORT_NAME opcode changes) Python 3.15a8 3662 (Add counter to RESUME) Python 3.15a8 3663 (Merge GET_ITER and GET_YIELD_FROM_ITER. Modify SEND to make it a bit more like FOR_ITER) + Python 3.15a8 3664 (Fix __qualname__ for __annotate__ functions) + Python 3.15a8 3665 (Add FOR_ITER_VIRTUAL and GET_ITER specializations) + Python 3.15b1 3666 (Add SEND_VIRTUAL and SEND_ASYNC_GEN specializations) + Python 3.16a0 3700 (Initial version) + Python 3.16a0 3701 (Add CONSTANT_EMPTY_TUPLE to LOAD_COMMON_CONSTANT) + Python 3.16a1 3702 (Replace DELETE_NAME with PUSH_NULL; STORE_NAME) + Python 3.16a1 3703 (Replace DELETE_GLOBAL with PUSH_NULL; STORE_GLOBAL) + Python 3.16a1 3704 (Replace DELETE_ATTR with PUSH_NULL; STORE_ATTR) + Python 3.16a1 3705 (Add INTRINSIC_ADD_CONDITIONAL_ANNOTATION) - - Python 3.16 will start with 3700 + Python 3.17 will start with 3750 Please don't copy-paste the same pre-release tag for new entries above!!! You should always use the *upcoming* tag. For example, if 3.12a6 came out a week ago, I should put "Python 3.12a7" next to my new magic number. -Whenever PYC_MAGIC_NUMBER is changed, the ranges in the magic_values array in -PC/launcher.c must also be updated. - */ -#define PYC_MAGIC_NUMBER 3663 +#define PYC_MAGIC_NUMBER 3705 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_mmap.h b/Include/internal/pycore_mmap.h index 897816db01077f6..c117cbd16283da9 100644 --- a/Include/internal/pycore_mmap.h +++ b/Include/internal/pycore_mmap.h @@ -11,12 +11,12 @@ extern "C" { #include "pycore_pystate.h" -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) # include # include #endif -#if defined(HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) +#if defined(_Py_HAVE_PR_SET_VMA_ANON_NAME) && defined(__linux__) static inline int _PyAnnotateMemoryMap(void *addr, size_t size, const char *name) { diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index de701ced675cd46..41786cb267c2e96 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -266,12 +266,6 @@ _Py_REF_IS_MERGED(Py_ssize_t ob_ref_shared) return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_MERGED; } -static inline int -_Py_REF_IS_QUEUED(Py_ssize_t ob_ref_shared) -{ - return (ob_ref_shared & _Py_REF_SHARED_FLAG_MASK) == _Py_REF_QUEUED; -} - // Merge the local and shared reference count fields and add `extra` to the // refcount when merging. Py_ssize_t _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra); @@ -291,8 +285,6 @@ _PyType_HasFeature(PyTypeObject *type, unsigned long feature) { return ((type->tp_flags) & feature) != 0; } -extern void _PyType_InitCache(PyInterpreterState *interp); - extern PyStatus _PyObject_InitState(PyInterpreterState *interp); extern void _PyObject_FiniState(PyInterpreterState *interp); extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj); @@ -836,9 +828,14 @@ _PyObject_IS_GC(PyObject *obj) && (type->tp_is_gc == NULL || type->tp_is_gc(obj))); } -// Fast inlined version of PyObject_Hash() -static inline Py_hash_t -_PyObject_HashFast(PyObject *op) +// Fast inlined version of PyObject_Hash(). Dictionaries are very +// likely to include string keys (class and instance attributes, +// json, ...) so we include a fast path for strings. +// This function should not be used in a collection if str is not +// very likely, since it is slower than PyObject_Hash() on types +// other than str. See gh-137759. +static inline Py_ALWAYS_INLINE Py_hash_t +_PyObject_HashDictKey(PyObject *op) { if (PyUnicode_CheckExact(op)) { Py_hash_t hash = PyUnstable_Unicode_GET_CACHED_HASH(op); @@ -873,20 +870,23 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { return (type->tp_weaklistoffset != 0); } -extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +// Export for 'array' shared extension. +PyAPI_FUNC(PyObject*) _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); -extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); +// Exported for external JIT support +PyAPI_FUNC(int) _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value); extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr); -extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, unsigned int *); // Internal API to look for a name through the MRO. @@ -910,7 +910,9 @@ PyAPI_FUNC(_PyStackRef) _PyObject_GetAttrStackRef(PyObject *obj, PyObject *name) // deferred reference counting. // // Returns 1 if the value was cached or 0 otherwise. -extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, +// +// Exported for external JIT support +PyAPI_FUNC(int) _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, unsigned int tp_version); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 79ca8a1bfa88c0a..38bccdd48c0f862 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -178,16 +178,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1 + (oparg-1); case COPY_FREE_VARS: return 0; - case DELETE_ATTR: - return 1; case DELETE_DEREF: return 0; case DELETE_FAST: return 0; - case DELETE_GLOBAL: - return 0; - case DELETE_NAME: - return 0; case DELETE_SUBSCR: return 2; case DICT_MERGE: @@ -220,6 +214,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2; case FOR_ITER_TUPLE: return 2; + case FOR_ITER_VIRTUAL: + return 2; case GET_AITER: return 1; case GET_ANEXT: @@ -228,6 +224,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case GET_ITER: return 1; + case GET_ITER_SELF: + return 1; + case GET_ITER_VIRTUAL: + return 1; case GET_LEN: return 1; case IMPORT_FROM: @@ -432,8 +432,12 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 1; case SEND: return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: return 3; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -669,16 +673,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 2 + (oparg-1); case COPY_FREE_VARS: return 0; - case DELETE_ATTR: - return 0; case DELETE_DEREF: return 0; case DELETE_FAST: return 0; - case DELETE_GLOBAL: - return 0; - case DELETE_NAME: - return 0; case DELETE_SUBSCR: return 0; case DICT_MERGE: @@ -711,6 +709,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 3; case FOR_ITER_TUPLE: return 3; + case FOR_ITER_VIRTUAL: + return 3; case GET_AITER: return 1; case GET_ANEXT: @@ -719,6 +719,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case GET_ITER: return 2; + case GET_ITER_SELF: + return 2; + case GET_ITER_VIRTUAL: + return 2; case GET_LEN: return 2; case IMPORT_FROM: @@ -800,7 +804,7 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { case LOAD_ATTR_CLASS_WITH_METACLASS_CHECK: return 1 + (oparg & 1); case LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN: - return 1; + return 0; case LOAD_ATTR_INSTANCE_VALUE: return 1 + (oparg & 1); case LOAD_ATTR_METHOD_LAZY_DICT: @@ -923,8 +927,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case SEND: return 3; + case SEND_ASYNC_GEN: + return 3; case SEND_GEN: return 2; + case SEND_VIRTUAL: + return 3; case SETUP_ANNOTATIONS: return 0; case SETUP_CLEANUP: @@ -1092,7 +1100,7 @@ struct opcode_metadata { PyAPI_DATA(const struct opcode_metadata) _PyOpcode_opcode_metadata[267]; #ifdef NEED_OPCODE_METADATA const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { - [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP] = { true, INSTR_FMT_IBC0000, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, @@ -1100,7 +1108,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG }, - [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_SUBSCR_DICT] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [BINARY_OP_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_SUBSCR_LIST_SLICE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1123,9 +1131,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, - [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_BUILTIN_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_EX_NON_PY_GENERAL] = { true, INSTR_FMT_IXC, HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_EX_PY] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1134,13 +1142,13 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CALL_INTRINSIC_2] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ISINSTANCE] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_KW] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_BOUND_METHOD] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_KW_NON_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [CALL_KW_PY] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_LEN] = { true, INSTR_FMT_IXC00, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_LIST_APPEND] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1162,11 +1170,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [CONVERT_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [COPY_FREE_VARS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DELETE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DELETE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [DICT_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, @@ -1178,15 +1183,18 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG | HAS_RECORDS_VALUE_FLAG }, [FOR_ITER_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [FOR_ITER_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_RANGE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [FOR_ITER_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, + [FOR_ITER_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [GET_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [GET_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_SELF] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, + [GET_ITER_VIRTUAL] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, [GET_LEN] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_FROM] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [IMPORT_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1208,9 +1216,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [INSTRUMENTED_POP_JUMP_IF_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [INSTRUMENTED_POP_JUMP_IF_TRUE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, - [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG }, + [INSTRUMENTED_RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [INTERPRETER_EXIT] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, [IS_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [JUMP_BACKWARD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1221,9 +1229,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [LIST_EXTEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_ATTR_CLASS] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, - [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_ATTR_METHOD_NO_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_RECORDS_VALUE_FLAG }, @@ -1237,7 +1245,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_COMMON_CONSTANT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [LOAD_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, - [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_FAST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [LOAD_FAST_AND_CLEAR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG }, [LOAD_FAST_BORROW] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, @@ -1252,12 +1260,12 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SMALL_INT] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, - [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, + [LOAD_SPECIAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [MAKE_CELL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [MAKE_FUNCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [MATCH_CLASS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1280,16 +1288,18 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [RESUME] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [RESUME_CHECK_JIT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, - [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, - [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_GENERATOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [RETURN_VALUE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [SEND_ASYNC_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG | HAS_RECORDS_VALUE_FLAG }, + [SEND_VIRTUAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_UNPREDICTABLE_JUMP_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, + [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_DEREF] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG }, @@ -1300,7 +1310,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_RECORDS_VALUE_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_PURE_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1320,7 +1330,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1345,7 +1355,7 @@ extern const struct opcode_macro_expansion _PyOpcode_macro_expansion[256]; #ifdef NEED_OPCODE_METADATA const struct opcode_macro_expansion _PyOpcode_macro_expansion[256] = { - [BINARY_OP] = { .nuops = 3, .uops = { { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [BINARY_OP] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _BINARY_OP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [BINARY_OP_ADD_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_ADD_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 5, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_ADD_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 5 } } }, @@ -1353,7 +1363,7 @@ _PyOpcode_macro_expansion[256] = { [BINARY_OP_INPLACE_ADD_UNICODE] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_UNICODE, OPARG_SIMPLE, 0 }, { _BINARY_OP_INPLACE_ADD_UNICODE, OPARG_SIMPLE, 5 } } }, [BINARY_OP_MULTIPLY_FLOAT] = { .nuops = 5, .uops = { { _GUARD_TOS_FLOAT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_FLOAT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 }, { _POP_TOP_FLOAT, OPARG_SIMPLE, 5 } } }, [BINARY_OP_MULTIPLY_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_INT, OPARG_SIMPLE, 0 }, { _BINARY_OP_MULTIPLY_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 } } }, - [BINARY_OP_SUBSCR_DICT] = { .nuops = 4, .uops = { { _GUARD_NOS_ANY_DICT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, + [BINARY_OP_SUBSCR_DICT] = { .nuops = 5, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_DICT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_GETITEM] = { .nuops = 5, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_CHECK_FUNC, OPARG_SIMPLE, 5 }, { _BINARY_OP_SUBSCR_INIT_CALL, OPARG_SIMPLE, 5 }, { _PUSH_FRAME, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_INT, OPARG_SIMPLE, 5 }, { _POP_TOP_INT, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, [BINARY_OP_SUBSCR_LIST_SLICE] = { .nuops = 5, .uops = { { _GUARD_TOS_SLICE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _BINARY_OP_SUBSCR_LIST_SLICE, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 }, { _POP_TOP, OPARG_SIMPLE, 5 } } }, @@ -1371,25 +1381,25 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, OPARG_SIMPLE, 0 } } }, [BUILD_TEMPLATE] = { .nuops = 1, .uops = { { _BUILD_TEMPLATE, OPARG_SIMPLE, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, OPARG_SIMPLE, 0 } } }, - [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 5, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_AND_ALLOCATE_OBJECT, 2, 1 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_ALLOC_AND_ENTER_INIT] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_OBJECT, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _ALLOCATE_OBJECT, OPARG_SIMPLE, 3 }, { _CREATE_INIT_FRAME, OPARG_SIMPLE, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 11, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _CHECK_FUNCTION_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _CHECK_STACK_SPACE, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _INIT_CALL_PY_EXACT_ARGS, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_BOUND_METHOD_GENERAL] = { .nuops = 8, .uops = { { _RECORD_BOUND_METHOD, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_GENERAL, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, - [CALL_BUILTIN_CLASS] = { .nuops = 3, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_BUILTIN_O] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_CLASS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_CLASS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_BUILTIN_O] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_BUILTIN_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_BUILTIN_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_EX_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CALL_FUNCTION_EX_NON_PY_GENERAL, OPARG_SIMPLE, 1 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 1 } } }, [CALL_EX_PY] = { .nuops = 7, .uops = { { _RECORD_4OS, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _MAKE_CALLARGS_A_TUPLE, OPARG_SIMPLE, 1 }, { _CHECK_IS_PY_CALLABLE_EX, OPARG_SIMPLE, 1 }, { _PY_FRAME_EX, OPARG_SIMPLE, 1 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [CALL_INTRINSIC_1] = { .nuops = 2, .uops = { { _CALL_INTRINSIC_1, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_INTRINSIC_2] = { .nuops = 3, .uops = { { _CALL_INTRINSIC_2, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [CALL_ISINSTANCE] = { .nuops = 3, .uops = { { _GUARD_THIRD_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_ISINSTANCE, OPARG_SIMPLE, 3 }, { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, - [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_BOUND_METHOD] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY] = { .nuops = 7, .uops = { { _RECORD_CALLABLE_KW, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 5, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 6, .uops = { { _GUARD_CALLABLE_LIST_APPEND, OPARG_SIMPLE, 3 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 3 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 3 }, { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 3, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 5, .uops = { { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 6, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, OPARG_SIMPLE, 3 }, { _POP_TOP_OPARG, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 7, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_NOARGS, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 8, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _CHECK_RECURSION_LIMIT, OPARG_SIMPLE, 3 }, { _CALL_METHOD_DESCRIPTOR_O, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, [CALL_NON_PY_GENERAL] = { .nuops = 4, .uops = { { _RECORD_CALLABLE, OPARG_SIMPLE, 0 }, { _CHECK_IS_NOT_PY_CALLABLE, OPARG_SIMPLE, 3 }, { _CALL_NON_PY_GENERAL, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC_AT_END, OPARG_REPLACED, 3 } } }, @@ -1410,11 +1420,8 @@ _PyOpcode_macro_expansion[256] = { [CONVERT_VALUE] = { .nuops = 1, .uops = { { _CONVERT_VALUE, OPARG_SIMPLE, 0 } } }, [COPY] = { .nuops = 1, .uops = { { _COPY, OPARG_SIMPLE, 0 } } }, [COPY_FREE_VARS] = { .nuops = 1, .uops = { { _COPY_FREE_VARS, OPARG_SIMPLE, 0 } } }, - [DELETE_ATTR] = { .nuops = 1, .uops = { { _DELETE_ATTR, OPARG_SIMPLE, 0 } } }, [DELETE_DEREF] = { .nuops = 1, .uops = { { _DELETE_DEREF, OPARG_SIMPLE, 0 } } }, [DELETE_FAST] = { .nuops = 1, .uops = { { _DELETE_FAST, OPARG_SIMPLE, 0 } } }, - [DELETE_GLOBAL] = { .nuops = 1, .uops = { { _DELETE_GLOBAL, OPARG_SIMPLE, 0 } } }, - [DELETE_NAME] = { .nuops = 1, .uops = { { _DELETE_NAME, OPARG_SIMPLE, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { _DELETE_SUBSCR, OPARG_SIMPLE, 0 } } }, [DICT_MERGE] = { .nuops = 2, .uops = { { _DICT_MERGE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [DICT_UPDATE] = { .nuops = 2, .uops = { { _DICT_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, @@ -1423,15 +1430,18 @@ _PyOpcode_macro_expansion[256] = { [EXIT_INIT_CHECK] = { .nuops = 1, .uops = { { _EXIT_INIT_CHECK, OPARG_SIMPLE, 0 } } }, [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { _FORMAT_SIMPLE, OPARG_SIMPLE, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { _FORMAT_WITH_SPEC, OPARG_SIMPLE, 0 } } }, - [FOR_ITER] = { .nuops = 1, .uops = { { _FOR_ITER, OPARG_REPLACED, 0 } } }, + [FOR_ITER] = { .nuops = 2, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _FOR_ITER, OPARG_REPLACED, 0 } } }, [FOR_ITER_GEN] = { .nuops = 4, .uops = { { _RECORD_NOS_GEN_FUNC, OPARG_SIMPLE, 0 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _FOR_ITER_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, [FOR_ITER_LIST] = { .nuops = 3, .uops = { { _ITER_CHECK_LIST, OPARG_SIMPLE, 1 }, { _ITER_JUMP_LIST, OPARG_REPLACED, 1 }, { _ITER_NEXT_LIST, OPARG_REPLACED, 1 } } }, [FOR_ITER_RANGE] = { .nuops = 3, .uops = { { _ITER_CHECK_RANGE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_RANGE, OPARG_REPLACED, 1 }, { _ITER_NEXT_RANGE, OPARG_SIMPLE, 1 } } }, [FOR_ITER_TUPLE] = { .nuops = 3, .uops = { { _ITER_CHECK_TUPLE, OPARG_SIMPLE, 1 }, { _ITER_JUMP_TUPLE, OPARG_REPLACED, 1 }, { _ITER_NEXT_TUPLE, OPARG_SIMPLE, 1 } } }, + [FOR_ITER_VIRTUAL] = { .nuops = 2, .uops = { { _GUARD_TOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _FOR_ITER_VIRTUAL, OPARG_REPLACED, 1 } } }, [GET_AITER] = { .nuops = 1, .uops = { { _GET_AITER, OPARG_SIMPLE, 0 } } }, [GET_ANEXT] = { .nuops = 1, .uops = { { _GET_ANEXT, OPARG_SIMPLE, 0 } } }, [GET_AWAITABLE] = { .nuops = 1, .uops = { { _GET_AWAITABLE, OPARG_SIMPLE, 0 } } }, - [GET_ITER] = { .nuops = 1, .uops = { { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER] = { .nuops = 2, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GET_ITER, OPARG_SIMPLE, 0 } } }, + [GET_ITER_SELF] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITERATOR, OPARG_SIMPLE, 1 }, { _PUSH_NULL, OPARG_SIMPLE, 1 } } }, + [GET_ITER_VIRTUAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_ITER_VIRTUAL, OPARG_SIMPLE, 1 }, { _PUSH_TAGGED_ZERO, OPARG_SIMPLE, 1 } } }, [GET_LEN] = { .nuops = 1, .uops = { { _GET_LEN, OPARG_SIMPLE, 0 } } }, [IMPORT_FROM] = { .nuops = 1, .uops = { { _IMPORT_FROM, OPARG_SIMPLE, 0 } } }, [IMPORT_NAME] = { .nuops = 1, .uops = { { _IMPORT_NAME, OPARG_SIMPLE, 0 } } }, @@ -1442,16 +1452,17 @@ _PyOpcode_macro_expansion[256] = { [LIST_APPEND] = { .nuops = 1, .uops = { { _LIST_APPEND, OPARG_SIMPLE, 0 } } }, [LIST_EXTEND] = { .nuops = 2, .uops = { { _LIST_EXTEND, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [LOAD_ATTR] = { .nuops = 1, .uops = { { _LOAD_ATTR, OPARG_SIMPLE, 8 } } }, - [LOAD_ATTR_CLASS] = { .nuops = 3, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, - [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS] = { .nuops = 4, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_CLASS_WITH_METACLASS_CHECK] = { .nuops = 5, .uops = { { _RECORD_TOS, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_CLASS, 2, 3 }, { _LOAD_ATTR_CLASS, 4, 5 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, 2, 3 }, { _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_METHOD_LAZY_DICT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_METHOD_LAZY_DICT, 1, 3 }, { _LOAD_ATTR_METHOD_LAZY_DICT, 4, 5 } } }, [LOAD_ATTR_METHOD_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_METHOD_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_METHOD_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_METHOD_WITH_VALUES, 4, 5 } } }, [LOAD_ATTR_MODULE] = { .nuops = 4, .uops = { { _LOAD_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, OPERAND1_1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, 4, 5 } } }, - [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, OPARG_SIMPLE, 3 }, { _GUARD_KEYS_VERSION, 2, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, - [LOAD_ATTR_PROPERTY] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, + [LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, 4, 5 } } }, + [LOAD_ATTR_PROPERTY] = { .nuops = 7, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, 2, 3 }, { _LOAD_ATTR_PROPERTY_FRAME, OPERAND1_4, 5 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 9 }, { _PUSH_FRAME, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_SLOT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 5, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _LOAD_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 }, { _PUSH_NULL_CONDITIONAL, OPARG_SIMPLE, 9 } } }, [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { _LOAD_BUILD_CLASS, OPARG_SIMPLE, 0 } } }, @@ -1471,11 +1482,11 @@ _PyOpcode_macro_expansion[256] = { [LOAD_LOCALS] = { .nuops = 1, .uops = { { _LOAD_LOCALS, OPARG_SIMPLE, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { _LOAD_NAME, OPARG_SIMPLE, 0 } } }, [LOAD_SMALL_INT] = { .nuops = 1, .uops = { { _LOAD_SMALL_INT, OPARG_SIMPLE, 0 } } }, - [LOAD_SPECIAL] = { .nuops = 2, .uops = { { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, + [LOAD_SPECIAL] = { .nuops = 3, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 0 }, { _INSERT_NULL, OPARG_SIMPLE, 0 }, { _LOAD_SPECIAL, OPARG_SIMPLE, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { _LOAD_SUPER_ATTR_ATTR, OPARG_SIMPLE, 1 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 3, .uops = { { _RECORD_NOS, OPARG_SIMPLE, 0 }, { _GUARD_LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 }, { _LOAD_SUPER_ATTR_METHOD, OPARG_SIMPLE, 1 } } }, [MAKE_CELL] = { .nuops = 1, .uops = { { _MAKE_CELL, OPARG_SIMPLE, 0 } } }, - [MAKE_FUNCTION] = { .nuops = 1, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 } } }, + [MAKE_FUNCTION] = { .nuops = 2, .uops = { { _MAKE_FUNCTION, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MAP_ADD] = { .nuops = 1, .uops = { { _MAP_ADD, OPARG_SIMPLE, 0 } } }, [MATCH_CLASS] = { .nuops = 4, .uops = { { _MATCH_CLASS, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [MATCH_KEYS] = { .nuops = 1, .uops = { { _MATCH_KEYS, OPARG_SIMPLE, 0 } } }, @@ -1495,13 +1506,15 @@ _PyOpcode_macro_expansion[256] = { [RESUME_CHECK] = { .nuops = 1, .uops = { { _RESUME_CHECK, OPARG_SIMPLE, 1 } } }, [RETURN_GENERATOR] = { .nuops = 1, .uops = { { _RETURN_GENERATOR, OPARG_SIMPLE, 0 } } }, [RETURN_VALUE] = { .nuops = 2, .uops = { { _MAKE_HEAP_SAFE, OPARG_SIMPLE, 0 }, { _RETURN_VALUE, OPARG_SIMPLE, 0 } } }, + [SEND_ASYNC_GEN] = { .nuops = 2, .uops = { { _GUARD_3OS_ASYNC_GEN_ASEND, OPARG_SIMPLE, 1 }, { _SEND_ASYNC_GEN, OPARG_REPLACED, 1 } } }, [SEND_GEN] = { .nuops = 4, .uops = { { _RECORD_3OS_GEN_FUNC, OPARG_SIMPLE, 1 }, { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _SEND_GEN_FRAME, OPARG_SIMPLE, 1 }, { _PUSH_FRAME, OPARG_SIMPLE, 1 } } }, + [SEND_VIRTUAL] = { .nuops = 3, .uops = { { _GUARD_TOS_IS_NONE, OPARG_SIMPLE, 1 }, { _GUARD_NOS_NOT_NULL, OPARG_SIMPLE, 1 }, { _SEND_VIRTUAL, OPARG_REPLACED, 1 } } }, [SETUP_ANNOTATIONS] = { .nuops = 1, .uops = { { _SETUP_ANNOTATIONS, OPARG_SIMPLE, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { _SET_ADD, OPARG_SIMPLE, 0 } } }, [SET_FUNCTION_ATTRIBUTE] = { .nuops = 1, .uops = { { _SET_FUNCTION_ATTRIBUTE, OPARG_SIMPLE, 0 } } }, [SET_UPDATE] = { .nuops = 2, .uops = { { _SET_UPDATE, OPARG_SIMPLE, 0 }, { _POP_TOP, OPARG_SIMPLE, 0 } } }, [STORE_ATTR] = { .nuops = 1, .uops = { { _STORE_ATTR, OPARG_SIMPLE, 3 } } }, - [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 5, .uops = { { _LOCK_OBJECT, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION_LOCKED, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, + [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 6, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _LOCK_OBJECT, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION_LOCKED, 2, 1 }, { _GUARD_DORV_NO_DICT, OPARG_SIMPLE, 3 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_ATTR_SLOT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_ATTR_WITH_HINT] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_WITH_HINT, 1, 3 }, { _POP_TOP, OPARG_SIMPLE, 4 } } }, [STORE_DEREF] = { .nuops = 1, .uops = { { _STORE_DEREF, OPARG_SIMPLE, 0 } } }, @@ -1512,13 +1525,13 @@ _PyOpcode_macro_expansion[256] = { [STORE_NAME] = { .nuops = 1, .uops = { { _STORE_NAME, OPARG_SIMPLE, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { _STORE_SLICE, OPARG_SIMPLE, 0 } } }, [STORE_SUBSCR] = { .nuops = 1, .uops = { { _STORE_SUBSCR, OPARG_SIMPLE, 0 } } }, - [STORE_SUBSCR_DICT] = { .nuops = 3, .uops = { { _GUARD_NOS_DICT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, + [STORE_SUBSCR_DICT] = { .nuops = 4, .uops = { { _RECORD_NOS_TYPE, OPARG_SIMPLE, 0 }, { _GUARD_NOS_DICT_STORE_SUBSCRIPT, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_DICT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [STORE_SUBSCR_LIST_INT] = { .nuops = 5, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _GUARD_NOS_LIST, OPARG_SIMPLE, 0 }, { _STORE_SUBSCR_LIST_INT, OPARG_SIMPLE, 1 }, { _POP_TOP_INT, OPARG_SIMPLE, 1 }, { _POP_TOP, OPARG_SIMPLE, 1 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, OPARG_SIMPLE, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, OPARG_SIMPLE, 2 } } }, [TO_BOOL_ALWAYS_TRUE] = { .nuops = 4, .uops = { { _RECORD_TOS_TYPE, OPARG_SIMPLE, 1 }, { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, OPARG_SIMPLE, 3 } } }, - [TO_BOOL_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_INT, OPARG_SIMPLE, 0 }, { _TO_BOOL_INT, OPARG_SIMPLE, 3 }, { _POP_TOP_INT, OPARG_SIMPLE, 3 } } }, + [TO_BOOL_INT] = { .nuops = 3, .uops = { { _GUARD_TOS_EXACT_INT, OPARG_SIMPLE, 0 }, { _TO_BOOL_INT, OPARG_SIMPLE, 3 }, { _POP_TOP_INT, OPARG_SIMPLE, 3 } } }, [TO_BOOL_LIST] = { .nuops = 3, .uops = { { _GUARD_TOS_LIST, OPARG_SIMPLE, 0 }, { _TO_BOOL_LIST, OPARG_SIMPLE, 3 }, { _POP_TOP, OPARG_SIMPLE, 3 } } }, [TO_BOOL_NONE] = { .nuops = 1, .uops = { { _TO_BOOL_NONE, OPARG_SIMPLE, 3 } } }, [TO_BOOL_STR] = { .nuops = 3, .uops = { { _GUARD_TOS_UNICODE, OPARG_SIMPLE, 0 }, { _TO_BOOL_STR, OPARG_SIMPLE, 3 }, { _POP_TOP_UNICODE, OPARG_SIMPLE, 3 } } }, @@ -1609,11 +1622,8 @@ const char *_PyOpcode_OpName[267] = { [CONVERT_VALUE] = "CONVERT_VALUE", [COPY] = "COPY", [COPY_FREE_VARS] = "COPY_FREE_VARS", - [DELETE_ATTR] = "DELETE_ATTR", [DELETE_DEREF] = "DELETE_DEREF", [DELETE_FAST] = "DELETE_FAST", - [DELETE_GLOBAL] = "DELETE_GLOBAL", - [DELETE_NAME] = "DELETE_NAME", [DELETE_SUBSCR] = "DELETE_SUBSCR", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", @@ -1630,10 +1640,13 @@ const char *_PyOpcode_OpName[267] = { [FOR_ITER_LIST] = "FOR_ITER_LIST", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", + [FOR_ITER_VIRTUAL] = "FOR_ITER_VIRTUAL", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", [GET_AWAITABLE] = "GET_AWAITABLE", [GET_ITER] = "GET_ITER", + [GET_ITER_SELF] = "GET_ITER_SELF", + [GET_ITER_VIRTUAL] = "GET_ITER_VIRTUAL", [GET_LEN] = "GET_LEN", [IMPORT_FROM] = "IMPORT_FROM", [IMPORT_NAME] = "IMPORT_NAME", @@ -1736,7 +1749,9 @@ const char *_PyOpcode_OpName[267] = { [RETURN_GENERATOR] = "RETURN_GENERATOR", [RETURN_VALUE] = "RETURN_VALUE", [SEND] = "SEND", + [SEND_ASYNC_GEN] = "SEND_ASYNC_GEN", [SEND_GEN] = "SEND_GEN", + [SEND_VIRTUAL] = "SEND_VIRTUAL", [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_FINALLY] = "SETUP_FINALLY", @@ -1788,6 +1803,7 @@ const uint8_t _PyOpcode_Caches[256] = { [TO_BOOL] = 3, [STORE_SUBSCR] = 1, [SEND] = 1, + [FOR_ITER] = 1, [UNPACK_SEQUENCE] = 1, [STORE_ATTR] = 4, [LOAD_GLOBAL] = 4, @@ -1800,7 +1816,7 @@ const uint8_t _PyOpcode_Caches[256] = { [POP_JUMP_IF_FALSE] = 1, [POP_JUMP_IF_NONE] = 1, [POP_JUMP_IF_NOT_NONE] = 1, - [FOR_ITER] = 1, + [GET_ITER] = 1, [CALL] = 3, [CALL_KW] = 3, [CALL_FUNCTION_EX] = 1, @@ -1811,6 +1827,9 @@ const uint8_t _PyOpcode_Caches[256] = { PyAPI_DATA(const uint8_t) _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [117] = 117, + [118] = 118, + [119] = 119, [120] = 120, [121] = 121, [122] = 122, @@ -1819,11 +1838,6 @@ const uint8_t _PyOpcode_Deopt[256] = { [125] = 125, [126] = 126, [127] = 127, - [214] = 214, - [215] = 215, - [216] = 216, - [217] = 217, - [218] = 218, [219] = 219, [220] = 220, [221] = 221, @@ -1908,11 +1922,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [CONVERT_VALUE] = CONVERT_VALUE, [COPY] = COPY, [COPY_FREE_VARS] = COPY_FREE_VARS, - [DELETE_ATTR] = DELETE_ATTR, [DELETE_DEREF] = DELETE_DEREF, [DELETE_FAST] = DELETE_FAST, - [DELETE_GLOBAL] = DELETE_GLOBAL, - [DELETE_NAME] = DELETE_NAME, [DELETE_SUBSCR] = DELETE_SUBSCR, [DICT_MERGE] = DICT_MERGE, [DICT_UPDATE] = DICT_UPDATE, @@ -1929,10 +1940,13 @@ const uint8_t _PyOpcode_Deopt[256] = { [FOR_ITER_LIST] = FOR_ITER, [FOR_ITER_RANGE] = FOR_ITER, [FOR_ITER_TUPLE] = FOR_ITER, + [FOR_ITER_VIRTUAL] = FOR_ITER, [GET_AITER] = GET_AITER, [GET_ANEXT] = GET_ANEXT, [GET_AWAITABLE] = GET_AWAITABLE, [GET_ITER] = GET_ITER, + [GET_ITER_SELF] = GET_ITER, + [GET_ITER_VIRTUAL] = GET_ITER, [GET_LEN] = GET_LEN, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, @@ -2029,7 +2043,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [RETURN_GENERATOR] = RETURN_GENERATOR, [RETURN_VALUE] = RETURN_VALUE, [SEND] = SEND, + [SEND_ASYNC_GEN] = SEND, [SEND_GEN] = SEND, + [SEND_VIRTUAL] = SEND, [SETUP_ANNOTATIONS] = SETUP_ANNOTATIONS, [SET_ADD] = SET_ADD, [SET_FUNCTION_ATTRIBUTE] = SET_FUNCTION_ATTRIBUTE, @@ -2072,6 +2088,9 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_METADATA #define EXTRA_CASES \ + case 117: \ + case 118: \ + case 119: \ case 120: \ case 121: \ case 122: \ @@ -2080,11 +2099,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 125: \ case 126: \ case 127: \ - case 214: \ - case 215: \ - case 216: \ - case 217: \ - case 218: \ case 219: \ case 220: \ case 221: \ diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 69178381993fba1..7067b48ec22cb3c 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -75,16 +75,24 @@ extern "C" { #define CONSTANT_BUILTIN_ANY 4 #define CONSTANT_BUILTIN_LIST 5 #define CONSTANT_BUILTIN_SET 6 -#define NUM_COMMON_CONSTANTS 7 +#define CONSTANT_NONE 7 +#define CONSTANT_EMPTY_STR 8 +#define CONSTANT_TRUE 9 +#define CONSTANT_FALSE 10 +#define CONSTANT_MINUS_ONE 11 +#define CONSTANT_BUILTIN_FROZENSET 12 +#define CONSTANT_EMPTY_TUPLE 13 +#define NUM_COMMON_CONSTANTS 14 /* Values used in the oparg for RESUME */ #define RESUME_AT_FUNC_START 0 #define RESUME_AFTER_YIELD 1 #define RESUME_AFTER_YIELD_FROM 2 #define RESUME_AFTER_AWAIT 3 +#define RESUME_AT_GEN_EXPR_START 4 -#define RESUME_OPARG_LOCATION_MASK 0x3 -#define RESUME_OPARG_DEPTH1_MASK 0x4 +#define RESUME_OPARG_LOCATION_MASK 0x7 +#define RESUME_OPARG_DEPTH1_MASK 0x8 #define GET_ITER_YIELD_FROM 1 #define GET_ITER_YIELD_FROM_NO_CHECK 2 diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index cf01c620476ff78..9f4f8918a40d1ad 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -9,12 +9,56 @@ extern "C" { #endif #include "pycore_typedefs.h" // _PyInterpreterFrame +#include "pycore_jit_publish.h" #include "pycore_uop.h" // _PyUOpInstruction #include "pycore_uop_ids.h" #include "pycore_stackref.h" // _PyStackRef #include "pycore_optimizer_types.h" #include +/* Fitness controls how long a trace can grow. + * Starts at FITNESS_INITIAL, then decreases from per-bytecode buffer usage + * plus branch/frame heuristics. The trace stops when fitness drops below the + * current exit_quality. + * + * Design targets for the constants below: + * 1. Reaching the abstract frame-depth limit should drop fitness below + * EXIT_QUALITY_SPECIALIZABLE. + * 2. A backward edge should leave budget for roughly N_BACKWARD_SLACK more + * bytecodes, assuming AVG_SLOTS_PER_INSTRUCTION. + * 3. Roughly seven balanced branches should reduce fitness to + * EXIT_QUALITY_DEFAULT after per-slot costs. + * 4. A push followed by a matching return is net-zero on frame-specific + * fitness, excluding per-slot costs. + */ +#define OPTIMIZER_EFFECTIVENESS 2 +#define MAX_TARGET_LENGTH (FITNESS_INITIAL / OPTIMIZER_EFFECTIVENESS) + +/* Exit quality thresholds: trace stops when fitness < exit_quality. + * Higher = trace is more willing to stop here. */ +#define EXIT_QUALITY_CLOSE_LOOP (FITNESS_INITIAL - AVG_SLOTS_PER_INSTRUCTION*4) +#define EXIT_QUALITY_ENTER_EXECUTOR (FITNESS_INITIAL * 1 / 8) +#define EXIT_QUALITY_DEFAULT (FITNESS_INITIAL / 40) +#define EXIT_QUALITY_SPECIALIZABLE (FITNESS_INITIAL / 80) + +/* Estimated buffer slots per bytecode, used only to derive heuristics. + * Runtime charging uses trace-buffer capacity consumed for each bytecode. */ +#define AVG_SLOTS_PER_INSTRUCTION 6 + +/* Heuristic backward-edge exit quality: leave room for about 1 unroll and + * N_BACKWARD_SLACK more bytecodes before reaching EXIT_QUALITY_CLOSE_LOOP, + * based on AVG_SLOTS_PER_INSTRUCTION. */ +#define N_BACKWARD_SLACK 10 +#define EXIT_QUALITY_BACKWARD_EDGE (EXIT_QUALITY_CLOSE_LOOP / 2 - N_BACKWARD_SLACK * AVG_SLOTS_PER_INSTRUCTION) + +/* Penalty for a balanced branch. + * It is sized so repeated balanced branches can drive a trace toward + * EXIT_QUALITY_DEFAULT, while compute_branch_penalty() keeps any single branch + * from dominating the budget. + */ +#define FITNESS_BRANCH_BALANCED ((FITNESS_INITIAL - EXIT_QUALITY_DEFAULT - \ + (MAX_TARGET_LENGTH / 14 * AVG_SLOTS_PER_INSTRUCTION)) / (14)) + typedef struct _PyJitUopBuffer { _PyUOpInstruction *start; @@ -91,17 +135,20 @@ typedef struct _PyJitTracerInitialState { _Py_CODEUNIT *jump_backward_instr; } _PyJitTracerInitialState; +#define MAX_RECORDED_VALUES 3 typedef struct _PyJitTracerPreviousState { int instr_oparg; int instr_stacklevel; _Py_CODEUNIT *instr; PyCodeObject *instr_code; // Strong struct _PyInterpreterFrame *instr_frame; - PyObject *recorded_value; // Strong, may be NULL + PyObject *recorded_values[MAX_RECORDED_VALUES]; // Strong, may be NULL + int recorded_count; } _PyJitTracerPreviousState; typedef struct _PyJitTracerTranslatorState { - int jump_backward_seen; + int32_t fitness; // Current trace fitness, starts high, decrements + int frame_depth; // Current inline depth (0 = root frame) } _PyJitTracerTranslatorState; typedef struct _PyJitTracerState { @@ -151,6 +198,7 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; + _PyJitCodeRegistration *jit_registration; _PyExitData exits[1]; } _PyExecutorObject; @@ -158,7 +206,7 @@ typedef struct _PyExecutorObject { PyAPI_FUNC(_PyExecutorObject*) _Py_GetExecutor(PyCodeObject *code, int offset); int _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); -void _Py_ExecutorDetach(_PyExecutorObject *); +PyAPI_FUNC(void) _Py_ExecutorDetach(_PyExecutorObject *); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); /* We use a bloomfilter with k = 6, m = 256 @@ -264,7 +312,7 @@ int _Py_uop_analyze_and_optimize( _PyUOpInstruction *input, int trace_len, int curr_stackentries, _PyUOpInstruction *output, _PyBloomFilter *dependencies); -extern PyTypeObject _PyUOpExecutor_Type; +PyAPI_DATA(PyTypeObject) _PyUOpExecutor_Type; #define UOP_FORMAT_TARGET 0 @@ -400,6 +448,7 @@ extern JitOptRef _Py_uop_sym_new_null(JitOptContext *ctx); extern bool _Py_uop_sym_has_type(JitOptRef sym); extern bool _Py_uop_sym_matches_type(JitOptRef sym, PyTypeObject *typ); extern bool _Py_uop_sym_matches_type_version(JitOptRef sym, unsigned int version); +extern unsigned int _Py_uop_sym_get_type_version(JitOptRef sym); extern void _Py_uop_sym_set_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_non_null(JitOptContext *ctx, JitOptRef sym); extern void _Py_uop_sym_set_type(JitOptContext *ctx, JitOptRef sym, PyTypeObject *typ); @@ -424,8 +473,6 @@ extern PyCodeObject *_Py_uop_sym_get_probable_func_code(JitOptRef sym); extern PyObject *_Py_uop_sym_get_probable_value(JitOptRef sym); extern PyTypeObject *_Py_uop_sym_get_probable_type(JitOptRef sym); extern JitOptRef *_Py_uop_sym_set_stack_depth(JitOptContext *ctx, int stack_depth, JitOptRef *current_sp); -extern uint32_t _Py_uop_sym_get_func_version(JitOptRef ref); -bool _Py_uop_sym_set_func_version(JitOptContext *ctx, JitOptRef ref, uint32_t version); extern void _Py_uop_abstractcontext_init(JitOptContext *ctx, _PyBloomFilter *dependencies); extern void _Py_uop_abstractcontext_fini(JitOptContext *ctx); @@ -483,7 +530,26 @@ void _PyJit_TracerFree(_PyThreadStateImpl *_tstate); #ifdef _Py_TIER2 typedef void (*_Py_RecordFuncPtr)(_PyInterpreterFrame *frame, _PyStackRef *stackpointer, int oparg, PyObject **recorded_value); PyAPI_DATA(const _Py_RecordFuncPtr) _PyOpcode_RecordFunctions[]; -PyAPI_DATA(const uint8_t) _PyOpcode_RecordFunctionIndices[256]; + +typedef struct { + uint8_t count; + uint8_t indices[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordEntry; + +typedef struct { + uint8_t count; + uint8_t transform_mask; + uint8_t slots[MAX_RECORDED_VALUES]; +} _PyOpcodeRecordSlotMap; + +PyAPI_DATA(const _PyOpcodeRecordEntry) _PyOpcode_RecordEntries[256]; +PyAPI_DATA(const _PyOpcodeRecordSlotMap) _PyOpcode_RecordSlotMaps[256]; + +/* Convert a family-recorded value to the form a recorder uop expects. + * If no transform is needed, return the input value unchanged. + * Takes ownership of `value` and returns a new strong reference or NULL. + */ +PyAPI_FUNC(PyObject *) _PyOpcode_RecordTransformValue(int uop, PyObject *value); #endif #ifdef __cplusplus diff --git a/Include/internal/pycore_optimizer_types.h b/Include/internal/pycore_optimizer_types.h index 8ecfbea387460ba..a722652cc8163aa 100644 --- a/Include/internal/pycore_optimizer_types.h +++ b/Include/internal/pycore_optimizer_types.h @@ -36,7 +36,6 @@ typedef enum _JitSymType { JIT_SYM_NON_NULL_TAG = 3, JIT_SYM_BOTTOM_TAG = 4, JIT_SYM_TYPE_VERSION_TAG = 5, - JIT_SYM_FUNC_VERSION_TAG = 6, JIT_SYM_KNOWN_CLASS_TAG = 7, JIT_SYM_KNOWN_VALUE_TAG = 8, JIT_SYM_TUPLE_TAG = 9, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index c0f859a23e10b89..d8ec306a0dae3fc 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -49,6 +49,10 @@ extern "C" { _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) \ + _Py_atomic_load_uint64_acquire(&value) +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ + _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ @@ -71,6 +75,12 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_AND_UINT64(value, new_value) \ + (void)_Py_atomic_and_uint64(&value, new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) \ + (void)_Py_atomic_or_uint64(&value, new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) \ + (void)_Py_atomic_add_uint64(&value, new_value) #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ @@ -128,6 +138,7 @@ extern "C" { #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) #define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) #define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else @@ -146,6 +157,8 @@ extern "C" { #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value @@ -157,6 +170,9 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_AND_UINT64(value, new_value) (void)(value &= new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) (void)(value |= new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value @@ -186,6 +202,7 @@ extern "C" { #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) #define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) #define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 1023dbc3395b2ff..c1f9d71e40077c1 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -29,7 +29,8 @@ PyAPI_FUNC(PyObject*) _PyErr_FormatFromCause( ... ); -extern int _PyException_AddNote( +// Export for 'pyexpat' shared extension. +PyAPI_FUNC(int) _PyException_AddNote( PyObject *exc, PyObject *note); @@ -122,9 +123,11 @@ extern void _PyErr_SetNone(PyThreadState *tstate, PyObject *exception); extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); -extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset, - PyObject *module); +// Export for test_peg_generator +PyAPI_FUNC(int) _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, + int lineno, int col_offset, + int end_lineno, int end_col_offset, + PyObject *module); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); @@ -169,7 +172,8 @@ extern PyObject* _PyErr_FormatFromCauseTstate( const char *format, ...); -extern PyObject* _PyExc_CreateExceptionGroup( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyExc_CreateExceptionGroup( const char *msg, PyObject *excs); @@ -180,7 +184,8 @@ extern PyObject* _PyExc_PrepReraiseStar( extern int _PyErr_CheckSignalsTstate(PyThreadState *tstate); extern void _Py_DumpExtensionModules(int fd, PyInterpreterState *interp); -extern PyObject* _Py_CalculateSuggestions(PyObject *dir, PyObject *name); +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _Py_CalculateSuggestions(PyObject *dir, PyObject *name); extern PyObject* _Py_Offer_Suggestions(PyObject* exception); // Export for '_testinternalcapi' shared extension diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 84cb72fa6fd1b26..3056dc44cc0f1b1 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -27,14 +27,14 @@ _Py_HashPointerRaw(const void *ptr) * pppppppp ssssssss ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt + * eeeeeeee eeeeeeee eeeeeeee pyexpat XML hash salt * * memory layout on 32 bit systems * cccccccc cccccccc cccccccc uc * ppppssss ........ ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt + * eeeeeeee eeeeeeee eeee.... pyexpat XML hash salt * * (*) The siphash member may not be available on 32 bit platforms without * an unsigned int64 data type. @@ -58,7 +58,9 @@ typedef union { Py_hash_t suffix; } djbx33a; struct { - unsigned char padding[16]; + /* 16 bytes for XML_SetHashSalt16Bytes */ + uint8_t hashsalt16[16]; + /* 4/8 bytes for legacy XML_SetHashSalt */ Py_hash_t hashsalt; } expat; } _Py_HashSecret_t; diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 8faf7a4d403f846..ab627c28c1fa5ee 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -18,9 +18,6 @@ extern int _Py_SetFileSystemEncoding( const char *errors); extern void _Py_ClearFileSystemEncoding(void); extern PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate); -#ifdef MS_WINDOWS -extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void); -#endif extern int _Py_IsLocaleCoercionTarget(const char *ctype_loc); @@ -111,9 +108,6 @@ extern int _Py_LegacyLocaleDetected(int warn); // Export for 'readline' shared extension PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); -// Export for special main.c string compiling with source tracebacks -int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); - /* interpreter config */ diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 189a8dde9f09ed6..253d26fe3a3cd8b 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -266,6 +266,8 @@ extern int _PyOS_InterruptOccurred(PyThreadState *tstate); PyMutex_LockFlags(&(runtime)->interpreters.mutex, _Py_LOCK_DONT_DETACH) #define HEAD_UNLOCK(runtime) \ PyMutex_Unlock(&(runtime)->interpreters.mutex) +#define ASSERT_HEAD_IS_LOCKED(runtime) \ + assert(PyMutex_IsLocked(&(runtime)->interpreters.mutex)) #define _Py_FOR_EACH_TSTATE_UNLOCKED(interp, t) \ for (PyThreadState *t = interp->threads.head; t; t = t->next) @@ -314,15 +316,20 @@ static uintptr_t return_pointer_as_int(char* p) { static inline uintptr_t _Py_get_machine_stack_pointer(void) { -#if _Py__has_builtin(__builtin_frame_address) || defined(__GNUC__) - return (uintptr_t)__builtin_frame_address(0); -#elif defined(_MSC_VER) - return (uintptr_t)_AddressOfReturnAddress(); + uintptr_t result; +#if defined(_M_ARM64) + result = __getReg(31); +#elif defined(_M_X64) || defined(_M_IX86) + result = (uintptr_t)_AddressOfReturnAddress(); +#elif defined(__aarch64__) + __asm__ ("mov %0, sp" : "=r" (result)); +#elif defined(__x86_64__) + __asm__("{movq %%rsp, %0" : "=r" (result)); #else char here; - /* Avoid compiler warning about returning stack address */ - return return_pointer_as_int(&here); + result = (uintptr_t)&here; #endif + return result; } static inline intptr_t @@ -338,6 +345,24 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) #endif } +/* PEP 788 structures. */ + +struct PyInterpreterGuard { + PyInterpreterState *interp; +}; + +struct PyInterpreterView { + int64_t id; +}; + +// Exports for '_testinternalcapi' shared extension +PyAPI_FUNC(Py_ssize_t) _PyInterpreterState_GuardCountdown(PyInterpreterState *interp); +PyAPI_FUNC(PyInterpreterState *) _PyInterpreterGuard_GetInterpreter(PyInterpreterGuard *guard); + +extern int _PyInterpreterGuard_TryAcquire(PyInterpreterState *interp, + PyInterpreterGuard *guard); +extern void _PyInterpreterGuard_Release(PyInterpreterGuard *guard); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 2a544edc431e6b1..d333eb2ccf7c413 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -8,23 +8,18 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern int _PyRun_SimpleFileObject( +extern PyObject* _PyRun_SimpleFile( FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags); -extern int _PyRun_AnyFileObject( +extern PyObject* _PyRun_AnyFile( FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags); -extern int _PyRun_InteractiveLoopObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); - extern int _PyObject_SupportedAsScript(PyObject *); extern const char* _Py_SourceAsString( PyObject *cmd, @@ -33,12 +28,18 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); -extern PyObject * _Py_CompileStringObjectWithModule( +extern PyObject * _Py_CompileString( const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize, PyObject *module); +// Export for special main.c string compiling with source tracebacks +extern PyObject* _PyRun_SimpleString( + const char *command, + PyObject* name, + PyCompilerFlags *flags); + /* Stack size, in "pointers". This must be large enough, so * no two calls to check recursion depth are more than this far @@ -46,7 +47,8 @@ extern PyObject * _Py_CompileStringObjectWithModule( * stack consumption of PyEval_EvalDefault */ #if (defined(Py_DEBUG) \ || defined(_Py_ADDRESS_SANITIZER) \ - || defined(_Py_THREAD_SANITIZER)) + || defined(_Py_THREAD_SANITIZER)) \ + || defined(_Py_UNDEFINED_BEHAVIOR_SANITIZER) # define _PyOS_LOG2_STACK_MARGIN 12 #else # define _PyOS_LOG2_STACK_MARGIN 11 diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 7fc7f343fe600f8..fcd2ae9b1d1f1ae 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -56,6 +56,29 @@ _PyRuntimeState_SetFinalizing(_PyRuntimeState *runtime, PyThreadState *tstate) { } } +// Atomic so a thread that reads initialized=1 observes all writes +// from the initialization sequence (gh-146302). + +static inline int +_PyRuntimeState_GetCoreInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->core_initialized); +} + +static inline void +_PyRuntimeState_SetCoreInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->core_initialized, initialized); +} + +static inline int +_PyRuntimeState_GetInitialized(_PyRuntimeState *runtime) { + return _Py_atomic_load_int(&runtime->initialized); +} + +static inline void +_PyRuntimeState_SetInitialized(_PyRuntimeState *runtime, int initialized) { + _Py_atomic_store_int(&runtime->initialized, initialized); +} + #ifdef __cplusplus } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index e8d1098c2078fc9..6c48ac0dccfcb16 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -130,13 +130,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .young = { .threshold = 2000, }, \ - .old = { \ - { .threshold = 10, }, \ - { .threshold = 0, }, \ - }, \ - .work_to_do = -5000, \ - .phase = GC_PHASE_MARK, \ + GC_GENERATION_INIT \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 64b029797ab9b3e..c80925f020186ba 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -1542,6 +1542,7 @@ extern "C" { INIT_ID(_filters), \ INIT_ID(_finalizing), \ INIT_ID(_find_and_load), \ + INIT_ID(_find_and_load_lazy_submodule), \ INIT_ID(_fix_up_module), \ INIT_ID(_flags_), \ INIT_ID(_get_sourcefile), \ @@ -1580,6 +1581,7 @@ extern "C" { INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_interpreters), \ INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(alphabet), \ @@ -1592,8 +1594,10 @@ extern "C" { INIT_ID(argv), \ INIT_ID(as_integer_ratio), \ INIT_ID(asend), \ + INIT_ID(assoclen), \ INIT_ID(ast), \ INIT_ID(athrow), \ + INIT_ID(attr), \ INIT_ID(attribute), \ INIT_ID(autocommit), \ INIT_ID(backtick), \ @@ -1634,6 +1638,7 @@ extern "C" { INIT_ID(callable), \ INIT_ID(callback), \ INIT_ID(cancel), \ + INIT_ID(canonical), \ INIT_ID(capath), \ INIT_ID(capitals), \ INIT_ID(category), \ @@ -1717,6 +1722,7 @@ extern "C" { INIT_ID(dont_inherit), \ INIT_ID(dst), \ INIT_ID(dst_dir_fd), \ + INIT_ID(duration_sec), \ INIT_ID(eager_start), \ INIT_ID(effective_ids), \ INIT_ID(element_factory), \ @@ -1730,6 +1736,7 @@ extern "C" { INIT_ID(endpos), \ INIT_ID(entrypoint), \ INIT_ID(env), \ + INIT_ID(error_rate), \ INIT_ID(errors), \ INIT_ID(event), \ INIT_ID(eventmask), \ @@ -1789,7 +1796,6 @@ extern "C" { INIT_ID(generation), \ INIT_ID(get), \ INIT_ID(get_debug), \ - INIT_ID(get_event_loop), \ INIT_ID(get_loop), \ INIT_ID(get_source), \ INIT_ID(getattr), \ @@ -1806,6 +1812,7 @@ extern "C" { INIT_ID(headers), \ INIT_ID(hi), \ INIT_ID(hook), \ + INIT_ID(host), \ INIT_ID(hour), \ INIT_ID(hours), \ INIT_ID(id), \ @@ -1854,6 +1861,7 @@ extern "C" { INIT_ID(iter), \ INIT_ID(iterable), \ INIT_ID(iterations), \ + INIT_ID(iv), \ INIT_ID(join), \ INIT_ID(jump), \ INIT_ID(keepends), \ @@ -1894,6 +1902,7 @@ extern "C" { INIT_ID(mask), \ INIT_ID(match), \ INIT_ID(max_length), \ + INIT_ID(max_threads), \ INIT_ID(maxdigits), \ INIT_ID(maxevents), \ INIT_ID(maxlen), \ @@ -1912,6 +1921,7 @@ extern "C" { INIT_ID(milliseconds), \ INIT_ID(minute), \ INIT_ID(minutes), \ + INIT_ID(missed_samples), \ INIT_ID(mod), \ INIT_ID(mode), \ INIT_ID(module), \ @@ -1931,6 +1941,7 @@ extern "C" { INIT_ID(namespace_separator), \ INIT_ID(namespaces), \ INIT_ID(native), \ + INIT_ID(nbytes), \ INIT_ID(ndigits), \ INIT_ID(nested), \ INIT_ID(new_file_name), \ @@ -1955,6 +1966,7 @@ extern "C" { INIT_ID(onceregistry), \ INIT_ID(only_active_thread), \ INIT_ID(only_keys), \ + INIT_ID(op), \ INIT_ID(oparg), \ INIT_ID(opcode), \ INIT_ID(opcodes), \ @@ -1974,6 +1986,7 @@ extern "C" { INIT_ID(pad), \ INIT_ID(padded), \ INIT_ID(pages), \ + INIT_ID(pair), \ INIT_ID(parameter), \ INIT_ID(parent), \ INIT_ID(password), \ @@ -1985,8 +1998,10 @@ extern "C" { INIT_ID(person), \ INIT_ID(pi_factory), \ INIT_ID(pid), \ + INIT_ID(pidfd), \ INIT_ID(pointer_bits), \ INIT_ID(policy), \ + INIT_ID(port), \ INIT_ID(pos), \ INIT_ID(pos1), \ INIT_ID(pos2), \ @@ -2026,6 +2041,7 @@ extern "C" { INIT_ID(repeat), \ INIT_ID(repl), \ INIT_ID(replace), \ + INIT_ID(repr), \ INIT_ID(reqrefs), \ INIT_ID(require_ready), \ INIT_ID(reserved), \ @@ -2038,6 +2054,7 @@ extern "C" { INIT_ID(rounding), \ INIT_ID(salt), \ INIT_ID(sample_interval_us), \ + INIT_ID(sample_rate), \ INIT_ID(sched_priority), \ INIT_ID(scheduler), \ INIT_ID(script), \ @@ -2089,6 +2106,8 @@ extern "C" { INIT_ID(stdout), \ INIT_ID(step), \ INIT_ID(steps), \ + INIT_ID(stop_exception), \ + INIT_ID(stop_value), \ INIT_ID(store_name), \ INIT_ID(strategy), \ INIT_ID(strftime), \ @@ -2103,6 +2122,7 @@ extern "C" { INIT_ID(take_bytes), \ INIT_ID(target), \ INIT_ID(target_is_directory), \ + INIT_ID(targetfd), \ INIT_ID(task), \ INIT_ID(tb_frame), \ INIT_ID(tb_lasti), \ @@ -2145,6 +2165,7 @@ extern "C" { INIT_ID(updates), \ INIT_ID(uri), \ INIT_ID(usedforsecurity), \ + INIT_ID(utcoffset), \ INIT_ID(value), \ INIT_ID(values), \ INIT_ID(version), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 05369ef9f009e6f..145e66de9984ca7 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -158,10 +158,18 @@ struct pyruntimestate { /* Is Python preinitialized? Set to 1 by Py_PreInitialize() */ int preinitialized; - /* Is Python core initialized? Set to 1 by _Py_InitializeCore() */ + /* Is Python core initialized? Set to 1 by _Py_InitializeCore(). + + Use _PyRuntimeState_GetCoreInitialized() and + _PyRuntimeState_SetCoreInitialized() to access it, + don't access it directly. */ int core_initialized; - /* Is Python fully initialized? Set to 1 by Py_Initialize() */ + /* Is Python fully initialized? Set to 1 by Py_Initialize(). + + Use _PyRuntimeState_GetInitialized() and + _PyRuntimeState_SetInitialized() to access it, + don't access it directly. */ int initialized; /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize() diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 24d0135ed1aeca8..92d1a15177f79eb 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -35,6 +35,9 @@ extern void _PySet_ClearInternal(PySetObject *so); PyAPI_FUNC(int) _PySet_AddTakeRef(PySetObject *so, PyObject *key); +PyObject * +_PySet_Freeze(PyObject *set); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_sliceobject.h b/Include/internal/pycore_sliceobject.h index ba8b1f1cb27dee3..b6c821764886c36 100644 --- a/Include/internal/pycore_sliceobject.h +++ b/Include/internal/pycore_sliceobject.h @@ -12,7 +12,7 @@ extern "C" { /* runtime lifecycle */ PyAPI_FUNC(PyObject *) -_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop); +_PyBuildSlice_ConsumeRefs(PyObject *start, PyObject *stop, PyObject *step); #ifdef __cplusplus } diff --git a/Include/internal/pycore_slots.h b/Include/internal/pycore_slots.h new file mode 100644 index 000000000000000..960a632aea17eea --- /dev/null +++ b/Include/internal/pycore_slots.h @@ -0,0 +1,138 @@ +#ifndef _Py_PYCORE_SLOTS_H +#define _Py_PYCORE_SLOTS_H + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +/* Slot data type */ +typedef enum _PySlot_DTYPE { + _PySlot_DTYPE_VOID, + _PySlot_DTYPE_FUNC, + _PySlot_DTYPE_PTR, + _PySlot_DTYPE_SIZE, + _PySlot_DTYPE_INT64, + _PySlot_DTYPE_UINT64, +}_PySlot_DTYPE; + +/* Slot kind, used to identify: + * - the thing the slot initializes (type/module/special) + * - the struct type (PySlot/PyType_Slot/PyModuleDef_Slot) + */ +typedef enum _PySlot_KIND { + _PySlot_KIND_TYPE, + _PySlot_KIND_MOD, + _PySlot_KIND_COMPAT, + _PySlot_KIND_SLOT, +} _PySlot_KIND; + +typedef enum _PySlot_PROBLEM_HANDLING { + _PySlot_PROBLEM_ALLOW, + _PySlot_PROBLEM_DEPRECATED, + _PySlot_PROBLEM_REJECT, +} _PySlot_PROBLEM_HANDLING; + +PyAPI_DATA(const char *const) _PySlot_names[]; + +#define _PySlot_MAX_NESTING 5 + +/* State for one nesting level of a slots iterator */ +typedef struct _PySlotIterator_state { + union { + // tagged by slot_struct_kind: + const PySlot *slot; // with _PySlot_KIND_SLOT + const PyType_Slot *tp_slot; // with _PySlot_KIND_TYPE + const PyModuleDef_Slot *mod_slot; // with _PySlot_KIND_MOD + const void *any_slot; + }; + _PySlot_KIND slot_struct_kind; +} _PySlotIterator_state; + +#define _PySlot_SEEN_ENTRY_BITS (8 * sizeof(unsigned int)) + +/* State for a slots iterator */ +typedef struct { + _PySlotIterator_state *state; + _PySlotIterator_state states[_PySlot_MAX_NESTING]; + unsigned int seen[_Py_slot_COUNT / _PySlot_SEEN_ENTRY_BITS + 1]; + _PySlot_KIND kind; + uint8_t recursion_level; + bool is_at_end :1; + bool is_first_run :1; + + // Name of the object (type/module) being defined, NULL if unknown. + // Must be set by the callers as soon as it's known. + const char *name; + + /* Output information: */ + + // The slot. Always a copy; may be modified by caller of the iterator. + PySlot current; + +} _PySlotIterator; + +/* Initialize an iterator using a PySlot array */ +PyAPI_FUNC(void) +_PySlotIterator_Init(_PySlotIterator *it, const PySlot *slots, + _PySlot_KIND result_kind); + +/* Initialize an iterator using a legacy slot array */ +PyAPI_FUNC(void) +_PySlotIterator_InitLegacy(_PySlotIterator *it, const void *slots, + _PySlot_KIND kind); + +/* Reset a *successfully exhausted* iterator to the beginning. + * The *slots* must be the same as for the previous + * `_PySlotIterator_InitWithKind` call. + * (Unlike creating a new iterator, we can skip some validation after Rewind.) + */ +PyAPI_FUNC(void) _PySlotIterator_Rewind(_PySlotIterator *it, const void *slots); + +/* Iteration function. + * + * Return false at the end (when successfully exhausted). + * Otherwise (even on error), fill output information in `it` and return true. + * + * On error, set an exception and set `it->current.sl_id` to `Py_slot_invalid`. + */ +PyAPI_FUNC(bool) _PySlotIterator_Next(_PySlotIterator *it); + +/* Return 1 if given slot was "seen" by an earlier _PySlotIterator_Next call. + * (This state is not reset by rewinding.) + */ +PyAPI_FUNC(bool) _PySlotIterator_SawSlot(_PySlotIterator *, int); + +static inline const char * +_PySlot_GetName(uint16_t id) +{ + if (id >= _Py_slot_COUNT) { + return ""; + } + if (id == Py_slot_invalid) { + return "Py_slot_invalid"; + } + return _PySlot_names[id]; +} + +static inline void +_PySlot_err_bad_slot(char *kind, uint16_t id) +{ + if (id < _Py_slot_COUNT) { + PyErr_Format(PyExc_SystemError, "invalid %s slot %d (%s)", + kind, (unsigned int)id, _PySlot_names[id]); + } + else if (id == Py_slot_invalid) { + PyErr_Format(PyExc_SystemError, "invalid slot (Py_slot_invalid, %u)", + (unsigned int)id); + } + else { + PyErr_Format(PyExc_SystemError, "unknown %s slot ID %u", + kind, (unsigned int)id); + } +} + +#include "internal/pycore_slots_generated.h" + +#endif // _Py_PYCORE_SLOTS_H diff --git a/Include/internal/pycore_slots_generated.h b/Include/internal/pycore_slots_generated.h new file mode 100644 index 000000000000000..73a77070038cef4 --- /dev/null +++ b/Include/internal/pycore_slots_generated.h @@ -0,0 +1,958 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_INTERNAL_SLOTS_GENERATED_H +#define _PY_HAVE_INTERNAL_SLOTS_GENERATED_H + +static inline uint16_t +_PySlot_resolve_type_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_bf_getbuffer; + case 2: + return Py_bf_releasebuffer; + case 3: + return Py_mp_ass_subscript; + case 4: + return Py_mp_length; + case Py_slot_end: + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_doc: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_members: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + case Py_slot_subslots: + case Py_tp_slots: + case Py_tp_name: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_tp_metaclass: + case Py_tp_module: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline uint16_t +_PySlot_resolve_mod_slot(uint16_t slot_id) +{ + switch (slot_id) { + case 1: + return Py_mod_create; + case 2: + return Py_mod_exec; + case 3: + return Py_mod_multiple_interpreters; + case 4: + return Py_mod_gil; + case Py_slot_end: + case Py_mod_create: + case Py_mod_exec: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_mod_slots: + case Py_mod_name: + case Py_mod_doc: + case Py_mod_state_size: + case Py_mod_methods: + case Py_mod_state_traverse: + case Py_mod_state_clear: + case Py_mod_state_free: + case Py_mod_abi: + case Py_mod_token: + return slot_id; + default: + return Py_slot_invalid; + } +} + +static inline void* +_PySlot_type_getslot(PyTypeObject *tp, uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_subscript; + case Py_nb_absolute: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_absolute; + case Py_nb_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_add; + case Py_nb_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_and; + case Py_nb_bool: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_bool; + case Py_nb_divmod: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_divmod; + case Py_nb_float: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_float; + case Py_nb_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_floor_divide; + case Py_nb_index: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_index; + case Py_nb_inplace_add: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_add; + case Py_nb_inplace_and: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_and; + case Py_nb_inplace_floor_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_floor_divide; + case Py_nb_inplace_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_lshift; + case Py_nb_inplace_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_multiply; + case Py_nb_inplace_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_or; + case Py_nb_inplace_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_power; + case Py_nb_inplace_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_remainder; + case Py_nb_inplace_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_rshift; + case Py_nb_inplace_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_subtract; + case Py_nb_inplace_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_true_divide; + case Py_nb_inplace_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_xor; + case Py_nb_int: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_int; + case Py_nb_invert: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_invert; + case Py_nb_lshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_lshift; + case Py_nb_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_multiply; + case Py_nb_negative: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_negative; + case Py_nb_or: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_or; + case Py_nb_positive: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_positive; + case Py_nb_power: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_power; + case Py_nb_remainder: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_remainder; + case Py_nb_rshift: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_rshift; + case Py_nb_subtract: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_subtract; + case Py_nb_true_divide: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_true_divide; + case Py_nb_xor: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_xor; + case Py_sq_ass_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_ass_item; + case Py_sq_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_concat; + case Py_sq_contains: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_contains; + case Py_sq_inplace_concat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_concat; + case Py_sq_inplace_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_inplace_repeat; + case Py_sq_item: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_item; + case Py_sq_length: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_length; + case Py_sq_repeat: + if (!(tp->tp_as_sequence)) return NULL; + return (void*)tp->tp_as_sequence->sq_repeat; + case Py_tp_alloc: + return (void*)tp->tp_alloc; + case Py_tp_base: + return (void*)tp->tp_base; + case Py_tp_bases: + return (void*)tp->tp_bases; + case Py_tp_call: + return (void*)tp->tp_call; + case Py_tp_clear: + return (void*)tp->tp_clear; + case Py_tp_dealloc: + return (void*)tp->tp_dealloc; + case Py_tp_del: + return (void*)tp->tp_del; + case Py_tp_descr_get: + return (void*)tp->tp_descr_get; + case Py_tp_descr_set: + return (void*)tp->tp_descr_set; + case Py_tp_doc: + return (void*)tp->tp_doc; + case Py_tp_getattr: + return (void*)tp->tp_getattr; + case Py_tp_getattro: + return (void*)tp->tp_getattro; + case Py_tp_hash: + return (void*)tp->tp_hash; + case Py_tp_init: + return (void*)tp->tp_init; + case Py_tp_is_gc: + return (void*)tp->tp_is_gc; + case Py_tp_iter: + return (void*)tp->tp_iter; + case Py_tp_iternext: + return (void*)tp->tp_iternext; + case Py_tp_methods: + return (void*)tp->tp_methods; + case Py_tp_new: + return (void*)tp->tp_new; + case Py_tp_repr: + return (void*)tp->tp_repr; + case Py_tp_richcompare: + return (void*)tp->tp_richcompare; + case Py_tp_setattr: + return (void*)tp->tp_setattr; + case Py_tp_setattro: + return (void*)tp->tp_setattro; + case Py_tp_str: + return (void*)tp->tp_str; + case Py_tp_traverse: + return (void*)tp->tp_traverse; + case Py_tp_members: + return (void*)tp->tp_members; + case Py_tp_getset: + return (void*)tp->tp_getset; + case Py_tp_free: + return (void*)tp->tp_free; + case Py_nb_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_matrix_multiply; + case Py_nb_inplace_matrix_multiply: + if (!(tp->tp_as_number)) return NULL; + return (void*)tp->tp_as_number->nb_inplace_matrix_multiply; + case Py_am_await: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_await; + case Py_am_aiter: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_aiter; + case Py_am_anext: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_anext; + case Py_tp_finalize: + return (void*)tp->tp_finalize; + case Py_am_send: + if (!(tp->tp_as_async)) return NULL; + return (void*)tp->tp_as_async->am_send; + case Py_tp_vectorcall: + return (void*)tp->tp_vectorcall; + case Py_tp_token: + if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) return NULL; + return (void*)((PyHeapTypeObject*)tp)->ht_token; + case Py_bf_getbuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_getbuffer; + case Py_bf_releasebuffer: + if (!(tp->tp_as_buffer)) return NULL; + return (void*)tp->tp_as_buffer->bf_releasebuffer; + case Py_mp_ass_subscript: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_ass_subscript; + case Py_mp_length: + if (!(tp->tp_as_mapping)) return NULL; + return (void*)tp->tp_as_mapping->mp_length; + } + _PySlot_err_bad_slot("PyType_GetSlot", slot_id); + return NULL; +} + +static inline void +_PySlot_heaptype_apply_field_slot(PyHeapTypeObject *ht, PySlot slot) +{ + switch (slot.sl_id) { + case Py_mp_subscript: + ht->as_mapping.mp_subscript = (binaryfunc)slot.sl_func; + break; + case Py_nb_absolute: + ht->as_number.nb_absolute = (unaryfunc)slot.sl_func; + break; + case Py_nb_add: + ht->as_number.nb_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_and: + ht->as_number.nb_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_bool: + ht->as_number.nb_bool = (inquiry)slot.sl_func; + break; + case Py_nb_divmod: + ht->as_number.nb_divmod = (binaryfunc)slot.sl_func; + break; + case Py_nb_float: + ht->as_number.nb_float = (unaryfunc)slot.sl_func; + break; + case Py_nb_floor_divide: + ht->as_number.nb_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_index: + ht->as_number.nb_index = (unaryfunc)slot.sl_func; + break; + case Py_nb_inplace_add: + ht->as_number.nb_inplace_add = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_and: + ht->as_number.nb_inplace_and = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_floor_divide: + ht->as_number.nb_inplace_floor_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_lshift: + ht->as_number.nb_inplace_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_multiply: + ht->as_number.nb_inplace_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_or: + ht->as_number.nb_inplace_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_power: + ht->as_number.nb_inplace_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_inplace_remainder: + ht->as_number.nb_inplace_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_rshift: + ht->as_number.nb_inplace_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_subtract: + ht->as_number.nb_inplace_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_true_divide: + ht->as_number.nb_inplace_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_xor: + ht->as_number.nb_inplace_xor = (binaryfunc)slot.sl_func; + break; + case Py_nb_int: + ht->as_number.nb_int = (unaryfunc)slot.sl_func; + break; + case Py_nb_invert: + ht->as_number.nb_invert = (unaryfunc)slot.sl_func; + break; + case Py_nb_lshift: + ht->as_number.nb_lshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_multiply: + ht->as_number.nb_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_negative: + ht->as_number.nb_negative = (unaryfunc)slot.sl_func; + break; + case Py_nb_or: + ht->as_number.nb_or = (binaryfunc)slot.sl_func; + break; + case Py_nb_positive: + ht->as_number.nb_positive = (unaryfunc)slot.sl_func; + break; + case Py_nb_power: + ht->as_number.nb_power = (ternaryfunc)slot.sl_func; + break; + case Py_nb_remainder: + ht->as_number.nb_remainder = (binaryfunc)slot.sl_func; + break; + case Py_nb_rshift: + ht->as_number.nb_rshift = (binaryfunc)slot.sl_func; + break; + case Py_nb_subtract: + ht->as_number.nb_subtract = (binaryfunc)slot.sl_func; + break; + case Py_nb_true_divide: + ht->as_number.nb_true_divide = (binaryfunc)slot.sl_func; + break; + case Py_nb_xor: + ht->as_number.nb_xor = (binaryfunc)slot.sl_func; + break; + case Py_sq_ass_item: + ht->as_sequence.sq_ass_item = (ssizeobjargproc)slot.sl_func; + break; + case Py_sq_concat: + ht->as_sequence.sq_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_contains: + ht->as_sequence.sq_contains = (objobjproc)slot.sl_func; + break; + case Py_sq_inplace_concat: + ht->as_sequence.sq_inplace_concat = (binaryfunc)slot.sl_func; + break; + case Py_sq_inplace_repeat: + ht->as_sequence.sq_inplace_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_item: + ht->as_sequence.sq_item = (ssizeargfunc)slot.sl_func; + break; + case Py_sq_length: + ht->as_sequence.sq_length = (lenfunc)slot.sl_func; + break; + case Py_sq_repeat: + ht->as_sequence.sq_repeat = (ssizeargfunc)slot.sl_func; + break; + case Py_tp_alloc: + ht->ht_type.tp_alloc = (allocfunc)slot.sl_func; + break; + case Py_tp_base: + ht->ht_type.tp_base = slot.sl_ptr; + break; + case Py_tp_bases: + ht->ht_type.tp_bases = slot.sl_ptr; + break; + case Py_tp_call: + ht->ht_type.tp_call = (ternaryfunc)slot.sl_func; + break; + case Py_tp_clear: + ht->ht_type.tp_clear = (inquiry)slot.sl_func; + break; + case Py_tp_dealloc: + ht->ht_type.tp_dealloc = (destructor)slot.sl_func; + break; + case Py_tp_del: + ht->ht_type.tp_del = (destructor)slot.sl_func; + break; + case Py_tp_descr_get: + ht->ht_type.tp_descr_get = (descrgetfunc)slot.sl_func; + break; + case Py_tp_descr_set: + ht->ht_type.tp_descr_set = (descrsetfunc)slot.sl_func; + break; + case Py_tp_doc: + ht->ht_type.tp_doc = slot.sl_ptr; + break; + case Py_tp_getattr: + ht->ht_type.tp_getattr = (getattrfunc)slot.sl_func; + break; + case Py_tp_getattro: + ht->ht_type.tp_getattro = (getattrofunc)slot.sl_func; + break; + case Py_tp_hash: + ht->ht_type.tp_hash = (hashfunc)slot.sl_func; + break; + case Py_tp_init: + ht->ht_type.tp_init = (initproc)slot.sl_func; + break; + case Py_tp_is_gc: + ht->ht_type.tp_is_gc = (inquiry)slot.sl_func; + break; + case Py_tp_iter: + ht->ht_type.tp_iter = (getiterfunc)slot.sl_func; + break; + case Py_tp_iternext: + ht->ht_type.tp_iternext = (iternextfunc)slot.sl_func; + break; + case Py_tp_methods: + ht->ht_type.tp_methods = slot.sl_ptr; + break; + case Py_tp_new: + ht->ht_type.tp_new = (newfunc)slot.sl_func; + break; + case Py_tp_repr: + ht->ht_type.tp_repr = (reprfunc)slot.sl_func; + break; + case Py_tp_richcompare: + ht->ht_type.tp_richcompare = (richcmpfunc)slot.sl_func; + break; + case Py_tp_setattr: + ht->ht_type.tp_setattr = (setattrfunc)slot.sl_func; + break; + case Py_tp_setattro: + ht->ht_type.tp_setattro = (setattrofunc)slot.sl_func; + break; + case Py_tp_str: + ht->ht_type.tp_str = (reprfunc)slot.sl_func; + break; + case Py_tp_traverse: + ht->ht_type.tp_traverse = (traverseproc)slot.sl_func; + break; + case Py_tp_members: + ht->ht_type.tp_members = slot.sl_ptr; + break; + case Py_tp_getset: + ht->ht_type.tp_getset = slot.sl_ptr; + break; + case Py_tp_free: + ht->ht_type.tp_free = (freefunc)slot.sl_func; + break; + case Py_nb_matrix_multiply: + ht->as_number.nb_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_nb_inplace_matrix_multiply: + ht->as_number.nb_inplace_matrix_multiply = (binaryfunc)slot.sl_func; + break; + case Py_am_await: + ht->as_async.am_await = (unaryfunc)slot.sl_func; + break; + case Py_am_aiter: + ht->as_async.am_aiter = (unaryfunc)slot.sl_func; + break; + case Py_am_anext: + ht->as_async.am_anext = (unaryfunc)slot.sl_func; + break; + case Py_tp_finalize: + ht->ht_type.tp_finalize = (destructor)slot.sl_func; + break; + case Py_am_send: + ht->as_async.am_send = (sendfunc)slot.sl_func; + break; + case Py_tp_vectorcall: + ht->ht_type.tp_vectorcall = (vectorcallfunc)slot.sl_func; + break; + case Py_bf_getbuffer: + ht->as_buffer.bf_getbuffer = (getbufferproc)slot.sl_func; + break; + case Py_bf_releasebuffer: + ht->as_buffer.bf_releasebuffer = (releasebufferproc)slot.sl_func; + break; + case Py_mp_ass_subscript: + ht->as_mapping.mp_ass_subscript = (objobjargproc)slot.sl_func; + break; + case Py_mp_length: + ht->as_mapping.mp_length = (lenfunc)slot.sl_func; + break; + } +} + +static inline _PySlot_DTYPE +_PySlot_get_dtype(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: return _PySlot_DTYPE_VOID; + case Py_mp_subscript: return _PySlot_DTYPE_FUNC; + case Py_nb_absolute: return _PySlot_DTYPE_FUNC; + case Py_nb_add: return _PySlot_DTYPE_FUNC; + case Py_nb_and: return _PySlot_DTYPE_FUNC; + case Py_nb_bool: return _PySlot_DTYPE_FUNC; + case Py_nb_divmod: return _PySlot_DTYPE_FUNC; + case Py_nb_float: return _PySlot_DTYPE_FUNC; + case Py_nb_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_index: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_add: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_and: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_floor_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_or: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_power: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_xor: return _PySlot_DTYPE_FUNC; + case Py_nb_int: return _PySlot_DTYPE_FUNC; + case Py_nb_invert: return _PySlot_DTYPE_FUNC; + case Py_nb_lshift: return _PySlot_DTYPE_FUNC; + case Py_nb_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_negative: return _PySlot_DTYPE_FUNC; + case Py_nb_or: return _PySlot_DTYPE_FUNC; + case Py_nb_positive: return _PySlot_DTYPE_FUNC; + case Py_nb_power: return _PySlot_DTYPE_FUNC; + case Py_nb_remainder: return _PySlot_DTYPE_FUNC; + case Py_nb_rshift: return _PySlot_DTYPE_FUNC; + case Py_nb_subtract: return _PySlot_DTYPE_FUNC; + case Py_nb_true_divide: return _PySlot_DTYPE_FUNC; + case Py_nb_xor: return _PySlot_DTYPE_FUNC; + case Py_sq_ass_item: return _PySlot_DTYPE_FUNC; + case Py_sq_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_contains: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_concat: return _PySlot_DTYPE_FUNC; + case Py_sq_inplace_repeat: return _PySlot_DTYPE_FUNC; + case Py_sq_item: return _PySlot_DTYPE_FUNC; + case Py_sq_length: return _PySlot_DTYPE_FUNC; + case Py_sq_repeat: return _PySlot_DTYPE_FUNC; + case Py_tp_alloc: return _PySlot_DTYPE_FUNC; + case Py_tp_base: return _PySlot_DTYPE_PTR; + case Py_tp_bases: return _PySlot_DTYPE_PTR; + case Py_tp_call: return _PySlot_DTYPE_FUNC; + case Py_tp_clear: return _PySlot_DTYPE_FUNC; + case Py_tp_dealloc: return _PySlot_DTYPE_FUNC; + case Py_tp_del: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_get: return _PySlot_DTYPE_FUNC; + case Py_tp_descr_set: return _PySlot_DTYPE_FUNC; + case Py_tp_doc: return _PySlot_DTYPE_PTR; + case Py_tp_getattr: return _PySlot_DTYPE_FUNC; + case Py_tp_getattro: return _PySlot_DTYPE_FUNC; + case Py_tp_hash: return _PySlot_DTYPE_FUNC; + case Py_tp_init: return _PySlot_DTYPE_FUNC; + case Py_tp_is_gc: return _PySlot_DTYPE_FUNC; + case Py_tp_iter: return _PySlot_DTYPE_FUNC; + case Py_tp_iternext: return _PySlot_DTYPE_FUNC; + case Py_tp_methods: return _PySlot_DTYPE_PTR; + case Py_tp_new: return _PySlot_DTYPE_FUNC; + case Py_tp_repr: return _PySlot_DTYPE_FUNC; + case Py_tp_richcompare: return _PySlot_DTYPE_FUNC; + case Py_tp_setattr: return _PySlot_DTYPE_FUNC; + case Py_tp_setattro: return _PySlot_DTYPE_FUNC; + case Py_tp_str: return _PySlot_DTYPE_FUNC; + case Py_tp_traverse: return _PySlot_DTYPE_FUNC; + case Py_tp_members: return _PySlot_DTYPE_PTR; + case Py_tp_getset: return _PySlot_DTYPE_PTR; + case Py_tp_free: return _PySlot_DTYPE_FUNC; + case Py_nb_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_nb_inplace_matrix_multiply: return _PySlot_DTYPE_FUNC; + case Py_am_await: return _PySlot_DTYPE_FUNC; + case Py_am_aiter: return _PySlot_DTYPE_FUNC; + case Py_am_anext: return _PySlot_DTYPE_FUNC; + case Py_tp_finalize: return _PySlot_DTYPE_FUNC; + case Py_am_send: return _PySlot_DTYPE_FUNC; + case Py_tp_vectorcall: return _PySlot_DTYPE_FUNC; + case Py_tp_token: return _PySlot_DTYPE_PTR; + case Py_mod_create: return _PySlot_DTYPE_FUNC; + case Py_mod_exec: return _PySlot_DTYPE_FUNC; + case Py_mod_multiple_interpreters: return _PySlot_DTYPE_UINT64; + case Py_mod_gil: return _PySlot_DTYPE_UINT64; + case Py_bf_getbuffer: return _PySlot_DTYPE_FUNC; + case Py_bf_releasebuffer: return _PySlot_DTYPE_FUNC; + case Py_mp_ass_subscript: return _PySlot_DTYPE_FUNC; + case Py_mp_length: return _PySlot_DTYPE_FUNC; + case Py_slot_subslots: return _PySlot_DTYPE_PTR; + case Py_tp_slots: return _PySlot_DTYPE_PTR; + case Py_mod_slots: return _PySlot_DTYPE_PTR; + case Py_tp_name: return _PySlot_DTYPE_PTR; + case Py_tp_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_extra_basicsize: return _PySlot_DTYPE_SIZE; + case Py_tp_itemsize: return _PySlot_DTYPE_SIZE; + case Py_tp_flags: return _PySlot_DTYPE_UINT64; + case Py_mod_name: return _PySlot_DTYPE_PTR; + case Py_mod_doc: return _PySlot_DTYPE_PTR; + case Py_mod_state_size: return _PySlot_DTYPE_SIZE; + case Py_mod_methods: return _PySlot_DTYPE_PTR; + case Py_mod_state_traverse: return _PySlot_DTYPE_FUNC; + case Py_mod_state_clear: return _PySlot_DTYPE_FUNC; + case Py_mod_state_free: return _PySlot_DTYPE_FUNC; + case Py_tp_metaclass: return _PySlot_DTYPE_PTR; + case Py_tp_module: return _PySlot_DTYPE_PTR; + case Py_mod_abi: return _PySlot_DTYPE_PTR; + case Py_mod_token: return _PySlot_DTYPE_PTR; + default: return _PySlot_DTYPE_VOID; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_duplicate_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_tp_token: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + case Py_mod_exec: + case Py_mod_abi: + return _PySlot_PROBLEM_ALLOW; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline _PySlot_PROBLEM_HANDLING +_PySlot_get_null_handling(uint16_t slot_id) +{ + switch (slot_id) { + case Py_slot_end: + case Py_tp_doc: + case Py_tp_token: + case Py_mod_multiple_interpreters: + case Py_mod_gil: + case Py_slot_subslots: + case Py_tp_slots: + case Py_mod_slots: + case Py_tp_basicsize: + case Py_tp_extra_basicsize: + case Py_tp_itemsize: + case Py_tp_flags: + case Py_mod_state_size: + return _PySlot_PROBLEM_ALLOW; + case Py_mp_subscript: + case Py_nb_absolute: + case Py_nb_add: + case Py_nb_and: + case Py_nb_bool: + case Py_nb_divmod: + case Py_nb_float: + case Py_nb_floor_divide: + case Py_nb_index: + case Py_nb_inplace_add: + case Py_nb_inplace_and: + case Py_nb_inplace_floor_divide: + case Py_nb_inplace_lshift: + case Py_nb_inplace_multiply: + case Py_nb_inplace_or: + case Py_nb_inplace_power: + case Py_nb_inplace_remainder: + case Py_nb_inplace_rshift: + case Py_nb_inplace_subtract: + case Py_nb_inplace_true_divide: + case Py_nb_inplace_xor: + case Py_nb_int: + case Py_nb_invert: + case Py_nb_lshift: + case Py_nb_multiply: + case Py_nb_negative: + case Py_nb_or: + case Py_nb_positive: + case Py_nb_power: + case Py_nb_remainder: + case Py_nb_rshift: + case Py_nb_subtract: + case Py_nb_true_divide: + case Py_nb_xor: + case Py_sq_ass_item: + case Py_sq_concat: + case Py_sq_contains: + case Py_sq_inplace_concat: + case Py_sq_inplace_repeat: + case Py_sq_item: + case Py_sq_length: + case Py_sq_repeat: + case Py_tp_alloc: + case Py_tp_base: + case Py_tp_bases: + case Py_tp_call: + case Py_tp_clear: + case Py_tp_dealloc: + case Py_tp_del: + case Py_tp_descr_get: + case Py_tp_descr_set: + case Py_tp_getattr: + case Py_tp_getattro: + case Py_tp_hash: + case Py_tp_init: + case Py_tp_is_gc: + case Py_tp_iter: + case Py_tp_iternext: + case Py_tp_methods: + case Py_tp_new: + case Py_tp_repr: + case Py_tp_richcompare: + case Py_tp_setattr: + case Py_tp_setattro: + case Py_tp_str: + case Py_tp_traverse: + case Py_tp_getset: + case Py_tp_free: + case Py_nb_matrix_multiply: + case Py_nb_inplace_matrix_multiply: + case Py_am_await: + case Py_am_aiter: + case Py_am_anext: + case Py_tp_finalize: + case Py_am_send: + case Py_tp_vectorcall: + case Py_mod_create: + case Py_bf_getbuffer: + case Py_bf_releasebuffer: + case Py_mp_ass_subscript: + case Py_mp_length: + return _PySlot_PROBLEM_DEPRECATED; + default: + return _PySlot_PROBLEM_REJECT; + } +} + +static inline bool +_PySlot_get_must_be_static(uint16_t slot_id) +{ + switch (slot_id) { + case Py_tp_methods: return true; + case Py_tp_members: return true; + case Py_tp_getset: return true; + case Py_mod_methods: return true; + } + return false; +} + +#endif /* _PY_HAVE_INTERNAL_SLOTS_GENERATED_H */ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 188da775eb1cc75..9495ccc8ac38896 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -71,8 +71,10 @@ static const _PyStackRef PyStackRef_NULL = { .index = 0 }; static const _PyStackRef PyStackRef_ERROR = { .index = (1 << Py_TAGGED_SHIFT) }; #define PyStackRef_None ((_PyStackRef){ .index = (2 << Py_TAGGED_SHIFT) } ) -#define PyStackRef_False ((_PyStackRef){ .index = (3 << Py_TAGGED_SHIFT) }) -#define PyStackRef_True ((_PyStackRef){ .index = (4 << Py_TAGGED_SHIFT) }) +#define _Py_STACKREF_FALSE_INDEX (3 << Py_TAGGED_SHIFT) +#define _Py_STACKREF_TRUE_INDEX (4 << Py_TAGGED_SHIFT) +#define PyStackRef_False ((_PyStackRef){ .index = _Py_STACKREF_FALSE_INDEX }) +#define PyStackRef_True ((_PyStackRef){ .index = _Py_STACKREF_TRUE_INDEX }) #define INITIAL_STACKREF_INDEX (5 << Py_TAGGED_SHIFT) @@ -261,6 +263,18 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber) } #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__) +static inline _PyStackRef +_PyStackRef_DupImmortal(_PyStackRef ref, const char *filename, int linenumber) +{ + assert(!PyStackRef_IsError(ref)); + assert(!PyStackRef_IsTaggedInt(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + PyObject *obj = _Py_stackref_get_object(ref); + assert(_Py_IsImmortal(obj)); + return _Py_stackref_create(obj, Py_TAG_REFCNT, filename, linenumber); +} +#define PyStackRef_DupImmortal(REF) _PyStackRef_DupImmortal((REF), __FILE__, __LINE__) + static inline void _PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct, const char *filename, int linenumber) { @@ -631,6 +645,15 @@ PyStackRef_DUP(_PyStackRef ref) } #endif +static inline _PyStackRef +PyStackRef_DupImmortal(_PyStackRef ref) +{ + assert(!PyStackRef_IsNull(ref)); + assert(!PyStackRef_RefcountOnObject(ref)); + assert(_Py_IsImmortal(BITS_TO_PTR_MASKED(ref))); + return ref; +} + static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { @@ -845,6 +868,18 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) } \ } while (0) +static inline void +_PyStackRef_CloseStack(_PyStackRef *arguments, int total_args) +{ + // arguments is a pointer into the GC visible stack, + // so we must NULL out values as we clear them. + for (int i = total_args-1; i >= 0; i--) { + _PyStackRef tmp = arguments[i]; + arguments[i] = PyStackRef_NULL; + PyStackRef_CLOSE(tmp); + } +} + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index e1176c65c5ca9e6..c650a94a1eab2e1 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -90,6 +90,7 @@ typedef struct _symtable_entry { PyObject *ste_id; /* int: key in ste_table->st_blocks */ PyObject *ste_symbols; /* dict: variable names to flags */ PyObject *ste_name; /* string: name of current block */ + PyObject *ste_function_name; /* string or NULL: for annotation blocks: name of the corresponding functions */ PyObject *ste_varnames; /* list of function parameters */ PyObject *ste_children; /* list of child blocks */ PyObject *ste_directives;/* locations of global and nonlocal statements */ diff --git a/Include/internal/pycore_sysmodule.h b/Include/internal/pycore_sysmodule.h index 347b0a7a790c066..d86599cbc3a0452 100644 --- a/Include/internal/pycore_sysmodule.h +++ b/Include/internal/pycore_sysmodule.h @@ -21,6 +21,12 @@ extern int _PySys_SetIntMaxStrDigits(int maxdigits); extern int _PySysRemoteDebug_SendExec(int pid, int tid, const char *debugger_script_path); +extern void _PySys_FormatV( + PyObject *key, + FILE *fp, + const char *format, + va_list va); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index 8357cce9d899fbb..e016afaa5c56879 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -14,55 +14,6 @@ PyAPI_FUNC(int) _Py_DisplaySourceLine(PyObject *, PyObject *, int, int, int *, P // Export for 'pyexact' shared extension PyAPI_FUNC(void) _PyTraceback_Add(const char *, const char *, int); -/* Write the Python traceback into the file 'fd'. For example: - - Traceback (most recent call first): - File "xxx", line xxx in - File "xxx", line xxx in - ... - File "xxx", line xxx in - - This function is written for debug purpose only, to dump the traceback in - the worst case: after a segmentation fault, at fatal error, etc. That's why, - it is very limited. Strings are truncated to 100 characters and encoded to - ASCII with backslashreplace. It doesn't write the source code, only the - function name, filename and line number of each frame. Write only the first - 100 frames: if the traceback is truncated, write the line " ...". - - This function is signal safe. */ - -extern void _Py_DumpTraceback( - int fd, - PyThreadState *tstate); - -/* Write the traceback of all threads into the file 'fd'. current_thread can be - NULL. - - Return NULL on success, or an error message on error. - - This function is written for debug purpose only. It calls - _Py_DumpTraceback() for each thread, and so has the same limitations. It - only write the traceback of the first 100 threads: write "..." if there are - more threads. - - If current_tstate is NULL, the function tries to get the Python thread state - of the current thread. It is not an error if the function is unable to get - the current Python thread state. - - If interp is NULL, the function tries to get the interpreter state from - the current Python thread state, or from - _PyGILState_GetInterpreterStateUnsafe() in last resort. - - It is better to pass NULL to interp and current_tstate, the function tries - different options to retrieve this information. - - This function is signal safe. */ - -extern const char* _Py_DumpTracebackThreads( - int fd, - PyInterpreterState *interp, - PyThreadState *current_tstate); - /* Write a Unicode object into the file descriptor fd. Encode the string to ASCII using the backslashreplace error handler. @@ -85,7 +36,8 @@ extern void _Py_DumpHexadecimal( uintptr_t value, Py_ssize_t width); -extern PyObject* _PyTraceBack_FromFrame( +// Exported for external JIT support +PyAPI_FUNC(PyObject *) _PyTraceBack_FromFrame( PyObject *tb_next, PyFrameObject *frame); diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 9974ea3c4143fa6..0522b24cd8fc67d 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -44,9 +44,10 @@ struct __attribute__((packed)) #endif tracemalloc_frame { - /* filename cannot be NULL: "" is used if the Python frame - filename is NULL */ - PyObject *filename; + /* Interned NUL terminated UTF-8 (surrogatepass) string. + Cannot be NULL: "" is used if the Python frame filename + cannot be captured. */ + const char *filename; unsigned int lineno; }; @@ -85,7 +86,7 @@ struct _tracemalloc_runtime_state { Protected by TABLES_LOCK(). */ size_t peak_traced_memory; /* Hash table used as a set to intern filenames: - PyObject* => PyObject*. + char* (NUL terminated UTF-8 string) => NULL. Protected by the TABLES_LOCK(). */ _Py_hashtable_t *filenames; /* Buffer to store a new traceback in traceback_new(). diff --git a/Include/internal/pycore_tuple.h b/Include/internal/pycore_tuple.h index 9409ec94976d3a7..e294b16b3df60f6 100644 --- a/Include/internal/pycore_tuple.h +++ b/Include/internal/pycore_tuple.h @@ -28,6 +28,8 @@ PyAPI_FUNC(void) _PyStolenTuple_Free(PyObject *self); PyAPI_FUNC(PyObject *)_PyTuple_FromStackRefStealOnSuccess(const union _PyStackRef *, Py_ssize_t); PyAPI_FUNC(PyObject *)_PyTuple_FromArraySteal(PyObject *const *, Py_ssize_t); PyAPI_FUNC(PyObject *) _PyTuple_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Concat(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyTuple_Repeat(PyObject *self, Py_ssize_t n); PyAPI_FUNC(PyObject *) _PyTuple_FromPair(PyObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyTuple_FromPairSteal(PyObject *, PyObject *); diff --git a/Include/internal/pycore_typecache.h b/Include/internal/pycore_typecache.h new file mode 100644 index 000000000000000..76f87e8548d9c90 --- /dev/null +++ b/Include/internal/pycore_typecache.h @@ -0,0 +1,47 @@ +#ifndef PY_INTERNAL_TYPECACHE_H +#define PY_INTERNAL_TYPECACHE_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include "pycore_stackref.h" + + +#define _Py_TYPECACHE_MINSIZE (1 << 3) +#define _Py_TYPECACHE_MAXSIZE (1 << 16) + +struct type_cache_entry { + PyObject *name; // name of the attribute or method, interned string, NULL if the entry is empty + PyObject *value; // borrowed reference or NULL +}; + +// Per-type attribute lookup cache: speed up attribute and method lookups, +// see _PyTypeCache_Lookup(). +struct type_cache { + uint32_t mask; // mask for indexing into hashtable, i.e. size of hashtable is mask + 1 + uint32_t available; // number of available entries in hashtable + uint32_t used; // number of used entries in hashtable + unsigned int version_tag; // initialized from type->tp_version_tag + struct type_cache_entry hashtable[_Py_TYPECACHE_MINSIZE]; // hashtable entries +}; + +struct _PyTypeCacheLookupResult { + _PyStackRef value; // value is a stack reference to the cached attribute or method, or NULL if not found + int cache_hit; // 1 if the cache entry is valid and matches the type's version tag, 0 otherwise + unsigned int version_tag; // version tag of the type when the value was cached +}; + + +extern void _PyTypeCache_InitType(PyTypeObject *type); +extern void _PyTypeCache_Insert(PyTypeObject *type, PyObject *name, PyObject *value); +PyAPI_FUNC(struct _PyTypeCacheLookupResult) _PyTypeCache_Lookup(PyTypeObject *type, PyObject *name); +PyAPI_FUNC(void) _PyTypeCache_Invalidate(PyTypeObject *type); + +#ifdef __cplusplus +} +#endif +#endif /* PY_INTERNAL_TYPECACHE_H */ diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8207de67cf17dbb..ecb3c78a35b9165 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -41,7 +41,6 @@ extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); extern void _PyTypes_FiniTypes(PyInterpreterState *); extern void _PyTypes_FiniExtTypes(PyInterpreterState *interp); extern void _PyTypes_Fini(PyInterpreterState *); -extern void _PyTypes_AfterFork(void); extern void _PyTypes_FiniCachedDescriptors(PyInterpreterState *); static inline PyObject ** @@ -60,7 +59,8 @@ extern void _PyStaticType_FiniBuiltin( extern void _PyStaticType_ClearWeakRefs( PyInterpreterState *interp, PyTypeObject *type); -extern managed_static_type_state * _PyStaticType_GetState( +// Exported for external JIT support +PyAPI_FUNC(managed_static_type_state *) _PyStaticType_GetState( PyInterpreterState *interp, PyTypeObject *type); @@ -121,6 +121,8 @@ extern PyObject* _Py_BaseObject_RichCompare(PyObject* self, PyObject* other, int extern PyObject* _Py_slot_tp_getattro(PyObject *self, PyObject *name); extern PyObject* _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +extern int _PyType_HasSlotTpIternext(PyTypeObject *type); + extern PyTypeObject _PyBufferWrapper_Type; PyAPI_FUNC(PyObject*) _PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, @@ -145,6 +147,9 @@ extern int _PyType_AddMethod(PyTypeObject *, PyMethodDef *); extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags); +// Raise PyType_Modified with the type lock already held. +extern void _PyType_Modified_Unlocked(PyTypeObject *type); + PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); PyTypeObject *_PyType_LookupByVersion(unsigned int version); @@ -156,8 +161,9 @@ typedef int (*_py_validate_type)(PyTypeObject *); // It will verify the ``ty`` through user-defined validation function ``validate``, // and if the validation is passed, it will set the ``tp_version`` as valid // tp_version_tag from the ``ty``. -extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); -extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +// Exported for external JIT support +int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); +int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); // Precalculates count of non-unique slots and fills wrapperbase.name_count. extern int _PyType_InitSlotDefs(PyInterpreterState *interp); diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 74d84052a2bb2be..012f5da2869cd53 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -33,6 +33,7 @@ extern PyObject* _PyUnicode_ResizeCompact( Py_ssize_t length); extern PyObject* _PyUnicode_GetEmpty(void); PyAPI_FUNC(PyObject*) _PyUnicode_BinarySlice(PyObject *, PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyUnicode_Repeat(PyObject *str, Py_ssize_t len); /* Generic helper macro to convert characters of different types. @@ -181,6 +182,22 @@ extern int _PyUnicodeWriter_FormatV( const char *format, va_list vargs); +/* --- iconv Codec -------------------------------------------------------- */ + +#ifdef HAVE_ICONV +extern PyObject* _PyUnicode_DecodeIconv( + const char *encoding, /* iconv encoding name */ + const char *string, /* encoded string */ + Py_ssize_t length, /* size of string */ + const char *errors, /* error handling */ + Py_ssize_t *consumed); /* bytes consumed, or NULL for non-stateful */ + +extern PyObject* _PyUnicode_EncodeIconv( + const char *encoding, /* iconv encoding name */ + PyObject *unicode, /* Unicode object */ + const char *errors); /* error handling */ +#endif + /* --- UTF-7 Codecs ------------------------------------------------------- */ extern PyObject* _PyUnicode_EncodeUTF7( diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 461ee36dcebb6d3..b30cfc678de1cd7 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -848,6 +848,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(_find_and_load_lazy_submodule); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(_fix_up_module); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1000,6 +1004,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_interpreters); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(all_threads); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1048,6 +1056,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(assoclen); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ast); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1056,6 +1068,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(attr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(attribute); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1216,6 +1232,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(canonical); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(capath); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1548,6 +1568,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(duration_sec); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(eager_start); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1600,6 +1624,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(error_rate); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(errors); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1836,10 +1864,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(get_event_loop); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(get_loop); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1904,6 +1928,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(host); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(hour); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2096,6 +2124,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(iv); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(join); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2256,6 +2288,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(max_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(maxdigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2328,6 +2364,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(missed_samples); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(mod); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2404,6 +2444,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(nbytes); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(ndigits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2500,6 +2544,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(op); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(oparg); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2576,6 +2624,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pair); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(parameter); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2620,6 +2672,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(pidfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pointer_bits); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2628,6 +2684,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(port); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(pos); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2784,6 +2844,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(repr); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(reqrefs); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2832,6 +2896,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(sample_rate); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(sched_priority); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3036,6 +3104,14 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stop_exception); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(stop_value); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(store_name); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3092,6 +3168,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(targetfd); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(task); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -3260,6 +3340,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(utcoffset); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(value); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop.h b/Include/internal/pycore_uop.h index 7bc8947cfa9a9d0..9ff87faec40d0b4 100644 --- a/Include/internal/pycore_uop.h +++ b/Include/internal/pycore_uop.h @@ -31,17 +31,17 @@ typedef struct _PyUOpInstruction{ uint64_t operand0; // A cache entry uint64_t operand1; #ifdef Py_STATS + int32_t fitness; uint64_t execution_count; #endif } _PyUOpInstruction; -// This is the length of the trace we translate initially. -#if defined(Py_DEBUG) && defined(_Py_JIT) - // With asserts, the stencils are a lot larger -#define UOP_MAX_TRACE_LENGTH 1000 -#else -#define UOP_MAX_TRACE_LENGTH 2500 -#endif +// Fitness is the target length of the trace we translate initially. The uop +// buffer has a small amount of extra space for entry/loop-closing overhead. +#define FITNESS_INITIAL 2500 + +#define UOP_TRACE_BUFFER_OVERHEAD 10 +#define UOP_MAX_TRACE_LENGTH (FITNESS_INITIAL + UOP_TRACE_BUFFER_OVERHEAD) /* Bloom filter with m = 256 * https://en.wikipedia.org/wiki/Bloom_filter */ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 24724cb4fbe28e6..625c33a6640b853 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -9,40 +9,364 @@ extern "C" { #endif -#define _EXIT_TRACE 300 -#define _SET_IP 301 -#define _BINARY_OP 302 -#define _BINARY_OP_ADD_FLOAT 303 -#define _BINARY_OP_ADD_FLOAT_INPLACE 304 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT 305 -#define _BINARY_OP_ADD_INT 306 -#define _BINARY_OP_ADD_INT_INPLACE 307 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT 308 -#define _BINARY_OP_ADD_UNICODE 309 -#define _BINARY_OP_EXTEND 310 -#define _BINARY_OP_INPLACE_ADD_UNICODE 311 -#define _BINARY_OP_MULTIPLY_FLOAT 312 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE 313 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT 314 -#define _BINARY_OP_MULTIPLY_INT 315 -#define _BINARY_OP_MULTIPLY_INT_INPLACE 316 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT 317 -#define _BINARY_OP_SUBSCR_CHECK_FUNC 318 -#define _BINARY_OP_SUBSCR_DICT 319 -#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH 320 -#define _BINARY_OP_SUBSCR_INIT_CALL 321 -#define _BINARY_OP_SUBSCR_LIST_INT 322 -#define _BINARY_OP_SUBSCR_LIST_SLICE 323 -#define _BINARY_OP_SUBSCR_STR_INT 324 -#define _BINARY_OP_SUBSCR_TUPLE_INT 325 -#define _BINARY_OP_SUBSCR_USTR_INT 326 -#define _BINARY_OP_SUBTRACT_FLOAT 327 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE 328 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT 329 -#define _BINARY_OP_SUBTRACT_INT 330 -#define _BINARY_OP_SUBTRACT_INT_INPLACE 331 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT 332 -#define _BINARY_SLICE 333 +enum { + _EXIT_TRACE = 300, + _SET_IP, + _ALLOCATE_OBJECT, + _BINARY_OP, + _BINARY_OP_ADD_FLOAT, + _BINARY_OP_ADD_FLOAT_INPLACE, + _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT, + _BINARY_OP_ADD_INT, + _BINARY_OP_ADD_INT_INPLACE, + _BINARY_OP_ADD_INT_INPLACE_RIGHT, + _BINARY_OP_ADD_UNICODE, + _BINARY_OP_EXTEND, + _BINARY_OP_INPLACE_ADD_UNICODE, + _BINARY_OP_MULTIPLY_FLOAT, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT, + _BINARY_OP_MULTIPLY_INT, + _BINARY_OP_MULTIPLY_INT_INPLACE, + _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT, + _BINARY_OP_SUBSCR_CHECK_FUNC, + _BINARY_OP_SUBSCR_DICT, + _BINARY_OP_SUBSCR_DICT_KNOWN_HASH, + _BINARY_OP_SUBSCR_INIT_CALL, + _BINARY_OP_SUBSCR_LIST_INT, + _BINARY_OP_SUBSCR_LIST_SLICE, + _BINARY_OP_SUBSCR_STR_INT, + _BINARY_OP_SUBSCR_TUPLE_INT, + _BINARY_OP_SUBSCR_USTR_INT, + _BINARY_OP_SUBTRACT_FLOAT, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + _BINARY_OP_SUBTRACT_INT, + _BINARY_OP_SUBTRACT_INT_INPLACE, + _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT, + _BINARY_OP_TRUEDIV_FLOAT, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + _BINARY_SLICE, + _CALL_BUILTIN_CLASS, + _CALL_BUILTIN_FAST, + _CALL_BUILTIN_FAST_WITH_KEYWORDS, + _CALL_BUILTIN_O, + _CALL_FUNCTION_EX_NON_PY_GENERAL, + _CALL_INTRINSIC_1, + _CALL_INTRINSIC_2, + _CALL_ISINSTANCE, + _CALL_KW_NON_PY, + _CALL_LEN, + _CALL_LIST_APPEND, + _CALL_METHOD_DESCRIPTOR_FAST, + _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + _CALL_METHOD_DESCRIPTOR_NOARGS, + _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, + _CALL_METHOD_DESCRIPTOR_O, + _CALL_METHOD_DESCRIPTOR_O_INLINE, + _CALL_NON_PY_GENERAL, + _CALL_STR_1, + _CALL_TUPLE_1, + _CALL_TYPE_1, + _CHECK_ATTR_CLASS, + _CHECK_ATTR_METHOD_LAZY_DICT, + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, + _CHECK_FUNCTION_EXACT_ARGS, + _CHECK_FUNCTION_VERSION, + _CHECK_FUNCTION_VERSION_INLINE, + _CHECK_FUNCTION_VERSION_KW, + _CHECK_IS_NOT_PY_CALLABLE, + _CHECK_IS_NOT_PY_CALLABLE_EX, + _CHECK_IS_NOT_PY_CALLABLE_KW, + _CHECK_IS_PY_CALLABLE_EX, + _CHECK_MANAGED_OBJECT_HAS_VALUES, + _CHECK_METHOD_VERSION, + _CHECK_METHOD_VERSION_KW, + _CHECK_OBJECT, + _CHECK_PEP_523, + _CHECK_PERIODIC, + _CHECK_PERIODIC_AT_END, + _CHECK_PERIODIC_IF_NOT_YIELD_FROM, + _CHECK_RECURSION_LIMIT, + _CHECK_RECURSION_REMAINING, + _CHECK_STACK_SPACE, + _CHECK_STACK_SPACE_OPERAND, + _CHECK_VALIDITY, + _COLD_DYNAMIC_EXIT, + _COLD_EXIT, + _COMPARE_OP, + _COMPARE_OP_FLOAT, + _COMPARE_OP_INT, + _COMPARE_OP_STR, + _CONTAINS_OP, + _CONTAINS_OP_DICT, + _CONTAINS_OP_SET, + _COPY, + _COPY_1, + _COPY_2, + _COPY_3, + _CREATE_INIT_FRAME, + _DEOPT, + _DICT_MERGE, + _DICT_UPDATE, + _DO_CALL, + _DO_CALL_FUNCTION_EX, + _DO_CALL_KW, + _DYNAMIC_EXIT, + _ERROR_POP_N, + _EXPAND_METHOD, + _EXPAND_METHOD_KW, + _FATAL_ERROR, + _FOR_ITER, + _FOR_ITER_GEN_FRAME, + _FOR_ITER_TIER_TWO, + _FOR_ITER_VIRTUAL, + _FOR_ITER_VIRTUAL_TIER_TWO, + _GET_ITER, + _GET_ITER_TRAD, + _GUARD_3OS_ASYNC_GEN_ASEND, + _GUARD_BINARY_OP_EXTEND, + _GUARD_BINARY_OP_EXTEND_LHS, + _GUARD_BINARY_OP_EXTEND_RHS, + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS, + _GUARD_BIT_IS_SET_POP, + _GUARD_BIT_IS_SET_POP_4, + _GUARD_BIT_IS_SET_POP_5, + _GUARD_BIT_IS_SET_POP_6, + _GUARD_BIT_IS_SET_POP_7, + _GUARD_BIT_IS_UNSET_POP, + _GUARD_BIT_IS_UNSET_POP_4, + _GUARD_BIT_IS_UNSET_POP_5, + _GUARD_BIT_IS_UNSET_POP_6, + _GUARD_BIT_IS_UNSET_POP_7, + _GUARD_CALLABLE_BUILTIN_CLASS, + _GUARD_CALLABLE_BUILTIN_FAST, + _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, + _GUARD_CALLABLE_BUILTIN_O, + _GUARD_CALLABLE_ISINSTANCE, + _GUARD_CALLABLE_LEN, + _GUARD_CALLABLE_LIST_APPEND, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_O, + _GUARD_CALLABLE_STR_1, + _GUARD_CALLABLE_TUPLE_1, + _GUARD_CALLABLE_TYPE_1, + _GUARD_CODE_VERSION_RETURN_GENERATOR, + _GUARD_CODE_VERSION_RETURN_VALUE, + _GUARD_CODE_VERSION_YIELD_VALUE, + _GUARD_CODE_VERSION__PUSH_FRAME, + _GUARD_DORV_NO_DICT, + _GUARD_GLOBALS_VERSION, + _GUARD_IP_RETURN_GENERATOR, + _GUARD_IP_RETURN_VALUE, + _GUARD_IP_YIELD_VALUE, + _GUARD_IP__PUSH_FRAME, + _GUARD_IS_FALSE_POP, + _GUARD_IS_NONE_POP, + _GUARD_IS_NOT_NONE_POP, + _GUARD_IS_TRUE_POP, + _GUARD_ITERATOR, + _GUARD_ITER_VIRTUAL, + _GUARD_LOAD_SUPER_ATTR_METHOD, + _GUARD_NOS_COMPACT_ASCII, + _GUARD_NOS_DICT_STORE_SUBSCRIPT, + _GUARD_NOS_DICT_SUBSCRIPT, + _GUARD_NOS_FLOAT, + _GUARD_NOS_INT, + _GUARD_NOS_ITER_VIRTUAL, + _GUARD_NOS_LIST, + _GUARD_NOS_NOT_NULL, + _GUARD_NOS_NULL, + _GUARD_NOS_OVERFLOWED, + _GUARD_NOS_TUPLE, + _GUARD_NOS_TYPE_VERSION, + _GUARD_NOS_UNICODE, + _GUARD_NOT_EXHAUSTED_LIST, + _GUARD_NOT_EXHAUSTED_RANGE, + _GUARD_NOT_EXHAUSTED_TUPLE, + _GUARD_THIRD_NULL, + _GUARD_TOS_ANY_DICT, + _GUARD_TOS_ANY_SET, + _GUARD_TOS_DICT, + _GUARD_TOS_EXACT_INT, + _GUARD_TOS_FLOAT, + _GUARD_TOS_FROZENDICT, + _GUARD_TOS_FROZENSET, + _GUARD_TOS_INT, + _GUARD_TOS_IS_NONE, + _GUARD_TOS_LIST, + _GUARD_TOS_NOT_NULL, + _GUARD_TOS_OVERFLOWED, + _GUARD_TOS_SET, + _GUARD_TOS_SLICE, + _GUARD_TOS_TUPLE, + _GUARD_TOS_UNICODE, + _GUARD_TYPE, + _GUARD_TYPE_ITER, + _GUARD_TYPE_VERSION, + _GUARD_TYPE_VERSION_LOCKED, + _HANDLE_PENDING_AND_DEOPT, + _INIT_CALL_BOUND_METHOD_EXACT_ARGS, + _INIT_CALL_PY_EXACT_ARGS, + _INIT_CALL_PY_EXACT_ARGS_0, + _INIT_CALL_PY_EXACT_ARGS_1, + _INIT_CALL_PY_EXACT_ARGS_2, + _INIT_CALL_PY_EXACT_ARGS_3, + _INIT_CALL_PY_EXACT_ARGS_4, + _INSERT_NULL, + _IS_NONE, + _IS_OP, + _ITER_CHECK_LIST, + _ITER_CHECK_RANGE, + _ITER_CHECK_TUPLE, + _ITER_JUMP_LIST, + _ITER_JUMP_RANGE, + _ITER_JUMP_TUPLE, + _ITER_NEXT_INLINE, + _ITER_NEXT_LIST, + _ITER_NEXT_LIST_TIER_TWO, + _ITER_NEXT_RANGE, + _ITER_NEXT_TUPLE, + _JUMP_TO_TOP, + _LIST_EXTEND, + _LOAD_ATTR, + _LOAD_ATTR_CLASS, + _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, + _LOAD_ATTR_INSTANCE_VALUE, + _LOAD_ATTR_METHOD_LAZY_DICT, + _LOAD_ATTR_METHOD_NO_DICT, + _LOAD_ATTR_METHOD_WITH_VALUES, + _LOAD_ATTR_MODULE, + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT, + _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES, + _LOAD_ATTR_PROPERTY_FRAME, + _LOAD_ATTR_SLOT, + _LOAD_ATTR_WITH_HINT, + _LOAD_BYTECODE, + _LOAD_CONST_INLINE, + _LOAD_CONST_INLINE_BORROW, + _LOAD_FAST, + _LOAD_FAST_0, + _LOAD_FAST_1, + _LOAD_FAST_2, + _LOAD_FAST_3, + _LOAD_FAST_4, + _LOAD_FAST_5, + _LOAD_FAST_6, + _LOAD_FAST_7, + _LOAD_FAST_BORROW, + _LOAD_FAST_BORROW_0, + _LOAD_FAST_BORROW_1, + _LOAD_FAST_BORROW_2, + _LOAD_FAST_BORROW_3, + _LOAD_FAST_BORROW_4, + _LOAD_FAST_BORROW_5, + _LOAD_FAST_BORROW_6, + _LOAD_FAST_BORROW_7, + _LOAD_GLOBAL, + _LOAD_GLOBAL_BUILTINS, + _LOAD_GLOBAL_MODULE, + _LOAD_SMALL_INT, + _LOAD_SMALL_INT_0, + _LOAD_SMALL_INT_1, + _LOAD_SMALL_INT_2, + _LOAD_SMALL_INT_3, + _LOAD_SPECIAL, + _LOAD_SUPER_ATTR_METHOD, + _LOCK_OBJECT, + _MAKE_CALLARGS_A_TUPLE, + _MAKE_FUNCTION, + _MAKE_HEAP_SAFE, + _MAKE_WARM, + _MATCH_CLASS, + _MAYBE_EXPAND_METHOD, + _MAYBE_EXPAND_METHOD_KW, + _MONITOR_CALL, + _MONITOR_CALL_KW, + _MONITOR_JUMP_BACKWARD, + _MONITOR_RESUME, + _POP_JUMP_IF_FALSE, + _POP_JUMP_IF_TRUE, + _POP_TOP_FLOAT, + _POP_TOP_INT, + _POP_TOP_NOP, + _POP_TOP_OPARG, + _POP_TOP_UNICODE, + _PUSH_FRAME, + _PUSH_NULL_CONDITIONAL, + _PUSH_TAGGED_ZERO, + _PY_FRAME_EX, + _PY_FRAME_GENERAL, + _PY_FRAME_KW, + _RECORD_3OS_GEN_FUNC, + _RECORD_4OS, + _RECORD_BOUND_METHOD, + _RECORD_CALLABLE, + _RECORD_CALLABLE_KW, + _RECORD_CODE, + _RECORD_NOS, + _RECORD_NOS_GEN_FUNC, + _RECORD_NOS_TYPE, + _RECORD_TOS, + _RECORD_TOS_TYPE, + _REPLACE_WITH_TRUE, + _RESUME_CHECK, + _RETURN_VALUE, + _RROT_3, + _SAVE_RETURN_OFFSET, + _SEND_ASYNC_GEN, + _SEND_ASYNC_GEN_TIER_TWO, + _SEND_GEN_FRAME, + _SEND_VIRTUAL, + _SEND_VIRTUAL_TIER_TWO, + _SET_UPDATE, + _SPILL_OR_RELOAD, + _START_EXECUTOR, + _STORE_ATTR, + _STORE_ATTR_INSTANCE_VALUE, + _STORE_ATTR_SLOT, + _STORE_ATTR_WITH_HINT, + _STORE_SLICE, + _STORE_SUBSCR, + _STORE_SUBSCR_DICT, + _STORE_SUBSCR_DICT_KNOWN_HASH, + _STORE_SUBSCR_LIST_INT, + _SWAP, + _SWAP_2, + _SWAP_3, + _SWAP_FAST, + _SWAP_FAST_0, + _SWAP_FAST_1, + _SWAP_FAST_2, + _SWAP_FAST_3, + _SWAP_FAST_4, + _SWAP_FAST_5, + _SWAP_FAST_6, + _SWAP_FAST_7, + _TIER2_RESUME_CHECK, + _TO_BOOL, + _TO_BOOL_INT, + _TO_BOOL_LIST, + _TO_BOOL_STR, + _UNARY_INVERT, + _UNARY_NEGATIVE, + _UNARY_NEGATIVE_FLOAT_INPLACE, + _UNPACK_SEQUENCE, + _UNPACK_SEQUENCE_LIST, + _UNPACK_SEQUENCE_TUPLE, + _UNPACK_SEQUENCE_TWO_TUPLE, + _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE, + _UNPACK_SEQUENCE_UNIQUE_TUPLE, + _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, + _YIELD_VALUE, +}; +#define MAX_UOP_ID 653 + #define _BUILD_INTERPOLATION BUILD_INTERPOLATION #define _BUILD_LIST BUILD_LIST #define _BUILD_MAP BUILD_MAP @@ -51,188 +375,24 @@ extern "C" { #define _BUILD_STRING BUILD_STRING #define _BUILD_TEMPLATE BUILD_TEMPLATE #define _BUILD_TUPLE BUILD_TUPLE -#define _CALL_BUILTIN_CLASS 334 -#define _CALL_BUILTIN_FAST 335 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS 336 -#define _CALL_BUILTIN_O 337 -#define _CALL_FUNCTION_EX_NON_PY_GENERAL 338 -#define _CALL_INTRINSIC_1 339 -#define _CALL_INTRINSIC_2 340 -#define _CALL_ISINSTANCE 341 -#define _CALL_KW_NON_PY 342 -#define _CALL_LEN 343 -#define _CALL_LIST_APPEND 344 -#define _CALL_METHOD_DESCRIPTOR_FAST 345 -#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE 346 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 347 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE 348 -#define _CALL_METHOD_DESCRIPTOR_NOARGS 349 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE 350 -#define _CALL_METHOD_DESCRIPTOR_O 351 -#define _CALL_METHOD_DESCRIPTOR_O_INLINE 352 -#define _CALL_NON_PY_GENERAL 353 -#define _CALL_STR_1 354 -#define _CALL_TUPLE_1 355 -#define _CALL_TYPE_1 356 -#define _CHECK_AND_ALLOCATE_OBJECT 357 -#define _CHECK_ATTR_CLASS 358 -#define _CHECK_ATTR_METHOD_LAZY_DICT 359 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 360 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION_EXACT_ARGS 361 -#define _CHECK_FUNCTION_VERSION 362 -#define _CHECK_FUNCTION_VERSION_INLINE 363 -#define _CHECK_FUNCTION_VERSION_KW 364 -#define _CHECK_IS_NOT_PY_CALLABLE 365 -#define _CHECK_IS_NOT_PY_CALLABLE_EX 366 -#define _CHECK_IS_NOT_PY_CALLABLE_KW 367 -#define _CHECK_IS_PY_CALLABLE_EX 368 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 369 -#define _CHECK_METHOD_VERSION 370 -#define _CHECK_METHOD_VERSION_KW 371 -#define _CHECK_PEP_523 372 -#define _CHECK_PERIODIC 373 -#define _CHECK_PERIODIC_AT_END 374 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM 375 -#define _CHECK_RECURSION_LIMIT 376 -#define _CHECK_RECURSION_REMAINING 377 -#define _CHECK_STACK_SPACE 378 -#define _CHECK_STACK_SPACE_OPERAND 379 -#define _CHECK_VALIDITY 380 -#define _COLD_DYNAMIC_EXIT 381 -#define _COLD_EXIT 382 -#define _COMPARE_OP 383 -#define _COMPARE_OP_FLOAT 384 -#define _COMPARE_OP_INT 385 -#define _COMPARE_OP_STR 386 -#define _CONTAINS_OP 387 -#define _CONTAINS_OP_DICT 388 -#define _CONTAINS_OP_SET 389 #define _CONVERT_VALUE CONVERT_VALUE -#define _COPY 390 -#define _COPY_1 391 -#define _COPY_2 392 -#define _COPY_3 393 #define _COPY_FREE_VARS COPY_FREE_VARS -#define _CREATE_INIT_FRAME 394 -#define _DELETE_ATTR DELETE_ATTR #define _DELETE_DEREF DELETE_DEREF #define _DELETE_FAST DELETE_FAST -#define _DELETE_GLOBAL DELETE_GLOBAL -#define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 395 -#define _DICT_MERGE 396 -#define _DICT_UPDATE 397 -#define _DO_CALL 398 -#define _DO_CALL_FUNCTION_EX 399 -#define _DO_CALL_KW 400 -#define _DYNAMIC_EXIT 401 #define _END_FOR END_FOR #define _END_SEND END_SEND -#define _ERROR_POP_N 402 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _EXPAND_METHOD 403 -#define _EXPAND_METHOD_KW 404 -#define _FATAL_ERROR 405 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 406 -#define _FOR_ITER_GEN_FRAME 407 -#define _FOR_ITER_TIER_TWO 408 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE -#define _GET_ITER GET_ITER #define _GET_LEN GET_LEN -#define _GUARD_BINARY_OP_EXTEND 409 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS 410 -#define _GUARD_BIT_IS_SET_POP 411 -#define _GUARD_BIT_IS_SET_POP_4 412 -#define _GUARD_BIT_IS_SET_POP_5 413 -#define _GUARD_BIT_IS_SET_POP_6 414 -#define _GUARD_BIT_IS_SET_POP_7 415 -#define _GUARD_BIT_IS_UNSET_POP 416 -#define _GUARD_BIT_IS_UNSET_POP_4 417 -#define _GUARD_BIT_IS_UNSET_POP_5 418 -#define _GUARD_BIT_IS_UNSET_POP_6 419 -#define _GUARD_BIT_IS_UNSET_POP_7 420 -#define _GUARD_CALLABLE_BUILTIN_FAST 421 -#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS 422 -#define _GUARD_CALLABLE_BUILTIN_O 423 -#define _GUARD_CALLABLE_ISINSTANCE 424 -#define _GUARD_CALLABLE_LEN 425 -#define _GUARD_CALLABLE_LIST_APPEND 426 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST 427 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 428 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS 429 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O 430 -#define _GUARD_CALLABLE_STR_1 431 -#define _GUARD_CALLABLE_TUPLE_1 432 -#define _GUARD_CALLABLE_TYPE_1 433 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR 434 -#define _GUARD_CODE_VERSION_RETURN_VALUE 435 -#define _GUARD_CODE_VERSION_YIELD_VALUE 436 -#define _GUARD_CODE_VERSION__PUSH_FRAME 437 -#define _GUARD_DORV_NO_DICT 438 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 439 -#define _GUARD_GLOBALS_VERSION 440 -#define _GUARD_IP_RETURN_GENERATOR 441 -#define _GUARD_IP_RETURN_VALUE 442 -#define _GUARD_IP_YIELD_VALUE 443 -#define _GUARD_IP__PUSH_FRAME 444 -#define _GUARD_IS_FALSE_POP 445 -#define _GUARD_IS_NONE_POP 446 -#define _GUARD_IS_NOT_NONE_POP 447 -#define _GUARD_IS_TRUE_POP 448 -#define _GUARD_KEYS_VERSION 449 -#define _GUARD_LOAD_SUPER_ATTR_METHOD 450 -#define _GUARD_NOS_ANY_DICT 451 -#define _GUARD_NOS_COMPACT_ASCII 452 -#define _GUARD_NOS_DICT 453 -#define _GUARD_NOS_FLOAT 454 -#define _GUARD_NOS_INT 455 -#define _GUARD_NOS_LIST 456 -#define _GUARD_NOS_NOT_NULL 457 -#define _GUARD_NOS_NULL 458 -#define _GUARD_NOS_OVERFLOWED 459 -#define _GUARD_NOS_TUPLE 460 -#define _GUARD_NOS_TYPE_VERSION 461 -#define _GUARD_NOS_UNICODE 462 -#define _GUARD_NOT_EXHAUSTED_LIST 463 -#define _GUARD_NOT_EXHAUSTED_RANGE 464 -#define _GUARD_NOT_EXHAUSTED_TUPLE 465 -#define _GUARD_THIRD_NULL 466 -#define _GUARD_TOS_ANY_DICT 467 -#define _GUARD_TOS_ANY_SET 468 -#define _GUARD_TOS_DICT 469 -#define _GUARD_TOS_FLOAT 470 -#define _GUARD_TOS_FROZENDICT 471 -#define _GUARD_TOS_FROZENSET 472 -#define _GUARD_TOS_INT 473 -#define _GUARD_TOS_LIST 474 -#define _GUARD_TOS_OVERFLOWED 475 -#define _GUARD_TOS_SET 476 -#define _GUARD_TOS_SLICE 477 -#define _GUARD_TOS_TUPLE 478 -#define _GUARD_TOS_UNICODE 479 -#define _GUARD_TYPE_VERSION 480 -#define _GUARD_TYPE_VERSION_LOCKED 481 -#define _HANDLE_PENDING_AND_DEOPT 482 #define _IMPORT_FROM IMPORT_FROM #define _IMPORT_NAME IMPORT_NAME -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 483 -#define _INIT_CALL_PY_EXACT_ARGS 484 -#define _INIT_CALL_PY_EXACT_ARGS_0 485 -#define _INIT_CALL_PY_EXACT_ARGS_1 486 -#define _INIT_CALL_PY_EXACT_ARGS_2 487 -#define _INIT_CALL_PY_EXACT_ARGS_3 488 -#define _INIT_CALL_PY_EXACT_ARGS_4 489 -#define _INSERT_1_LOAD_CONST_INLINE 490 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW 491 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW 492 -#define _INSERT_NULL 493 #define _INSTRUMENTED_FOR_ITER INSTRUMENTED_FOR_ITER #define _INSTRUMENTED_INSTRUCTION INSTRUMENTED_INSTRUCTION #define _INSTRUMENTED_JUMP_FORWARD INSTRUMENTED_JUMP_FORWARD @@ -242,1154 +402,1044 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _IS_NONE 494 -#define _IS_OP 495 -#define _ITER_CHECK_LIST 496 -#define _ITER_CHECK_RANGE 497 -#define _ITER_CHECK_TUPLE 498 -#define _ITER_JUMP_LIST 499 -#define _ITER_JUMP_RANGE 500 -#define _ITER_JUMP_TUPLE 501 -#define _ITER_NEXT_LIST 502 -#define _ITER_NEXT_LIST_TIER_TWO 503 -#define _ITER_NEXT_RANGE 504 -#define _ITER_NEXT_TUPLE 505 #define _JUMP_BACKWARD_NO_INTERRUPT JUMP_BACKWARD_NO_INTERRUPT -#define _JUMP_TO_TOP 506 #define _LIST_APPEND LIST_APPEND -#define _LIST_EXTEND 507 -#define _LOAD_ATTR 508 -#define _LOAD_ATTR_CLASS 509 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 510 -#define _LOAD_ATTR_METHOD_LAZY_DICT 511 -#define _LOAD_ATTR_METHOD_NO_DICT 512 -#define _LOAD_ATTR_METHOD_WITH_VALUES 513 -#define _LOAD_ATTR_MODULE 514 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 515 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 516 -#define _LOAD_ATTR_PROPERTY_FRAME 517 -#define _LOAD_ATTR_SLOT 518 -#define _LOAD_ATTR_WITH_HINT 519 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 520 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 521 -#define _LOAD_CONST_INLINE_BORROW 522 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 523 -#define _LOAD_FAST_0 524 -#define _LOAD_FAST_1 525 -#define _LOAD_FAST_2 526 -#define _LOAD_FAST_3 527 -#define _LOAD_FAST_4 528 -#define _LOAD_FAST_5 529 -#define _LOAD_FAST_6 530 -#define _LOAD_FAST_7 531 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR -#define _LOAD_FAST_BORROW 532 -#define _LOAD_FAST_BORROW_0 533 -#define _LOAD_FAST_BORROW_1 534 -#define _LOAD_FAST_BORROW_2 535 -#define _LOAD_FAST_BORROW_3 536 -#define _LOAD_FAST_BORROW_4 537 -#define _LOAD_FAST_BORROW_5 538 -#define _LOAD_FAST_BORROW_6 539 -#define _LOAD_FAST_BORROW_7 540 #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 541 -#define _LOAD_GLOBAL_BUILTINS 542 -#define _LOAD_GLOBAL_MODULE 543 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 544 -#define _LOAD_SMALL_INT_0 545 -#define _LOAD_SMALL_INT_1 546 -#define _LOAD_SMALL_INT_2 547 -#define _LOAD_SMALL_INT_3 548 -#define _LOAD_SPECIAL 549 #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR -#define _LOAD_SUPER_ATTR_METHOD 550 -#define _LOCK_OBJECT 551 -#define _MAKE_CALLARGS_A_TUPLE 552 #define _MAKE_CELL MAKE_CELL -#define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_HEAP_SAFE 553 -#define _MAKE_WARM 554 #define _MAP_ADD MAP_ADD -#define _MATCH_CLASS 555 #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 556 -#define _MAYBE_EXPAND_METHOD_KW 557 -#define _MONITOR_CALL 558 -#define _MONITOR_CALL_KW 559 -#define _MONITOR_JUMP_BACKWARD 560 -#define _MONITOR_RESUME 561 #define _NOP NOP -#define _POP_CALL 562 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW 563 -#define _POP_CALL_ONE 564 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW 565 -#define _POP_CALL_TWO 566 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW 567 #define _POP_EXCEPT POP_EXCEPT #define _POP_ITER POP_ITER -#define _POP_JUMP_IF_FALSE 568 -#define _POP_JUMP_IF_TRUE 569 #define _POP_TOP POP_TOP -#define _POP_TOP_FLOAT 570 -#define _POP_TOP_INT 571 -#define _POP_TOP_LOAD_CONST_INLINE 572 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 573 -#define _POP_TOP_NOP 574 -#define _POP_TOP_UNICODE 575 -#define _POP_TWO 576 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW 577 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 578 #define _PUSH_NULL PUSH_NULL -#define _PUSH_NULL_CONDITIONAL 579 -#define _PY_FRAME_EX 580 -#define _PY_FRAME_GENERAL 581 -#define _PY_FRAME_KW 582 -#define _RECORD_3OS_GEN_FUNC 583 -#define _RECORD_4OS 584 -#define _RECORD_BOUND_METHOD 585 -#define _RECORD_CALLABLE 586 -#define _RECORD_CODE 587 -#define _RECORD_NOS 588 -#define _RECORD_NOS_GEN_FUNC 589 -#define _RECORD_TOS 590 -#define _RECORD_TOS_TYPE 591 -#define _REPLACE_WITH_TRUE 592 -#define _RESUME_CHECK 593 #define _RETURN_GENERATOR RETURN_GENERATOR -#define _RETURN_VALUE 594 -#define _SAVE_RETURN_OFFSET 595 -#define _SEND 596 -#define _SEND_GEN_FRAME 597 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE -#define _SET_UPDATE 598 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW 599 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW 600 -#define _SPILL_OR_RELOAD 601 -#define _START_EXECUTOR 602 -#define _STORE_ATTR 603 -#define _STORE_ATTR_INSTANCE_VALUE 604 -#define _STORE_ATTR_SLOT 605 -#define _STORE_ATTR_WITH_HINT 606 #define _STORE_DEREF STORE_DEREF #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 607 -#define _STORE_SUBSCR 608 -#define _STORE_SUBSCR_DICT 609 -#define _STORE_SUBSCR_DICT_KNOWN_HASH 610 -#define _STORE_SUBSCR_LIST_INT 611 -#define _SWAP 612 -#define _SWAP_2 613 -#define _SWAP_3 614 -#define _SWAP_FAST 615 -#define _SWAP_FAST_0 616 -#define _SWAP_FAST_1 617 -#define _SWAP_FAST_2 618 -#define _SWAP_FAST_3 619 -#define _SWAP_FAST_4 620 -#define _SWAP_FAST_5 621 -#define _SWAP_FAST_6 622 -#define _SWAP_FAST_7 623 -#define _TIER2_RESUME_CHECK 624 -#define _TO_BOOL 625 #define _TO_BOOL_BOOL TO_BOOL_BOOL -#define _TO_BOOL_INT 626 -#define _TO_BOOL_LIST 627 #define _TO_BOOL_NONE TO_BOOL_NONE -#define _TO_BOOL_STR 628 #define _TRACE_RECORD TRACE_RECORD -#define _UNARY_INVERT 629 -#define _UNARY_NEGATIVE 630 -#define _UNARY_NEGATIVE_FLOAT_INPLACE 631 #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 632 -#define _UNPACK_SEQUENCE_LIST 633 -#define _UNPACK_SEQUENCE_TUPLE 634 -#define _UNPACK_SEQUENCE_TWO_TUPLE 635 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE 636 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE 637 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE 638 #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define _YIELD_VALUE 639 -#define MAX_UOP_ID 639 -#define _BINARY_OP_r23 640 -#define _BINARY_OP_ADD_FLOAT_r03 641 -#define _BINARY_OP_ADD_FLOAT_r13 642 -#define _BINARY_OP_ADD_FLOAT_r23 643 -#define _BINARY_OP_ADD_FLOAT_INPLACE_r03 644 -#define _BINARY_OP_ADD_FLOAT_INPLACE_r13 645 -#define _BINARY_OP_ADD_FLOAT_INPLACE_r23 646 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03 647 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13 648 -#define _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23 649 -#define _BINARY_OP_ADD_INT_r03 650 -#define _BINARY_OP_ADD_INT_r13 651 -#define _BINARY_OP_ADD_INT_r23 652 -#define _BINARY_OP_ADD_INT_INPLACE_r03 653 -#define _BINARY_OP_ADD_INT_INPLACE_r13 654 -#define _BINARY_OP_ADD_INT_INPLACE_r23 655 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03 656 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13 657 -#define _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23 658 -#define _BINARY_OP_ADD_UNICODE_r03 659 -#define _BINARY_OP_ADD_UNICODE_r13 660 -#define _BINARY_OP_ADD_UNICODE_r23 661 -#define _BINARY_OP_EXTEND_r23 662 -#define _BINARY_OP_INPLACE_ADD_UNICODE_r21 663 -#define _BINARY_OP_MULTIPLY_FLOAT_r03 664 -#define _BINARY_OP_MULTIPLY_FLOAT_r13 665 -#define _BINARY_OP_MULTIPLY_FLOAT_r23 666 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03 667 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13 668 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23 669 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03 670 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13 671 -#define _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23 672 -#define _BINARY_OP_MULTIPLY_INT_r03 673 -#define _BINARY_OP_MULTIPLY_INT_r13 674 -#define _BINARY_OP_MULTIPLY_INT_r23 675 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_r03 676 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_r13 677 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_r23 678 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03 679 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13 680 -#define _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23 681 -#define _BINARY_OP_SUBSCR_CHECK_FUNC_r23 682 -#define _BINARY_OP_SUBSCR_DICT_r23 683 -#define _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23 684 -#define _BINARY_OP_SUBSCR_INIT_CALL_r01 685 -#define _BINARY_OP_SUBSCR_INIT_CALL_r11 686 -#define _BINARY_OP_SUBSCR_INIT_CALL_r21 687 -#define _BINARY_OP_SUBSCR_INIT_CALL_r31 688 -#define _BINARY_OP_SUBSCR_LIST_INT_r23 689 -#define _BINARY_OP_SUBSCR_LIST_SLICE_r23 690 -#define _BINARY_OP_SUBSCR_STR_INT_r23 691 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r03 692 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r13 693 -#define _BINARY_OP_SUBSCR_TUPLE_INT_r23 694 -#define _BINARY_OP_SUBSCR_USTR_INT_r23 695 -#define _BINARY_OP_SUBTRACT_FLOAT_r03 696 -#define _BINARY_OP_SUBTRACT_FLOAT_r13 697 -#define _BINARY_OP_SUBTRACT_FLOAT_r23 698 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03 699 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13 700 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23 701 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03 702 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13 703 -#define _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23 704 -#define _BINARY_OP_SUBTRACT_INT_r03 705 -#define _BINARY_OP_SUBTRACT_INT_r13 706 -#define _BINARY_OP_SUBTRACT_INT_r23 707 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_r03 708 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_r13 709 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_r23 710 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03 711 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13 712 -#define _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23 713 -#define _BINARY_SLICE_r31 714 -#define _BUILD_INTERPOLATION_r01 715 -#define _BUILD_LIST_r01 716 -#define _BUILD_MAP_r01 717 -#define _BUILD_SET_r01 718 -#define _BUILD_SLICE_r01 719 -#define _BUILD_STRING_r01 720 -#define _BUILD_TEMPLATE_r21 721 -#define _BUILD_TUPLE_r01 722 -#define _CALL_BUILTIN_CLASS_r01 723 -#define _CALL_BUILTIN_FAST_r01 724 -#define _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 725 -#define _CALL_BUILTIN_O_r03 726 -#define _CALL_FUNCTION_EX_NON_PY_GENERAL_r31 727 -#define _CALL_INTRINSIC_1_r12 728 -#define _CALL_INTRINSIC_2_r23 729 -#define _CALL_ISINSTANCE_r31 730 -#define _CALL_KW_NON_PY_r11 731 -#define _CALL_LEN_r33 732 -#define _CALL_LIST_APPEND_r03 733 -#define _CALL_LIST_APPEND_r13 734 -#define _CALL_LIST_APPEND_r23 735 -#define _CALL_LIST_APPEND_r33 736 -#define _CALL_METHOD_DESCRIPTOR_FAST_r01 737 -#define _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01 738 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 739 -#define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01 740 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_r03 741 -#define _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03 742 -#define _CALL_METHOD_DESCRIPTOR_O_r03 743 -#define _CALL_METHOD_DESCRIPTOR_O_INLINE_r03 744 -#define _CALL_NON_PY_GENERAL_r01 745 -#define _CALL_STR_1_r32 746 -#define _CALL_TUPLE_1_r32 747 -#define _CALL_TYPE_1_r02 748 -#define _CALL_TYPE_1_r12 749 -#define _CALL_TYPE_1_r22 750 -#define _CALL_TYPE_1_r32 751 -#define _CHECK_AND_ALLOCATE_OBJECT_r00 752 -#define _CHECK_ATTR_CLASS_r01 753 -#define _CHECK_ATTR_CLASS_r11 754 -#define _CHECK_ATTR_CLASS_r22 755 -#define _CHECK_ATTR_CLASS_r33 756 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r01 757 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r11 758 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r22 759 -#define _CHECK_ATTR_METHOD_LAZY_DICT_r33 760 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00 761 -#define _CHECK_EG_MATCH_r22 762 -#define _CHECK_EXC_MATCH_r22 763 -#define _CHECK_FUNCTION_EXACT_ARGS_r00 764 -#define _CHECK_FUNCTION_VERSION_r00 765 -#define _CHECK_FUNCTION_VERSION_INLINE_r00 766 -#define _CHECK_FUNCTION_VERSION_INLINE_r11 767 -#define _CHECK_FUNCTION_VERSION_INLINE_r22 768 -#define _CHECK_FUNCTION_VERSION_INLINE_r33 769 -#define _CHECK_FUNCTION_VERSION_KW_r11 770 -#define _CHECK_IS_NOT_PY_CALLABLE_r00 771 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r03 772 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r13 773 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r23 774 -#define _CHECK_IS_NOT_PY_CALLABLE_EX_r33 775 -#define _CHECK_IS_NOT_PY_CALLABLE_KW_r11 776 -#define _CHECK_IS_PY_CALLABLE_EX_r03 777 -#define _CHECK_IS_PY_CALLABLE_EX_r13 778 -#define _CHECK_IS_PY_CALLABLE_EX_r23 779 -#define _CHECK_IS_PY_CALLABLE_EX_r33 780 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r01 781 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r11 782 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r22 783 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES_r33 784 -#define _CHECK_METHOD_VERSION_r00 785 -#define _CHECK_METHOD_VERSION_KW_r11 786 -#define _CHECK_PEP_523_r00 787 -#define _CHECK_PEP_523_r11 788 -#define _CHECK_PEP_523_r22 789 -#define _CHECK_PEP_523_r33 790 -#define _CHECK_PERIODIC_r00 791 -#define _CHECK_PERIODIC_AT_END_r00 792 -#define _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00 793 -#define _CHECK_RECURSION_LIMIT_r00 794 -#define _CHECK_RECURSION_LIMIT_r11 795 -#define _CHECK_RECURSION_LIMIT_r22 796 -#define _CHECK_RECURSION_LIMIT_r33 797 -#define _CHECK_RECURSION_REMAINING_r00 798 -#define _CHECK_RECURSION_REMAINING_r11 799 -#define _CHECK_RECURSION_REMAINING_r22 800 -#define _CHECK_RECURSION_REMAINING_r33 801 -#define _CHECK_STACK_SPACE_r00 802 -#define _CHECK_STACK_SPACE_OPERAND_r00 803 -#define _CHECK_STACK_SPACE_OPERAND_r11 804 -#define _CHECK_STACK_SPACE_OPERAND_r22 805 -#define _CHECK_STACK_SPACE_OPERAND_r33 806 -#define _CHECK_VALIDITY_r00 807 -#define _CHECK_VALIDITY_r11 808 -#define _CHECK_VALIDITY_r22 809 -#define _CHECK_VALIDITY_r33 810 -#define _COLD_DYNAMIC_EXIT_r00 811 -#define _COLD_EXIT_r00 812 -#define _COMPARE_OP_r21 813 -#define _COMPARE_OP_FLOAT_r03 814 -#define _COMPARE_OP_FLOAT_r13 815 -#define _COMPARE_OP_FLOAT_r23 816 -#define _COMPARE_OP_INT_r23 817 -#define _COMPARE_OP_STR_r23 818 -#define _CONTAINS_OP_r23 819 -#define _CONTAINS_OP_DICT_r23 820 -#define _CONTAINS_OP_SET_r23 821 -#define _CONVERT_VALUE_r11 822 -#define _COPY_r01 823 -#define _COPY_1_r02 824 -#define _COPY_1_r12 825 -#define _COPY_1_r23 826 -#define _COPY_2_r03 827 -#define _COPY_2_r13 828 -#define _COPY_2_r23 829 -#define _COPY_3_r03 830 -#define _COPY_3_r13 831 -#define _COPY_3_r23 832 -#define _COPY_3_r33 833 -#define _COPY_FREE_VARS_r00 834 -#define _COPY_FREE_VARS_r11 835 -#define _COPY_FREE_VARS_r22 836 -#define _COPY_FREE_VARS_r33 837 -#define _CREATE_INIT_FRAME_r01 838 -#define _DELETE_ATTR_r10 839 -#define _DELETE_DEREF_r00 840 -#define _DELETE_FAST_r00 841 -#define _DELETE_GLOBAL_r00 842 -#define _DELETE_NAME_r00 843 -#define _DELETE_SUBSCR_r20 844 -#define _DEOPT_r00 845 -#define _DEOPT_r10 846 -#define _DEOPT_r20 847 -#define _DEOPT_r30 848 -#define _DICT_MERGE_r11 849 -#define _DICT_UPDATE_r11 850 -#define _DO_CALL_r01 851 -#define _DO_CALL_FUNCTION_EX_r31 852 -#define _DO_CALL_KW_r11 853 -#define _DYNAMIC_EXIT_r00 854 -#define _DYNAMIC_EXIT_r10 855 -#define _DYNAMIC_EXIT_r20 856 -#define _DYNAMIC_EXIT_r30 857 -#define _END_FOR_r10 858 -#define _END_SEND_r31 859 -#define _ERROR_POP_N_r00 860 -#define _EXIT_INIT_CHECK_r10 861 -#define _EXIT_TRACE_r00 862 -#define _EXIT_TRACE_r10 863 -#define _EXIT_TRACE_r20 864 -#define _EXIT_TRACE_r30 865 -#define _EXPAND_METHOD_r00 866 -#define _EXPAND_METHOD_KW_r11 867 -#define _FATAL_ERROR_r00 868 -#define _FATAL_ERROR_r11 869 -#define _FATAL_ERROR_r22 870 -#define _FATAL_ERROR_r33 871 -#define _FORMAT_SIMPLE_r11 872 -#define _FORMAT_WITH_SPEC_r21 873 -#define _FOR_ITER_r23 874 -#define _FOR_ITER_GEN_FRAME_r03 875 -#define _FOR_ITER_GEN_FRAME_r13 876 -#define _FOR_ITER_GEN_FRAME_r23 877 -#define _FOR_ITER_TIER_TWO_r23 878 -#define _GET_AITER_r11 879 -#define _GET_ANEXT_r12 880 -#define _GET_AWAITABLE_r11 881 -#define _GET_ITER_r12 882 -#define _GET_LEN_r12 883 -#define _GUARD_BINARY_OP_EXTEND_r22 884 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02 885 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12 886 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22 887 -#define _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33 888 -#define _GUARD_BIT_IS_SET_POP_r00 889 -#define _GUARD_BIT_IS_SET_POP_r10 890 -#define _GUARD_BIT_IS_SET_POP_r21 891 -#define _GUARD_BIT_IS_SET_POP_r32 892 -#define _GUARD_BIT_IS_SET_POP_4_r00 893 -#define _GUARD_BIT_IS_SET_POP_4_r10 894 -#define _GUARD_BIT_IS_SET_POP_4_r21 895 -#define _GUARD_BIT_IS_SET_POP_4_r32 896 -#define _GUARD_BIT_IS_SET_POP_5_r00 897 -#define _GUARD_BIT_IS_SET_POP_5_r10 898 -#define _GUARD_BIT_IS_SET_POP_5_r21 899 -#define _GUARD_BIT_IS_SET_POP_5_r32 900 -#define _GUARD_BIT_IS_SET_POP_6_r00 901 -#define _GUARD_BIT_IS_SET_POP_6_r10 902 -#define _GUARD_BIT_IS_SET_POP_6_r21 903 -#define _GUARD_BIT_IS_SET_POP_6_r32 904 -#define _GUARD_BIT_IS_SET_POP_7_r00 905 -#define _GUARD_BIT_IS_SET_POP_7_r10 906 -#define _GUARD_BIT_IS_SET_POP_7_r21 907 -#define _GUARD_BIT_IS_SET_POP_7_r32 908 -#define _GUARD_BIT_IS_UNSET_POP_r00 909 -#define _GUARD_BIT_IS_UNSET_POP_r10 910 -#define _GUARD_BIT_IS_UNSET_POP_r21 911 -#define _GUARD_BIT_IS_UNSET_POP_r32 912 -#define _GUARD_BIT_IS_UNSET_POP_4_r00 913 -#define _GUARD_BIT_IS_UNSET_POP_4_r10 914 -#define _GUARD_BIT_IS_UNSET_POP_4_r21 915 -#define _GUARD_BIT_IS_UNSET_POP_4_r32 916 -#define _GUARD_BIT_IS_UNSET_POP_5_r00 917 -#define _GUARD_BIT_IS_UNSET_POP_5_r10 918 -#define _GUARD_BIT_IS_UNSET_POP_5_r21 919 -#define _GUARD_BIT_IS_UNSET_POP_5_r32 920 -#define _GUARD_BIT_IS_UNSET_POP_6_r00 921 -#define _GUARD_BIT_IS_UNSET_POP_6_r10 922 -#define _GUARD_BIT_IS_UNSET_POP_6_r21 923 -#define _GUARD_BIT_IS_UNSET_POP_6_r32 924 -#define _GUARD_BIT_IS_UNSET_POP_7_r00 925 -#define _GUARD_BIT_IS_UNSET_POP_7_r10 926 -#define _GUARD_BIT_IS_UNSET_POP_7_r21 927 -#define _GUARD_BIT_IS_UNSET_POP_7_r32 928 -#define _GUARD_CALLABLE_BUILTIN_FAST_r00 929 -#define _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00 930 -#define _GUARD_CALLABLE_BUILTIN_O_r00 931 -#define _GUARD_CALLABLE_ISINSTANCE_r03 932 -#define _GUARD_CALLABLE_ISINSTANCE_r13 933 -#define _GUARD_CALLABLE_ISINSTANCE_r23 934 -#define _GUARD_CALLABLE_ISINSTANCE_r33 935 -#define _GUARD_CALLABLE_LEN_r03 936 -#define _GUARD_CALLABLE_LEN_r13 937 -#define _GUARD_CALLABLE_LEN_r23 938 -#define _GUARD_CALLABLE_LEN_r33 939 -#define _GUARD_CALLABLE_LIST_APPEND_r03 940 -#define _GUARD_CALLABLE_LIST_APPEND_r13 941 -#define _GUARD_CALLABLE_LIST_APPEND_r23 942 -#define _GUARD_CALLABLE_LIST_APPEND_r33 943 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00 944 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 945 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00 946 -#define _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00 947 -#define _GUARD_CALLABLE_STR_1_r03 948 -#define _GUARD_CALLABLE_STR_1_r13 949 -#define _GUARD_CALLABLE_STR_1_r23 950 -#define _GUARD_CALLABLE_STR_1_r33 951 -#define _GUARD_CALLABLE_TUPLE_1_r03 952 -#define _GUARD_CALLABLE_TUPLE_1_r13 953 -#define _GUARD_CALLABLE_TUPLE_1_r23 954 -#define _GUARD_CALLABLE_TUPLE_1_r33 955 -#define _GUARD_CALLABLE_TYPE_1_r03 956 -#define _GUARD_CALLABLE_TYPE_1_r13 957 -#define _GUARD_CALLABLE_TYPE_1_r23 958 -#define _GUARD_CALLABLE_TYPE_1_r33 959 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r00 960 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r11 961 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r22 962 -#define _GUARD_CODE_VERSION_RETURN_GENERATOR_r33 963 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r00 964 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r11 965 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r22 966 -#define _GUARD_CODE_VERSION_RETURN_VALUE_r33 967 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r00 968 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r11 969 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r22 970 -#define _GUARD_CODE_VERSION_YIELD_VALUE_r33 971 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r00 972 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r11 973 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r22 974 -#define _GUARD_CODE_VERSION__PUSH_FRAME_r33 975 -#define _GUARD_DORV_NO_DICT_r01 976 -#define _GUARD_DORV_NO_DICT_r11 977 -#define _GUARD_DORV_NO_DICT_r22 978 -#define _GUARD_DORV_NO_DICT_r33 979 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 980 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 981 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 982 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 983 -#define _GUARD_GLOBALS_VERSION_r00 984 -#define _GUARD_GLOBALS_VERSION_r11 985 -#define _GUARD_GLOBALS_VERSION_r22 986 -#define _GUARD_GLOBALS_VERSION_r33 987 -#define _GUARD_IP_RETURN_GENERATOR_r00 988 -#define _GUARD_IP_RETURN_GENERATOR_r11 989 -#define _GUARD_IP_RETURN_GENERATOR_r22 990 -#define _GUARD_IP_RETURN_GENERATOR_r33 991 -#define _GUARD_IP_RETURN_VALUE_r00 992 -#define _GUARD_IP_RETURN_VALUE_r11 993 -#define _GUARD_IP_RETURN_VALUE_r22 994 -#define _GUARD_IP_RETURN_VALUE_r33 995 -#define _GUARD_IP_YIELD_VALUE_r00 996 -#define _GUARD_IP_YIELD_VALUE_r11 997 -#define _GUARD_IP_YIELD_VALUE_r22 998 -#define _GUARD_IP_YIELD_VALUE_r33 999 -#define _GUARD_IP__PUSH_FRAME_r00 1000 -#define _GUARD_IP__PUSH_FRAME_r11 1001 -#define _GUARD_IP__PUSH_FRAME_r22 1002 -#define _GUARD_IP__PUSH_FRAME_r33 1003 -#define _GUARD_IS_FALSE_POP_r00 1004 -#define _GUARD_IS_FALSE_POP_r10 1005 -#define _GUARD_IS_FALSE_POP_r21 1006 -#define _GUARD_IS_FALSE_POP_r32 1007 -#define _GUARD_IS_NONE_POP_r00 1008 -#define _GUARD_IS_NONE_POP_r10 1009 -#define _GUARD_IS_NONE_POP_r21 1010 -#define _GUARD_IS_NONE_POP_r32 1011 -#define _GUARD_IS_NOT_NONE_POP_r10 1012 -#define _GUARD_IS_TRUE_POP_r00 1013 -#define _GUARD_IS_TRUE_POP_r10 1014 -#define _GUARD_IS_TRUE_POP_r21 1015 -#define _GUARD_IS_TRUE_POP_r32 1016 -#define _GUARD_KEYS_VERSION_r01 1017 -#define _GUARD_KEYS_VERSION_r11 1018 -#define _GUARD_KEYS_VERSION_r22 1019 -#define _GUARD_KEYS_VERSION_r33 1020 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r03 1021 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r13 1022 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r23 1023 -#define _GUARD_LOAD_SUPER_ATTR_METHOD_r33 1024 -#define _GUARD_NOS_ANY_DICT_r02 1025 -#define _GUARD_NOS_ANY_DICT_r12 1026 -#define _GUARD_NOS_ANY_DICT_r22 1027 -#define _GUARD_NOS_ANY_DICT_r33 1028 -#define _GUARD_NOS_COMPACT_ASCII_r02 1029 -#define _GUARD_NOS_COMPACT_ASCII_r12 1030 -#define _GUARD_NOS_COMPACT_ASCII_r22 1031 -#define _GUARD_NOS_COMPACT_ASCII_r33 1032 -#define _GUARD_NOS_DICT_r02 1033 -#define _GUARD_NOS_DICT_r12 1034 -#define _GUARD_NOS_DICT_r22 1035 -#define _GUARD_NOS_DICT_r33 1036 -#define _GUARD_NOS_FLOAT_r02 1037 -#define _GUARD_NOS_FLOAT_r12 1038 -#define _GUARD_NOS_FLOAT_r22 1039 -#define _GUARD_NOS_FLOAT_r33 1040 -#define _GUARD_NOS_INT_r02 1041 -#define _GUARD_NOS_INT_r12 1042 -#define _GUARD_NOS_INT_r22 1043 -#define _GUARD_NOS_INT_r33 1044 -#define _GUARD_NOS_LIST_r02 1045 -#define _GUARD_NOS_LIST_r12 1046 -#define _GUARD_NOS_LIST_r22 1047 -#define _GUARD_NOS_LIST_r33 1048 -#define _GUARD_NOS_NOT_NULL_r02 1049 -#define _GUARD_NOS_NOT_NULL_r12 1050 -#define _GUARD_NOS_NOT_NULL_r22 1051 -#define _GUARD_NOS_NOT_NULL_r33 1052 -#define _GUARD_NOS_NULL_r02 1053 -#define _GUARD_NOS_NULL_r12 1054 -#define _GUARD_NOS_NULL_r22 1055 -#define _GUARD_NOS_NULL_r33 1056 -#define _GUARD_NOS_OVERFLOWED_r02 1057 -#define _GUARD_NOS_OVERFLOWED_r12 1058 -#define _GUARD_NOS_OVERFLOWED_r22 1059 -#define _GUARD_NOS_OVERFLOWED_r33 1060 -#define _GUARD_NOS_TUPLE_r02 1061 -#define _GUARD_NOS_TUPLE_r12 1062 -#define _GUARD_NOS_TUPLE_r22 1063 -#define _GUARD_NOS_TUPLE_r33 1064 -#define _GUARD_NOS_TYPE_VERSION_r02 1065 -#define _GUARD_NOS_TYPE_VERSION_r12 1066 -#define _GUARD_NOS_TYPE_VERSION_r22 1067 -#define _GUARD_NOS_TYPE_VERSION_r33 1068 -#define _GUARD_NOS_UNICODE_r02 1069 -#define _GUARD_NOS_UNICODE_r12 1070 -#define _GUARD_NOS_UNICODE_r22 1071 -#define _GUARD_NOS_UNICODE_r33 1072 -#define _GUARD_NOT_EXHAUSTED_LIST_r02 1073 -#define _GUARD_NOT_EXHAUSTED_LIST_r12 1074 -#define _GUARD_NOT_EXHAUSTED_LIST_r22 1075 -#define _GUARD_NOT_EXHAUSTED_LIST_r33 1076 -#define _GUARD_NOT_EXHAUSTED_RANGE_r02 1077 -#define _GUARD_NOT_EXHAUSTED_RANGE_r12 1078 -#define _GUARD_NOT_EXHAUSTED_RANGE_r22 1079 -#define _GUARD_NOT_EXHAUSTED_RANGE_r33 1080 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r02 1081 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r12 1082 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r22 1083 -#define _GUARD_NOT_EXHAUSTED_TUPLE_r33 1084 -#define _GUARD_THIRD_NULL_r03 1085 -#define _GUARD_THIRD_NULL_r13 1086 -#define _GUARD_THIRD_NULL_r23 1087 -#define _GUARD_THIRD_NULL_r33 1088 -#define _GUARD_TOS_ANY_DICT_r01 1089 -#define _GUARD_TOS_ANY_DICT_r11 1090 -#define _GUARD_TOS_ANY_DICT_r22 1091 -#define _GUARD_TOS_ANY_DICT_r33 1092 -#define _GUARD_TOS_ANY_SET_r01 1093 -#define _GUARD_TOS_ANY_SET_r11 1094 -#define _GUARD_TOS_ANY_SET_r22 1095 -#define _GUARD_TOS_ANY_SET_r33 1096 -#define _GUARD_TOS_DICT_r01 1097 -#define _GUARD_TOS_DICT_r11 1098 -#define _GUARD_TOS_DICT_r22 1099 -#define _GUARD_TOS_DICT_r33 1100 -#define _GUARD_TOS_FLOAT_r01 1101 -#define _GUARD_TOS_FLOAT_r11 1102 -#define _GUARD_TOS_FLOAT_r22 1103 -#define _GUARD_TOS_FLOAT_r33 1104 -#define _GUARD_TOS_FROZENDICT_r01 1105 -#define _GUARD_TOS_FROZENDICT_r11 1106 -#define _GUARD_TOS_FROZENDICT_r22 1107 -#define _GUARD_TOS_FROZENDICT_r33 1108 -#define _GUARD_TOS_FROZENSET_r01 1109 -#define _GUARD_TOS_FROZENSET_r11 1110 -#define _GUARD_TOS_FROZENSET_r22 1111 -#define _GUARD_TOS_FROZENSET_r33 1112 -#define _GUARD_TOS_INT_r01 1113 -#define _GUARD_TOS_INT_r11 1114 -#define _GUARD_TOS_INT_r22 1115 -#define _GUARD_TOS_INT_r33 1116 -#define _GUARD_TOS_LIST_r01 1117 -#define _GUARD_TOS_LIST_r11 1118 -#define _GUARD_TOS_LIST_r22 1119 -#define _GUARD_TOS_LIST_r33 1120 -#define _GUARD_TOS_OVERFLOWED_r01 1121 -#define _GUARD_TOS_OVERFLOWED_r11 1122 -#define _GUARD_TOS_OVERFLOWED_r22 1123 -#define _GUARD_TOS_OVERFLOWED_r33 1124 -#define _GUARD_TOS_SET_r01 1125 -#define _GUARD_TOS_SET_r11 1126 -#define _GUARD_TOS_SET_r22 1127 -#define _GUARD_TOS_SET_r33 1128 -#define _GUARD_TOS_SLICE_r01 1129 -#define _GUARD_TOS_SLICE_r11 1130 -#define _GUARD_TOS_SLICE_r22 1131 -#define _GUARD_TOS_SLICE_r33 1132 -#define _GUARD_TOS_TUPLE_r01 1133 -#define _GUARD_TOS_TUPLE_r11 1134 -#define _GUARD_TOS_TUPLE_r22 1135 -#define _GUARD_TOS_TUPLE_r33 1136 -#define _GUARD_TOS_UNICODE_r01 1137 -#define _GUARD_TOS_UNICODE_r11 1138 -#define _GUARD_TOS_UNICODE_r22 1139 -#define _GUARD_TOS_UNICODE_r33 1140 -#define _GUARD_TYPE_VERSION_r01 1141 -#define _GUARD_TYPE_VERSION_r11 1142 -#define _GUARD_TYPE_VERSION_r22 1143 -#define _GUARD_TYPE_VERSION_r33 1144 -#define _GUARD_TYPE_VERSION_LOCKED_r01 1145 -#define _GUARD_TYPE_VERSION_LOCKED_r11 1146 -#define _GUARD_TYPE_VERSION_LOCKED_r22 1147 -#define _GUARD_TYPE_VERSION_LOCKED_r33 1148 -#define _HANDLE_PENDING_AND_DEOPT_r00 1149 -#define _HANDLE_PENDING_AND_DEOPT_r10 1150 -#define _HANDLE_PENDING_AND_DEOPT_r20 1151 -#define _HANDLE_PENDING_AND_DEOPT_r30 1152 -#define _IMPORT_FROM_r12 1153 -#define _IMPORT_NAME_r21 1154 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00 1155 -#define _INIT_CALL_PY_EXACT_ARGS_r01 1156 -#define _INIT_CALL_PY_EXACT_ARGS_0_r01 1157 -#define _INIT_CALL_PY_EXACT_ARGS_1_r01 1158 -#define _INIT_CALL_PY_EXACT_ARGS_2_r01 1159 -#define _INIT_CALL_PY_EXACT_ARGS_3_r01 1160 -#define _INIT_CALL_PY_EXACT_ARGS_4_r01 1161 -#define _INSERT_1_LOAD_CONST_INLINE_r02 1162 -#define _INSERT_1_LOAD_CONST_INLINE_r12 1163 -#define _INSERT_1_LOAD_CONST_INLINE_r23 1164 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 1165 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 1166 -#define _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 1167 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 1168 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 1169 -#define _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 1170 -#define _INSERT_NULL_r10 1171 -#define _INSTRUMENTED_FOR_ITER_r23 1172 -#define _INSTRUMENTED_INSTRUCTION_r00 1173 -#define _INSTRUMENTED_JUMP_FORWARD_r00 1174 -#define _INSTRUMENTED_JUMP_FORWARD_r11 1175 -#define _INSTRUMENTED_JUMP_FORWARD_r22 1176 -#define _INSTRUMENTED_JUMP_FORWARD_r33 1177 -#define _INSTRUMENTED_LINE_r00 1178 -#define _INSTRUMENTED_NOT_TAKEN_r00 1179 -#define _INSTRUMENTED_NOT_TAKEN_r11 1180 -#define _INSTRUMENTED_NOT_TAKEN_r22 1181 -#define _INSTRUMENTED_NOT_TAKEN_r33 1182 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r00 1183 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r10 1184 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r21 1185 -#define _INSTRUMENTED_POP_JUMP_IF_FALSE_r32 1186 -#define _INSTRUMENTED_POP_JUMP_IF_NONE_r10 1187 -#define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10 1188 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r00 1189 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r10 1190 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r21 1191 -#define _INSTRUMENTED_POP_JUMP_IF_TRUE_r32 1192 -#define _IS_NONE_r11 1193 -#define _IS_OP_r03 1194 -#define _IS_OP_r13 1195 -#define _IS_OP_r23 1196 -#define _ITER_CHECK_LIST_r02 1197 -#define _ITER_CHECK_LIST_r12 1198 -#define _ITER_CHECK_LIST_r22 1199 -#define _ITER_CHECK_LIST_r33 1200 -#define _ITER_CHECK_RANGE_r02 1201 -#define _ITER_CHECK_RANGE_r12 1202 -#define _ITER_CHECK_RANGE_r22 1203 -#define _ITER_CHECK_RANGE_r33 1204 -#define _ITER_CHECK_TUPLE_r02 1205 -#define _ITER_CHECK_TUPLE_r12 1206 -#define _ITER_CHECK_TUPLE_r22 1207 -#define _ITER_CHECK_TUPLE_r33 1208 -#define _ITER_JUMP_LIST_r02 1209 -#define _ITER_JUMP_LIST_r12 1210 -#define _ITER_JUMP_LIST_r22 1211 -#define _ITER_JUMP_LIST_r33 1212 -#define _ITER_JUMP_RANGE_r02 1213 -#define _ITER_JUMP_RANGE_r12 1214 -#define _ITER_JUMP_RANGE_r22 1215 -#define _ITER_JUMP_RANGE_r33 1216 -#define _ITER_JUMP_TUPLE_r02 1217 -#define _ITER_JUMP_TUPLE_r12 1218 -#define _ITER_JUMP_TUPLE_r22 1219 -#define _ITER_JUMP_TUPLE_r33 1220 -#define _ITER_NEXT_LIST_r23 1221 -#define _ITER_NEXT_LIST_TIER_TWO_r23 1222 -#define _ITER_NEXT_RANGE_r03 1223 -#define _ITER_NEXT_RANGE_r13 1224 -#define _ITER_NEXT_RANGE_r23 1225 -#define _ITER_NEXT_TUPLE_r03 1226 -#define _ITER_NEXT_TUPLE_r13 1227 -#define _ITER_NEXT_TUPLE_r23 1228 -#define _JUMP_BACKWARD_NO_INTERRUPT_r00 1229 -#define _JUMP_BACKWARD_NO_INTERRUPT_r11 1230 -#define _JUMP_BACKWARD_NO_INTERRUPT_r22 1231 -#define _JUMP_BACKWARD_NO_INTERRUPT_r33 1232 -#define _JUMP_TO_TOP_r00 1233 -#define _LIST_APPEND_r10 1234 -#define _LIST_EXTEND_r11 1235 -#define _LOAD_ATTR_r10 1236 -#define _LOAD_ATTR_CLASS_r11 1237 -#define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_r11 1238 -#define _LOAD_ATTR_INSTANCE_VALUE_r02 1239 -#define _LOAD_ATTR_INSTANCE_VALUE_r12 1240 -#define _LOAD_ATTR_INSTANCE_VALUE_r23 1241 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r02 1242 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r12 1243 -#define _LOAD_ATTR_METHOD_LAZY_DICT_r23 1244 -#define _LOAD_ATTR_METHOD_NO_DICT_r02 1245 -#define _LOAD_ATTR_METHOD_NO_DICT_r12 1246 -#define _LOAD_ATTR_METHOD_NO_DICT_r23 1247 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r02 1248 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r12 1249 -#define _LOAD_ATTR_METHOD_WITH_VALUES_r23 1250 -#define _LOAD_ATTR_MODULE_r12 1251 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11 1252 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11 1253 -#define _LOAD_ATTR_PROPERTY_FRAME_r11 1254 -#define _LOAD_ATTR_SLOT_r02 1255 -#define _LOAD_ATTR_SLOT_r12 1256 -#define _LOAD_ATTR_SLOT_r23 1257 -#define _LOAD_ATTR_WITH_HINT_r12 1258 -#define _LOAD_BUILD_CLASS_r01 1259 -#define _LOAD_BYTECODE_r00 1260 -#define _LOAD_COMMON_CONSTANT_r01 1261 -#define _LOAD_COMMON_CONSTANT_r12 1262 -#define _LOAD_COMMON_CONSTANT_r23 1263 -#define _LOAD_CONST_r01 1264 -#define _LOAD_CONST_r12 1265 -#define _LOAD_CONST_r23 1266 -#define _LOAD_CONST_INLINE_r01 1267 -#define _LOAD_CONST_INLINE_r12 1268 -#define _LOAD_CONST_INLINE_r23 1269 -#define _LOAD_CONST_INLINE_BORROW_r01 1270 -#define _LOAD_CONST_INLINE_BORROW_r12 1271 -#define _LOAD_CONST_INLINE_BORROW_r23 1272 -#define _LOAD_DEREF_r01 1273 -#define _LOAD_FAST_r01 1274 -#define _LOAD_FAST_r12 1275 -#define _LOAD_FAST_r23 1276 -#define _LOAD_FAST_0_r01 1277 -#define _LOAD_FAST_0_r12 1278 -#define _LOAD_FAST_0_r23 1279 -#define _LOAD_FAST_1_r01 1280 -#define _LOAD_FAST_1_r12 1281 -#define _LOAD_FAST_1_r23 1282 -#define _LOAD_FAST_2_r01 1283 -#define _LOAD_FAST_2_r12 1284 -#define _LOAD_FAST_2_r23 1285 -#define _LOAD_FAST_3_r01 1286 -#define _LOAD_FAST_3_r12 1287 -#define _LOAD_FAST_3_r23 1288 -#define _LOAD_FAST_4_r01 1289 -#define _LOAD_FAST_4_r12 1290 -#define _LOAD_FAST_4_r23 1291 -#define _LOAD_FAST_5_r01 1292 -#define _LOAD_FAST_5_r12 1293 -#define _LOAD_FAST_5_r23 1294 -#define _LOAD_FAST_6_r01 1295 -#define _LOAD_FAST_6_r12 1296 -#define _LOAD_FAST_6_r23 1297 -#define _LOAD_FAST_7_r01 1298 -#define _LOAD_FAST_7_r12 1299 -#define _LOAD_FAST_7_r23 1300 -#define _LOAD_FAST_AND_CLEAR_r01 1301 -#define _LOAD_FAST_AND_CLEAR_r12 1302 -#define _LOAD_FAST_AND_CLEAR_r23 1303 -#define _LOAD_FAST_BORROW_r01 1304 -#define _LOAD_FAST_BORROW_r12 1305 -#define _LOAD_FAST_BORROW_r23 1306 -#define _LOAD_FAST_BORROW_0_r01 1307 -#define _LOAD_FAST_BORROW_0_r12 1308 -#define _LOAD_FAST_BORROW_0_r23 1309 -#define _LOAD_FAST_BORROW_1_r01 1310 -#define _LOAD_FAST_BORROW_1_r12 1311 -#define _LOAD_FAST_BORROW_1_r23 1312 -#define _LOAD_FAST_BORROW_2_r01 1313 -#define _LOAD_FAST_BORROW_2_r12 1314 -#define _LOAD_FAST_BORROW_2_r23 1315 -#define _LOAD_FAST_BORROW_3_r01 1316 -#define _LOAD_FAST_BORROW_3_r12 1317 -#define _LOAD_FAST_BORROW_3_r23 1318 -#define _LOAD_FAST_BORROW_4_r01 1319 -#define _LOAD_FAST_BORROW_4_r12 1320 -#define _LOAD_FAST_BORROW_4_r23 1321 -#define _LOAD_FAST_BORROW_5_r01 1322 -#define _LOAD_FAST_BORROW_5_r12 1323 -#define _LOAD_FAST_BORROW_5_r23 1324 -#define _LOAD_FAST_BORROW_6_r01 1325 -#define _LOAD_FAST_BORROW_6_r12 1326 -#define _LOAD_FAST_BORROW_6_r23 1327 -#define _LOAD_FAST_BORROW_7_r01 1328 -#define _LOAD_FAST_BORROW_7_r12 1329 -#define _LOAD_FAST_BORROW_7_r23 1330 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1331 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1332 -#define _LOAD_FAST_CHECK_r01 1333 -#define _LOAD_FAST_CHECK_r12 1334 -#define _LOAD_FAST_CHECK_r23 1335 -#define _LOAD_FAST_LOAD_FAST_r02 1336 -#define _LOAD_FAST_LOAD_FAST_r13 1337 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1338 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1339 -#define _LOAD_GLOBAL_r00 1340 -#define _LOAD_GLOBAL_BUILTINS_r01 1341 -#define _LOAD_GLOBAL_MODULE_r01 1342 -#define _LOAD_LOCALS_r01 1343 -#define _LOAD_LOCALS_r12 1344 -#define _LOAD_LOCALS_r23 1345 -#define _LOAD_NAME_r01 1346 -#define _LOAD_SMALL_INT_r01 1347 -#define _LOAD_SMALL_INT_r12 1348 -#define _LOAD_SMALL_INT_r23 1349 -#define _LOAD_SMALL_INT_0_r01 1350 -#define _LOAD_SMALL_INT_0_r12 1351 -#define _LOAD_SMALL_INT_0_r23 1352 -#define _LOAD_SMALL_INT_1_r01 1353 -#define _LOAD_SMALL_INT_1_r12 1354 -#define _LOAD_SMALL_INT_1_r23 1355 -#define _LOAD_SMALL_INT_2_r01 1356 -#define _LOAD_SMALL_INT_2_r12 1357 -#define _LOAD_SMALL_INT_2_r23 1358 -#define _LOAD_SMALL_INT_3_r01 1359 -#define _LOAD_SMALL_INT_3_r12 1360 -#define _LOAD_SMALL_INT_3_r23 1361 -#define _LOAD_SPECIAL_r00 1362 -#define _LOAD_SUPER_ATTR_ATTR_r31 1363 -#define _LOAD_SUPER_ATTR_METHOD_r32 1364 -#define _LOCK_OBJECT_r01 1365 -#define _LOCK_OBJECT_r11 1366 -#define _LOCK_OBJECT_r22 1367 -#define _LOCK_OBJECT_r33 1368 -#define _MAKE_CALLARGS_A_TUPLE_r33 1369 -#define _MAKE_CELL_r00 1370 -#define _MAKE_FUNCTION_r11 1371 -#define _MAKE_HEAP_SAFE_r01 1372 -#define _MAKE_HEAP_SAFE_r11 1373 -#define _MAKE_HEAP_SAFE_r22 1374 -#define _MAKE_HEAP_SAFE_r33 1375 -#define _MAKE_WARM_r00 1376 -#define _MAKE_WARM_r11 1377 -#define _MAKE_WARM_r22 1378 -#define _MAKE_WARM_r33 1379 -#define _MAP_ADD_r20 1380 -#define _MATCH_CLASS_r33 1381 -#define _MATCH_KEYS_r23 1382 -#define _MATCH_MAPPING_r02 1383 -#define _MATCH_MAPPING_r12 1384 -#define _MATCH_MAPPING_r23 1385 -#define _MATCH_SEQUENCE_r02 1386 -#define _MATCH_SEQUENCE_r12 1387 -#define _MATCH_SEQUENCE_r23 1388 -#define _MAYBE_EXPAND_METHOD_r00 1389 -#define _MAYBE_EXPAND_METHOD_KW_r11 1390 -#define _MONITOR_CALL_r00 1391 -#define _MONITOR_CALL_KW_r11 1392 -#define _MONITOR_JUMP_BACKWARD_r00 1393 -#define _MONITOR_JUMP_BACKWARD_r11 1394 -#define _MONITOR_JUMP_BACKWARD_r22 1395 -#define _MONITOR_JUMP_BACKWARD_r33 1396 -#define _MONITOR_RESUME_r00 1397 -#define _NOP_r00 1398 -#define _NOP_r11 1399 -#define _NOP_r22 1400 -#define _NOP_r33 1401 -#define _POP_CALL_r20 1402 -#define _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 1403 -#define _POP_CALL_ONE_r30 1404 -#define _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 1405 -#define _POP_CALL_TWO_r30 1406 -#define _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 1407 -#define _POP_EXCEPT_r10 1408 -#define _POP_ITER_r20 1409 -#define _POP_JUMP_IF_FALSE_r00 1410 -#define _POP_JUMP_IF_FALSE_r10 1411 -#define _POP_JUMP_IF_FALSE_r21 1412 -#define _POP_JUMP_IF_FALSE_r32 1413 -#define _POP_JUMP_IF_TRUE_r00 1414 -#define _POP_JUMP_IF_TRUE_r10 1415 -#define _POP_JUMP_IF_TRUE_r21 1416 -#define _POP_JUMP_IF_TRUE_r32 1417 -#define _POP_TOP_r10 1418 -#define _POP_TOP_FLOAT_r00 1419 -#define _POP_TOP_FLOAT_r10 1420 -#define _POP_TOP_FLOAT_r21 1421 -#define _POP_TOP_FLOAT_r32 1422 -#define _POP_TOP_INT_r00 1423 -#define _POP_TOP_INT_r10 1424 -#define _POP_TOP_INT_r21 1425 -#define _POP_TOP_INT_r32 1426 -#define _POP_TOP_LOAD_CONST_INLINE_r11 1427 -#define _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 1428 -#define _POP_TOP_NOP_r00 1429 -#define _POP_TOP_NOP_r10 1430 -#define _POP_TOP_NOP_r21 1431 -#define _POP_TOP_NOP_r32 1432 -#define _POP_TOP_UNICODE_r00 1433 -#define _POP_TOP_UNICODE_r10 1434 -#define _POP_TOP_UNICODE_r21 1435 -#define _POP_TOP_UNICODE_r32 1436 -#define _POP_TWO_r20 1437 -#define _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 1438 -#define _PUSH_EXC_INFO_r02 1439 -#define _PUSH_EXC_INFO_r12 1440 -#define _PUSH_EXC_INFO_r23 1441 -#define _PUSH_FRAME_r10 1442 -#define _PUSH_NULL_r01 1443 -#define _PUSH_NULL_r12 1444 -#define _PUSH_NULL_r23 1445 -#define _PUSH_NULL_CONDITIONAL_r00 1446 -#define _PY_FRAME_EX_r31 1447 -#define _PY_FRAME_GENERAL_r01 1448 -#define _PY_FRAME_KW_r11 1449 -#define _REPLACE_WITH_TRUE_r02 1450 -#define _REPLACE_WITH_TRUE_r12 1451 -#define _REPLACE_WITH_TRUE_r23 1452 -#define _RESUME_CHECK_r00 1453 -#define _RESUME_CHECK_r11 1454 -#define _RESUME_CHECK_r22 1455 -#define _RESUME_CHECK_r33 1456 -#define _RETURN_GENERATOR_r01 1457 -#define _RETURN_VALUE_r11 1458 -#define _SAVE_RETURN_OFFSET_r00 1459 -#define _SAVE_RETURN_OFFSET_r11 1460 -#define _SAVE_RETURN_OFFSET_r22 1461 -#define _SAVE_RETURN_OFFSET_r33 1462 -#define _SEND_r33 1463 -#define _SEND_GEN_FRAME_r33 1464 -#define _SETUP_ANNOTATIONS_r00 1465 -#define _SET_ADD_r10 1466 -#define _SET_FUNCTION_ATTRIBUTE_r01 1467 -#define _SET_FUNCTION_ATTRIBUTE_r11 1468 -#define _SET_FUNCTION_ATTRIBUTE_r21 1469 -#define _SET_FUNCTION_ATTRIBUTE_r32 1470 -#define _SET_IP_r00 1471 -#define _SET_IP_r11 1472 -#define _SET_IP_r22 1473 -#define _SET_IP_r33 1474 -#define _SET_UPDATE_r11 1475 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 1476 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 1477 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 1478 -#define _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 1479 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1480 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1481 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1482 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1483 -#define _SPILL_OR_RELOAD_r01 1484 -#define _SPILL_OR_RELOAD_r02 1485 -#define _SPILL_OR_RELOAD_r03 1486 -#define _SPILL_OR_RELOAD_r10 1487 -#define _SPILL_OR_RELOAD_r12 1488 -#define _SPILL_OR_RELOAD_r13 1489 -#define _SPILL_OR_RELOAD_r20 1490 -#define _SPILL_OR_RELOAD_r21 1491 -#define _SPILL_OR_RELOAD_r23 1492 -#define _SPILL_OR_RELOAD_r30 1493 -#define _SPILL_OR_RELOAD_r31 1494 -#define _SPILL_OR_RELOAD_r32 1495 -#define _START_EXECUTOR_r00 1496 -#define _STORE_ATTR_r20 1497 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1498 -#define _STORE_ATTR_SLOT_r21 1499 -#define _STORE_ATTR_WITH_HINT_r21 1500 -#define _STORE_DEREF_r10 1501 -#define _STORE_FAST_LOAD_FAST_r11 1502 -#define _STORE_FAST_STORE_FAST_r20 1503 -#define _STORE_GLOBAL_r10 1504 -#define _STORE_NAME_r10 1505 -#define _STORE_SLICE_r30 1506 -#define _STORE_SUBSCR_r30 1507 -#define _STORE_SUBSCR_DICT_r31 1508 -#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1509 -#define _STORE_SUBSCR_LIST_INT_r32 1510 -#define _SWAP_r11 1511 -#define _SWAP_2_r02 1512 -#define _SWAP_2_r12 1513 -#define _SWAP_2_r22 1514 -#define _SWAP_2_r33 1515 -#define _SWAP_3_r03 1516 -#define _SWAP_3_r13 1517 -#define _SWAP_3_r23 1518 -#define _SWAP_3_r33 1519 -#define _SWAP_FAST_r01 1520 -#define _SWAP_FAST_r11 1521 -#define _SWAP_FAST_r22 1522 -#define _SWAP_FAST_r33 1523 -#define _SWAP_FAST_0_r01 1524 -#define _SWAP_FAST_0_r11 1525 -#define _SWAP_FAST_0_r22 1526 -#define _SWAP_FAST_0_r33 1527 -#define _SWAP_FAST_1_r01 1528 -#define _SWAP_FAST_1_r11 1529 -#define _SWAP_FAST_1_r22 1530 -#define _SWAP_FAST_1_r33 1531 -#define _SWAP_FAST_2_r01 1532 -#define _SWAP_FAST_2_r11 1533 -#define _SWAP_FAST_2_r22 1534 -#define _SWAP_FAST_2_r33 1535 -#define _SWAP_FAST_3_r01 1536 -#define _SWAP_FAST_3_r11 1537 -#define _SWAP_FAST_3_r22 1538 -#define _SWAP_FAST_3_r33 1539 -#define _SWAP_FAST_4_r01 1540 -#define _SWAP_FAST_4_r11 1541 -#define _SWAP_FAST_4_r22 1542 -#define _SWAP_FAST_4_r33 1543 -#define _SWAP_FAST_5_r01 1544 -#define _SWAP_FAST_5_r11 1545 -#define _SWAP_FAST_5_r22 1546 -#define _SWAP_FAST_5_r33 1547 -#define _SWAP_FAST_6_r01 1548 -#define _SWAP_FAST_6_r11 1549 -#define _SWAP_FAST_6_r22 1550 -#define _SWAP_FAST_6_r33 1551 -#define _SWAP_FAST_7_r01 1552 -#define _SWAP_FAST_7_r11 1553 -#define _SWAP_FAST_7_r22 1554 -#define _SWAP_FAST_7_r33 1555 -#define _TIER2_RESUME_CHECK_r00 1556 -#define _TIER2_RESUME_CHECK_r11 1557 -#define _TIER2_RESUME_CHECK_r22 1558 -#define _TIER2_RESUME_CHECK_r33 1559 -#define _TO_BOOL_r11 1560 -#define _TO_BOOL_BOOL_r01 1561 -#define _TO_BOOL_BOOL_r11 1562 -#define _TO_BOOL_BOOL_r22 1563 -#define _TO_BOOL_BOOL_r33 1564 -#define _TO_BOOL_INT_r02 1565 -#define _TO_BOOL_INT_r12 1566 -#define _TO_BOOL_INT_r23 1567 -#define _TO_BOOL_LIST_r02 1568 -#define _TO_BOOL_LIST_r12 1569 -#define _TO_BOOL_LIST_r23 1570 -#define _TO_BOOL_NONE_r01 1571 -#define _TO_BOOL_NONE_r11 1572 -#define _TO_BOOL_NONE_r22 1573 -#define _TO_BOOL_NONE_r33 1574 -#define _TO_BOOL_STR_r02 1575 -#define _TO_BOOL_STR_r12 1576 -#define _TO_BOOL_STR_r23 1577 -#define _TRACE_RECORD_r00 1578 -#define _UNARY_INVERT_r12 1579 -#define _UNARY_NEGATIVE_r12 1580 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1581 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1582 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1583 -#define _UNARY_NOT_r01 1584 -#define _UNARY_NOT_r11 1585 -#define _UNARY_NOT_r22 1586 -#define _UNARY_NOT_r33 1587 -#define _UNPACK_EX_r10 1588 -#define _UNPACK_SEQUENCE_r10 1589 -#define _UNPACK_SEQUENCE_LIST_r10 1590 -#define _UNPACK_SEQUENCE_TUPLE_r10 1591 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1592 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1593 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1594 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1595 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1596 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1597 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1598 -#define _WITH_EXCEPT_START_r33 1599 -#define _YIELD_VALUE_r11 1600 -#define MAX_UOP_REGS_ID 1600 + +enum { + _ALLOCATE_OBJECT_r00 = MAX_UOP_ID + 1, + _BINARY_OP_r23, + _BINARY_OP_ADD_FLOAT_r03, + _BINARY_OP_ADD_FLOAT_r13, + _BINARY_OP_ADD_FLOAT_r23, + _BINARY_OP_ADD_FLOAT_INPLACE_r03, + _BINARY_OP_ADD_FLOAT_INPLACE_r13, + _BINARY_OP_ADD_FLOAT_INPLACE_r23, + _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r03, + _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r13, + _BINARY_OP_ADD_FLOAT_INPLACE_RIGHT_r23, + _BINARY_OP_ADD_INT_r03, + _BINARY_OP_ADD_INT_r13, + _BINARY_OP_ADD_INT_r23, + _BINARY_OP_ADD_INT_INPLACE_r03, + _BINARY_OP_ADD_INT_INPLACE_r13, + _BINARY_OP_ADD_INT_INPLACE_r23, + _BINARY_OP_ADD_INT_INPLACE_RIGHT_r03, + _BINARY_OP_ADD_INT_INPLACE_RIGHT_r13, + _BINARY_OP_ADD_INT_INPLACE_RIGHT_r23, + _BINARY_OP_ADD_UNICODE_r03, + _BINARY_OP_ADD_UNICODE_r13, + _BINARY_OP_ADD_UNICODE_r23, + _BINARY_OP_EXTEND_r23, + _BINARY_OP_INPLACE_ADD_UNICODE_r21, + _BINARY_OP_MULTIPLY_FLOAT_r03, + _BINARY_OP_MULTIPLY_FLOAT_r13, + _BINARY_OP_MULTIPLY_FLOAT_r23, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r03, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r13, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_r23, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r03, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r13, + _BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT_r23, + _BINARY_OP_MULTIPLY_INT_r03, + _BINARY_OP_MULTIPLY_INT_r13, + _BINARY_OP_MULTIPLY_INT_r23, + _BINARY_OP_MULTIPLY_INT_INPLACE_r03, + _BINARY_OP_MULTIPLY_INT_INPLACE_r13, + _BINARY_OP_MULTIPLY_INT_INPLACE_r23, + _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r03, + _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r13, + _BINARY_OP_MULTIPLY_INT_INPLACE_RIGHT_r23, + _BINARY_OP_SUBSCR_CHECK_FUNC_r23, + _BINARY_OP_SUBSCR_DICT_r23, + _BINARY_OP_SUBSCR_DICT_KNOWN_HASH_r23, + _BINARY_OP_SUBSCR_INIT_CALL_r01, + _BINARY_OP_SUBSCR_INIT_CALL_r11, + _BINARY_OP_SUBSCR_INIT_CALL_r21, + _BINARY_OP_SUBSCR_INIT_CALL_r31, + _BINARY_OP_SUBSCR_LIST_INT_r23, + _BINARY_OP_SUBSCR_LIST_SLICE_r23, + _BINARY_OP_SUBSCR_STR_INT_r23, + _BINARY_OP_SUBSCR_TUPLE_INT_r03, + _BINARY_OP_SUBSCR_TUPLE_INT_r13, + _BINARY_OP_SUBSCR_TUPLE_INT_r23, + _BINARY_OP_SUBSCR_USTR_INT_r23, + _BINARY_OP_SUBTRACT_FLOAT_r03, + _BINARY_OP_SUBTRACT_FLOAT_r13, + _BINARY_OP_SUBTRACT_FLOAT_r23, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r03, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r13, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_r23, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13, + _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23, + _BINARY_OP_SUBTRACT_INT_r03, + _BINARY_OP_SUBTRACT_INT_r13, + _BINARY_OP_SUBTRACT_INT_r23, + _BINARY_OP_SUBTRACT_INT_INPLACE_r03, + _BINARY_OP_SUBTRACT_INT_INPLACE_r13, + _BINARY_OP_SUBTRACT_INT_INPLACE_r23, + _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03, + _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13, + _BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23, + _BINARY_OP_TRUEDIV_FLOAT_r23, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13, + _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23, + _BINARY_SLICE_r31, + _BUILD_INTERPOLATION_r01, + _BUILD_LIST_r01, + _BUILD_MAP_r01, + _BUILD_SET_r01, + _BUILD_SLICE_r01, + _BUILD_STRING_r01, + _BUILD_TEMPLATE_r21, + _BUILD_TUPLE_r01, + _CALL_BUILTIN_CLASS_r00, + _CALL_BUILTIN_FAST_r00, + _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00, + _CALL_BUILTIN_O_r03, + _CALL_FUNCTION_EX_NON_PY_GENERAL_r31, + _CALL_INTRINSIC_1_r12, + _CALL_INTRINSIC_2_r23, + _CALL_ISINSTANCE_r31, + _CALL_KW_NON_PY_r11, + _CALL_LEN_r33, + _CALL_LIST_APPEND_r03, + _CALL_LIST_APPEND_r13, + _CALL_LIST_APPEND_r23, + _CALL_LIST_APPEND_r33, + _CALL_METHOD_DESCRIPTOR_FAST_r00, + _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00, + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00, + _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00, + _CALL_METHOD_DESCRIPTOR_NOARGS_r03, + _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03, + _CALL_METHOD_DESCRIPTOR_O_r03, + _CALL_METHOD_DESCRIPTOR_O_INLINE_r03, + _CALL_NON_PY_GENERAL_r01, + _CALL_STR_1_r32, + _CALL_TUPLE_1_r32, + _CALL_TYPE_1_r02, + _CALL_TYPE_1_r12, + _CALL_TYPE_1_r22, + _CALL_TYPE_1_r32, + _CHECK_ATTR_CLASS_r01, + _CHECK_ATTR_CLASS_r11, + _CHECK_ATTR_CLASS_r22, + _CHECK_ATTR_CLASS_r33, + _CHECK_ATTR_METHOD_LAZY_DICT_r01, + _CHECK_ATTR_METHOD_LAZY_DICT_r11, + _CHECK_ATTR_METHOD_LAZY_DICT_r22, + _CHECK_ATTR_METHOD_LAZY_DICT_r33, + _CHECK_CALL_BOUND_METHOD_EXACT_ARGS_r00, + _CHECK_EG_MATCH_r22, + _CHECK_EXC_MATCH_r22, + _CHECK_FUNCTION_EXACT_ARGS_r00, + _CHECK_FUNCTION_VERSION_r00, + _CHECK_FUNCTION_VERSION_INLINE_r00, + _CHECK_FUNCTION_VERSION_INLINE_r11, + _CHECK_FUNCTION_VERSION_INLINE_r22, + _CHECK_FUNCTION_VERSION_INLINE_r33, + _CHECK_FUNCTION_VERSION_KW_r11, + _CHECK_IS_NOT_PY_CALLABLE_r00, + _CHECK_IS_NOT_PY_CALLABLE_EX_r03, + _CHECK_IS_NOT_PY_CALLABLE_EX_r13, + _CHECK_IS_NOT_PY_CALLABLE_EX_r23, + _CHECK_IS_NOT_PY_CALLABLE_EX_r33, + _CHECK_IS_NOT_PY_CALLABLE_KW_r11, + _CHECK_IS_PY_CALLABLE_EX_r03, + _CHECK_IS_PY_CALLABLE_EX_r13, + _CHECK_IS_PY_CALLABLE_EX_r23, + _CHECK_IS_PY_CALLABLE_EX_r33, + _CHECK_MANAGED_OBJECT_HAS_VALUES_r01, + _CHECK_MANAGED_OBJECT_HAS_VALUES_r11, + _CHECK_MANAGED_OBJECT_HAS_VALUES_r22, + _CHECK_MANAGED_OBJECT_HAS_VALUES_r33, + _CHECK_METHOD_VERSION_r00, + _CHECK_METHOD_VERSION_KW_r11, + _CHECK_OBJECT_r00, + _CHECK_PEP_523_r00, + _CHECK_PEP_523_r11, + _CHECK_PEP_523_r22, + _CHECK_PEP_523_r33, + _CHECK_PERIODIC_r00, + _CHECK_PERIODIC_AT_END_r00, + _CHECK_PERIODIC_IF_NOT_YIELD_FROM_r00, + _CHECK_RECURSION_LIMIT_r00, + _CHECK_RECURSION_LIMIT_r11, + _CHECK_RECURSION_LIMIT_r22, + _CHECK_RECURSION_LIMIT_r33, + _CHECK_RECURSION_REMAINING_r00, + _CHECK_RECURSION_REMAINING_r11, + _CHECK_RECURSION_REMAINING_r22, + _CHECK_RECURSION_REMAINING_r33, + _CHECK_STACK_SPACE_r00, + _CHECK_STACK_SPACE_OPERAND_r00, + _CHECK_STACK_SPACE_OPERAND_r11, + _CHECK_STACK_SPACE_OPERAND_r22, + _CHECK_STACK_SPACE_OPERAND_r33, + _CHECK_VALIDITY_r00, + _CHECK_VALIDITY_r11, + _CHECK_VALIDITY_r22, + _CHECK_VALIDITY_r33, + _COLD_DYNAMIC_EXIT_r00, + _COLD_EXIT_r00, + _COMPARE_OP_r21, + _COMPARE_OP_FLOAT_r03, + _COMPARE_OP_FLOAT_r13, + _COMPARE_OP_FLOAT_r23, + _COMPARE_OP_INT_r23, + _COMPARE_OP_STR_r23, + _CONTAINS_OP_r23, + _CONTAINS_OP_DICT_r23, + _CONTAINS_OP_SET_r23, + _CONVERT_VALUE_r11, + _COPY_r01, + _COPY_1_r02, + _COPY_1_r12, + _COPY_1_r23, + _COPY_2_r03, + _COPY_2_r13, + _COPY_2_r23, + _COPY_3_r03, + _COPY_3_r13, + _COPY_3_r23, + _COPY_3_r33, + _COPY_FREE_VARS_r00, + _COPY_FREE_VARS_r11, + _COPY_FREE_VARS_r22, + _COPY_FREE_VARS_r33, + _CREATE_INIT_FRAME_r01, + _DELETE_DEREF_r00, + _DELETE_FAST_r00, + _DELETE_SUBSCR_r20, + _DEOPT_r00, + _DEOPT_r10, + _DEOPT_r20, + _DEOPT_r30, + _DICT_MERGE_r11, + _DICT_UPDATE_r11, + _DO_CALL_r01, + _DO_CALL_FUNCTION_EX_r31, + _DO_CALL_KW_r11, + _DYNAMIC_EXIT_r00, + _DYNAMIC_EXIT_r10, + _DYNAMIC_EXIT_r20, + _DYNAMIC_EXIT_r30, + _END_FOR_r10, + _END_SEND_r31, + _ERROR_POP_N_r00, + _EXIT_INIT_CHECK_r10, + _EXIT_TRACE_r00, + _EXIT_TRACE_r10, + _EXIT_TRACE_r20, + _EXIT_TRACE_r30, + _EXPAND_METHOD_r00, + _EXPAND_METHOD_KW_r11, + _FATAL_ERROR_r00, + _FATAL_ERROR_r11, + _FATAL_ERROR_r22, + _FATAL_ERROR_r33, + _FORMAT_SIMPLE_r11, + _FORMAT_WITH_SPEC_r21, + _FOR_ITER_r23, + _FOR_ITER_GEN_FRAME_r03, + _FOR_ITER_GEN_FRAME_r13, + _FOR_ITER_GEN_FRAME_r23, + _FOR_ITER_TIER_TWO_r23, + _FOR_ITER_VIRTUAL_r23, + _FOR_ITER_VIRTUAL_TIER_TWO_r23, + _GET_AITER_r11, + _GET_ANEXT_r12, + _GET_AWAITABLE_r11, + _GET_ITER_r12, + _GET_ITER_TRAD_r12, + _GET_LEN_r12, + _GUARD_3OS_ASYNC_GEN_ASEND_r03, + _GUARD_3OS_ASYNC_GEN_ASEND_r13, + _GUARD_3OS_ASYNC_GEN_ASEND_r23, + _GUARD_3OS_ASYNC_GEN_ASEND_r33, + _GUARD_BINARY_OP_EXTEND_r22, + _GUARD_BINARY_OP_EXTEND_LHS_r02, + _GUARD_BINARY_OP_EXTEND_LHS_r12, + _GUARD_BINARY_OP_EXTEND_LHS_r22, + _GUARD_BINARY_OP_EXTEND_LHS_r33, + _GUARD_BINARY_OP_EXTEND_RHS_r02, + _GUARD_BINARY_OP_EXTEND_RHS_r12, + _GUARD_BINARY_OP_EXTEND_RHS_r22, + _GUARD_BINARY_OP_EXTEND_RHS_r33, + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02, + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12, + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r22, + _GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r33, + _GUARD_BIT_IS_SET_POP_r00, + _GUARD_BIT_IS_SET_POP_r10, + _GUARD_BIT_IS_SET_POP_r21, + _GUARD_BIT_IS_SET_POP_r32, + _GUARD_BIT_IS_SET_POP_4_r00, + _GUARD_BIT_IS_SET_POP_4_r10, + _GUARD_BIT_IS_SET_POP_4_r21, + _GUARD_BIT_IS_SET_POP_4_r32, + _GUARD_BIT_IS_SET_POP_5_r00, + _GUARD_BIT_IS_SET_POP_5_r10, + _GUARD_BIT_IS_SET_POP_5_r21, + _GUARD_BIT_IS_SET_POP_5_r32, + _GUARD_BIT_IS_SET_POP_6_r00, + _GUARD_BIT_IS_SET_POP_6_r10, + _GUARD_BIT_IS_SET_POP_6_r21, + _GUARD_BIT_IS_SET_POP_6_r32, + _GUARD_BIT_IS_SET_POP_7_r00, + _GUARD_BIT_IS_SET_POP_7_r10, + _GUARD_BIT_IS_SET_POP_7_r21, + _GUARD_BIT_IS_SET_POP_7_r32, + _GUARD_BIT_IS_UNSET_POP_r00, + _GUARD_BIT_IS_UNSET_POP_r10, + _GUARD_BIT_IS_UNSET_POP_r21, + _GUARD_BIT_IS_UNSET_POP_r32, + _GUARD_BIT_IS_UNSET_POP_4_r00, + _GUARD_BIT_IS_UNSET_POP_4_r10, + _GUARD_BIT_IS_UNSET_POP_4_r21, + _GUARD_BIT_IS_UNSET_POP_4_r32, + _GUARD_BIT_IS_UNSET_POP_5_r00, + _GUARD_BIT_IS_UNSET_POP_5_r10, + _GUARD_BIT_IS_UNSET_POP_5_r21, + _GUARD_BIT_IS_UNSET_POP_5_r32, + _GUARD_BIT_IS_UNSET_POP_6_r00, + _GUARD_BIT_IS_UNSET_POP_6_r10, + _GUARD_BIT_IS_UNSET_POP_6_r21, + _GUARD_BIT_IS_UNSET_POP_6_r32, + _GUARD_BIT_IS_UNSET_POP_7_r00, + _GUARD_BIT_IS_UNSET_POP_7_r10, + _GUARD_BIT_IS_UNSET_POP_7_r21, + _GUARD_BIT_IS_UNSET_POP_7_r32, + _GUARD_CALLABLE_BUILTIN_CLASS_r00, + _GUARD_CALLABLE_BUILTIN_FAST_r00, + _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00, + _GUARD_CALLABLE_BUILTIN_O_r00, + _GUARD_CALLABLE_ISINSTANCE_r03, + _GUARD_CALLABLE_ISINSTANCE_r13, + _GUARD_CALLABLE_ISINSTANCE_r23, + _GUARD_CALLABLE_ISINSTANCE_r33, + _GUARD_CALLABLE_LEN_r03, + _GUARD_CALLABLE_LEN_r13, + _GUARD_CALLABLE_LEN_r23, + _GUARD_CALLABLE_LEN_r33, + _GUARD_CALLABLE_LIST_APPEND_r03, + _GUARD_CALLABLE_LIST_APPEND_r13, + _GUARD_CALLABLE_LIST_APPEND_r23, + _GUARD_CALLABLE_LIST_APPEND_r33, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00, + _GUARD_CALLABLE_METHOD_DESCRIPTOR_O_r00, + _GUARD_CALLABLE_STR_1_r03, + _GUARD_CALLABLE_STR_1_r13, + _GUARD_CALLABLE_STR_1_r23, + _GUARD_CALLABLE_STR_1_r33, + _GUARD_CALLABLE_TUPLE_1_r03, + _GUARD_CALLABLE_TUPLE_1_r13, + _GUARD_CALLABLE_TUPLE_1_r23, + _GUARD_CALLABLE_TUPLE_1_r33, + _GUARD_CALLABLE_TYPE_1_r03, + _GUARD_CALLABLE_TYPE_1_r13, + _GUARD_CALLABLE_TYPE_1_r23, + _GUARD_CALLABLE_TYPE_1_r33, + _GUARD_CODE_VERSION_RETURN_GENERATOR_r00, + _GUARD_CODE_VERSION_RETURN_GENERATOR_r11, + _GUARD_CODE_VERSION_RETURN_GENERATOR_r22, + _GUARD_CODE_VERSION_RETURN_GENERATOR_r33, + _GUARD_CODE_VERSION_RETURN_VALUE_r00, + _GUARD_CODE_VERSION_RETURN_VALUE_r11, + _GUARD_CODE_VERSION_RETURN_VALUE_r22, + _GUARD_CODE_VERSION_RETURN_VALUE_r33, + _GUARD_CODE_VERSION_YIELD_VALUE_r00, + _GUARD_CODE_VERSION_YIELD_VALUE_r11, + _GUARD_CODE_VERSION_YIELD_VALUE_r22, + _GUARD_CODE_VERSION_YIELD_VALUE_r33, + _GUARD_CODE_VERSION__PUSH_FRAME_r00, + _GUARD_CODE_VERSION__PUSH_FRAME_r11, + _GUARD_CODE_VERSION__PUSH_FRAME_r22, + _GUARD_CODE_VERSION__PUSH_FRAME_r33, + _GUARD_DORV_NO_DICT_r01, + _GUARD_DORV_NO_DICT_r11, + _GUARD_DORV_NO_DICT_r22, + _GUARD_DORV_NO_DICT_r33, + _GUARD_GLOBALS_VERSION_r00, + _GUARD_GLOBALS_VERSION_r11, + _GUARD_GLOBALS_VERSION_r22, + _GUARD_GLOBALS_VERSION_r33, + _GUARD_IP_RETURN_GENERATOR_r00, + _GUARD_IP_RETURN_GENERATOR_r11, + _GUARD_IP_RETURN_GENERATOR_r22, + _GUARD_IP_RETURN_GENERATOR_r33, + _GUARD_IP_RETURN_VALUE_r00, + _GUARD_IP_RETURN_VALUE_r11, + _GUARD_IP_RETURN_VALUE_r22, + _GUARD_IP_RETURN_VALUE_r33, + _GUARD_IP_YIELD_VALUE_r00, + _GUARD_IP_YIELD_VALUE_r11, + _GUARD_IP_YIELD_VALUE_r22, + _GUARD_IP_YIELD_VALUE_r33, + _GUARD_IP__PUSH_FRAME_r00, + _GUARD_IP__PUSH_FRAME_r11, + _GUARD_IP__PUSH_FRAME_r22, + _GUARD_IP__PUSH_FRAME_r33, + _GUARD_IS_FALSE_POP_r00, + _GUARD_IS_FALSE_POP_r10, + _GUARD_IS_FALSE_POP_r21, + _GUARD_IS_FALSE_POP_r32, + _GUARD_IS_NONE_POP_r00, + _GUARD_IS_NONE_POP_r10, + _GUARD_IS_NONE_POP_r21, + _GUARD_IS_NONE_POP_r32, + _GUARD_IS_NOT_NONE_POP_r10, + _GUARD_IS_TRUE_POP_r00, + _GUARD_IS_TRUE_POP_r10, + _GUARD_IS_TRUE_POP_r21, + _GUARD_IS_TRUE_POP_r32, + _GUARD_ITERATOR_r01, + _GUARD_ITERATOR_r11, + _GUARD_ITERATOR_r22, + _GUARD_ITERATOR_r33, + _GUARD_ITER_VIRTUAL_r01, + _GUARD_ITER_VIRTUAL_r11, + _GUARD_ITER_VIRTUAL_r22, + _GUARD_ITER_VIRTUAL_r33, + _GUARD_LOAD_SUPER_ATTR_METHOD_r03, + _GUARD_LOAD_SUPER_ATTR_METHOD_r13, + _GUARD_LOAD_SUPER_ATTR_METHOD_r23, + _GUARD_LOAD_SUPER_ATTR_METHOD_r33, + _GUARD_NOS_COMPACT_ASCII_r02, + _GUARD_NOS_COMPACT_ASCII_r12, + _GUARD_NOS_COMPACT_ASCII_r22, + _GUARD_NOS_COMPACT_ASCII_r33, + _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03, + _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13, + _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23, + _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33, + _GUARD_NOS_DICT_SUBSCRIPT_r02, + _GUARD_NOS_DICT_SUBSCRIPT_r12, + _GUARD_NOS_DICT_SUBSCRIPT_r22, + _GUARD_NOS_DICT_SUBSCRIPT_r33, + _GUARD_NOS_FLOAT_r02, + _GUARD_NOS_FLOAT_r12, + _GUARD_NOS_FLOAT_r22, + _GUARD_NOS_FLOAT_r33, + _GUARD_NOS_INT_r02, + _GUARD_NOS_INT_r12, + _GUARD_NOS_INT_r22, + _GUARD_NOS_INT_r33, + _GUARD_NOS_ITER_VIRTUAL_r02, + _GUARD_NOS_ITER_VIRTUAL_r12, + _GUARD_NOS_ITER_VIRTUAL_r22, + _GUARD_NOS_ITER_VIRTUAL_r33, + _GUARD_NOS_LIST_r02, + _GUARD_NOS_LIST_r12, + _GUARD_NOS_LIST_r22, + _GUARD_NOS_LIST_r33, + _GUARD_NOS_NOT_NULL_r02, + _GUARD_NOS_NOT_NULL_r12, + _GUARD_NOS_NOT_NULL_r22, + _GUARD_NOS_NOT_NULL_r33, + _GUARD_NOS_NULL_r02, + _GUARD_NOS_NULL_r12, + _GUARD_NOS_NULL_r22, + _GUARD_NOS_NULL_r33, + _GUARD_NOS_OVERFLOWED_r02, + _GUARD_NOS_OVERFLOWED_r12, + _GUARD_NOS_OVERFLOWED_r22, + _GUARD_NOS_OVERFLOWED_r33, + _GUARD_NOS_TUPLE_r02, + _GUARD_NOS_TUPLE_r12, + _GUARD_NOS_TUPLE_r22, + _GUARD_NOS_TUPLE_r33, + _GUARD_NOS_TYPE_VERSION_r02, + _GUARD_NOS_TYPE_VERSION_r12, + _GUARD_NOS_TYPE_VERSION_r22, + _GUARD_NOS_TYPE_VERSION_r33, + _GUARD_NOS_UNICODE_r02, + _GUARD_NOS_UNICODE_r12, + _GUARD_NOS_UNICODE_r22, + _GUARD_NOS_UNICODE_r33, + _GUARD_NOT_EXHAUSTED_LIST_r02, + _GUARD_NOT_EXHAUSTED_LIST_r12, + _GUARD_NOT_EXHAUSTED_LIST_r22, + _GUARD_NOT_EXHAUSTED_LIST_r33, + _GUARD_NOT_EXHAUSTED_RANGE_r02, + _GUARD_NOT_EXHAUSTED_RANGE_r12, + _GUARD_NOT_EXHAUSTED_RANGE_r22, + _GUARD_NOT_EXHAUSTED_RANGE_r33, + _GUARD_NOT_EXHAUSTED_TUPLE_r02, + _GUARD_NOT_EXHAUSTED_TUPLE_r12, + _GUARD_NOT_EXHAUSTED_TUPLE_r22, + _GUARD_NOT_EXHAUSTED_TUPLE_r33, + _GUARD_THIRD_NULL_r03, + _GUARD_THIRD_NULL_r13, + _GUARD_THIRD_NULL_r23, + _GUARD_THIRD_NULL_r33, + _GUARD_TOS_ANY_DICT_r01, + _GUARD_TOS_ANY_DICT_r11, + _GUARD_TOS_ANY_DICT_r22, + _GUARD_TOS_ANY_DICT_r33, + _GUARD_TOS_ANY_SET_r01, + _GUARD_TOS_ANY_SET_r11, + _GUARD_TOS_ANY_SET_r22, + _GUARD_TOS_ANY_SET_r33, + _GUARD_TOS_DICT_r01, + _GUARD_TOS_DICT_r11, + _GUARD_TOS_DICT_r22, + _GUARD_TOS_DICT_r33, + _GUARD_TOS_EXACT_INT_r01, + _GUARD_TOS_EXACT_INT_r11, + _GUARD_TOS_EXACT_INT_r22, + _GUARD_TOS_EXACT_INT_r33, + _GUARD_TOS_FLOAT_r01, + _GUARD_TOS_FLOAT_r11, + _GUARD_TOS_FLOAT_r22, + _GUARD_TOS_FLOAT_r33, + _GUARD_TOS_FROZENDICT_r01, + _GUARD_TOS_FROZENDICT_r11, + _GUARD_TOS_FROZENDICT_r22, + _GUARD_TOS_FROZENDICT_r33, + _GUARD_TOS_FROZENSET_r01, + _GUARD_TOS_FROZENSET_r11, + _GUARD_TOS_FROZENSET_r22, + _GUARD_TOS_FROZENSET_r33, + _GUARD_TOS_INT_r01, + _GUARD_TOS_INT_r11, + _GUARD_TOS_INT_r22, + _GUARD_TOS_INT_r33, + _GUARD_TOS_IS_NONE_r01, + _GUARD_TOS_IS_NONE_r11, + _GUARD_TOS_IS_NONE_r22, + _GUARD_TOS_IS_NONE_r33, + _GUARD_TOS_LIST_r01, + _GUARD_TOS_LIST_r11, + _GUARD_TOS_LIST_r22, + _GUARD_TOS_LIST_r33, + _GUARD_TOS_NOT_NULL_r01, + _GUARD_TOS_NOT_NULL_r11, + _GUARD_TOS_NOT_NULL_r22, + _GUARD_TOS_NOT_NULL_r33, + _GUARD_TOS_OVERFLOWED_r01, + _GUARD_TOS_OVERFLOWED_r11, + _GUARD_TOS_OVERFLOWED_r22, + _GUARD_TOS_OVERFLOWED_r33, + _GUARD_TOS_SET_r01, + _GUARD_TOS_SET_r11, + _GUARD_TOS_SET_r22, + _GUARD_TOS_SET_r33, + _GUARD_TOS_SLICE_r01, + _GUARD_TOS_SLICE_r11, + _GUARD_TOS_SLICE_r22, + _GUARD_TOS_SLICE_r33, + _GUARD_TOS_TUPLE_r01, + _GUARD_TOS_TUPLE_r11, + _GUARD_TOS_TUPLE_r22, + _GUARD_TOS_TUPLE_r33, + _GUARD_TOS_UNICODE_r01, + _GUARD_TOS_UNICODE_r11, + _GUARD_TOS_UNICODE_r22, + _GUARD_TOS_UNICODE_r33, + _GUARD_TYPE_r01, + _GUARD_TYPE_r11, + _GUARD_TYPE_r22, + _GUARD_TYPE_r33, + _GUARD_TYPE_ITER_r02, + _GUARD_TYPE_ITER_r12, + _GUARD_TYPE_ITER_r22, + _GUARD_TYPE_ITER_r33, + _GUARD_TYPE_VERSION_r01, + _GUARD_TYPE_VERSION_r11, + _GUARD_TYPE_VERSION_r22, + _GUARD_TYPE_VERSION_r33, + _GUARD_TYPE_VERSION_LOCKED_r01, + _GUARD_TYPE_VERSION_LOCKED_r11, + _GUARD_TYPE_VERSION_LOCKED_r22, + _GUARD_TYPE_VERSION_LOCKED_r33, + _HANDLE_PENDING_AND_DEOPT_r00, + _HANDLE_PENDING_AND_DEOPT_r10, + _HANDLE_PENDING_AND_DEOPT_r20, + _HANDLE_PENDING_AND_DEOPT_r30, + _IMPORT_FROM_r12, + _IMPORT_NAME_r21, + _INIT_CALL_BOUND_METHOD_EXACT_ARGS_r00, + _INIT_CALL_PY_EXACT_ARGS_r01, + _INIT_CALL_PY_EXACT_ARGS_0_r01, + _INIT_CALL_PY_EXACT_ARGS_1_r01, + _INIT_CALL_PY_EXACT_ARGS_2_r01, + _INIT_CALL_PY_EXACT_ARGS_3_r01, + _INIT_CALL_PY_EXACT_ARGS_4_r01, + _INSERT_NULL_r10, + _INSTRUMENTED_FOR_ITER_r23, + _INSTRUMENTED_INSTRUCTION_r00, + _INSTRUMENTED_JUMP_FORWARD_r00, + _INSTRUMENTED_JUMP_FORWARD_r11, + _INSTRUMENTED_JUMP_FORWARD_r22, + _INSTRUMENTED_JUMP_FORWARD_r33, + _INSTRUMENTED_LINE_r00, + _INSTRUMENTED_NOT_TAKEN_r00, + _INSTRUMENTED_NOT_TAKEN_r11, + _INSTRUMENTED_NOT_TAKEN_r22, + _INSTRUMENTED_NOT_TAKEN_r33, + _INSTRUMENTED_POP_JUMP_IF_FALSE_r00, + _INSTRUMENTED_POP_JUMP_IF_FALSE_r10, + _INSTRUMENTED_POP_JUMP_IF_FALSE_r21, + _INSTRUMENTED_POP_JUMP_IF_FALSE_r32, + _INSTRUMENTED_POP_JUMP_IF_NONE_r10, + _INSTRUMENTED_POP_JUMP_IF_NOT_NONE_r10, + _INSTRUMENTED_POP_JUMP_IF_TRUE_r00, + _INSTRUMENTED_POP_JUMP_IF_TRUE_r10, + _INSTRUMENTED_POP_JUMP_IF_TRUE_r21, + _INSTRUMENTED_POP_JUMP_IF_TRUE_r32, + _IS_NONE_r11, + _IS_OP_r03, + _IS_OP_r13, + _IS_OP_r23, + _ITER_CHECK_LIST_r02, + _ITER_CHECK_LIST_r12, + _ITER_CHECK_LIST_r22, + _ITER_CHECK_LIST_r33, + _ITER_CHECK_RANGE_r02, + _ITER_CHECK_RANGE_r12, + _ITER_CHECK_RANGE_r22, + _ITER_CHECK_RANGE_r33, + _ITER_CHECK_TUPLE_r02, + _ITER_CHECK_TUPLE_r12, + _ITER_CHECK_TUPLE_r22, + _ITER_CHECK_TUPLE_r33, + _ITER_JUMP_LIST_r02, + _ITER_JUMP_LIST_r12, + _ITER_JUMP_LIST_r22, + _ITER_JUMP_LIST_r33, + _ITER_JUMP_RANGE_r02, + _ITER_JUMP_RANGE_r12, + _ITER_JUMP_RANGE_r22, + _ITER_JUMP_RANGE_r33, + _ITER_JUMP_TUPLE_r02, + _ITER_JUMP_TUPLE_r12, + _ITER_JUMP_TUPLE_r22, + _ITER_JUMP_TUPLE_r33, + _ITER_NEXT_INLINE_r23, + _ITER_NEXT_LIST_r23, + _ITER_NEXT_LIST_TIER_TWO_r23, + _ITER_NEXT_RANGE_r03, + _ITER_NEXT_RANGE_r13, + _ITER_NEXT_RANGE_r23, + _ITER_NEXT_TUPLE_r03, + _ITER_NEXT_TUPLE_r13, + _ITER_NEXT_TUPLE_r23, + _JUMP_BACKWARD_NO_INTERRUPT_r00, + _JUMP_BACKWARD_NO_INTERRUPT_r11, + _JUMP_BACKWARD_NO_INTERRUPT_r22, + _JUMP_BACKWARD_NO_INTERRUPT_r33, + _JUMP_TO_TOP_r00, + _LIST_APPEND_r10, + _LIST_EXTEND_r11, + _LOAD_ATTR_r10, + _LOAD_ATTR_CLASS_r11, + _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11, + _LOAD_ATTR_INSTANCE_VALUE_r02, + _LOAD_ATTR_INSTANCE_VALUE_r12, + _LOAD_ATTR_INSTANCE_VALUE_r23, + _LOAD_ATTR_METHOD_LAZY_DICT_r02, + _LOAD_ATTR_METHOD_LAZY_DICT_r12, + _LOAD_ATTR_METHOD_LAZY_DICT_r23, + _LOAD_ATTR_METHOD_NO_DICT_r02, + _LOAD_ATTR_METHOD_NO_DICT_r12, + _LOAD_ATTR_METHOD_NO_DICT_r23, + _LOAD_ATTR_METHOD_WITH_VALUES_r02, + _LOAD_ATTR_METHOD_WITH_VALUES_r12, + _LOAD_ATTR_METHOD_WITH_VALUES_r23, + _LOAD_ATTR_MODULE_r12, + _LOAD_ATTR_NONDESCRIPTOR_NO_DICT_r11, + _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11, + _LOAD_ATTR_PROPERTY_FRAME_r01, + _LOAD_ATTR_PROPERTY_FRAME_r11, + _LOAD_ATTR_PROPERTY_FRAME_r22, + _LOAD_ATTR_PROPERTY_FRAME_r33, + _LOAD_ATTR_SLOT_r02, + _LOAD_ATTR_SLOT_r12, + _LOAD_ATTR_SLOT_r23, + _LOAD_ATTR_WITH_HINT_r12, + _LOAD_BUILD_CLASS_r01, + _LOAD_BYTECODE_r00, + _LOAD_COMMON_CONSTANT_r01, + _LOAD_COMMON_CONSTANT_r12, + _LOAD_COMMON_CONSTANT_r23, + _LOAD_CONST_r01, + _LOAD_CONST_r12, + _LOAD_CONST_r23, + _LOAD_CONST_INLINE_r01, + _LOAD_CONST_INLINE_r12, + _LOAD_CONST_INLINE_r23, + _LOAD_CONST_INLINE_BORROW_r01, + _LOAD_CONST_INLINE_BORROW_r12, + _LOAD_CONST_INLINE_BORROW_r23, + _LOAD_DEREF_r01, + _LOAD_FAST_r01, + _LOAD_FAST_r12, + _LOAD_FAST_r23, + _LOAD_FAST_0_r01, + _LOAD_FAST_0_r12, + _LOAD_FAST_0_r23, + _LOAD_FAST_1_r01, + _LOAD_FAST_1_r12, + _LOAD_FAST_1_r23, + _LOAD_FAST_2_r01, + _LOAD_FAST_2_r12, + _LOAD_FAST_2_r23, + _LOAD_FAST_3_r01, + _LOAD_FAST_3_r12, + _LOAD_FAST_3_r23, + _LOAD_FAST_4_r01, + _LOAD_FAST_4_r12, + _LOAD_FAST_4_r23, + _LOAD_FAST_5_r01, + _LOAD_FAST_5_r12, + _LOAD_FAST_5_r23, + _LOAD_FAST_6_r01, + _LOAD_FAST_6_r12, + _LOAD_FAST_6_r23, + _LOAD_FAST_7_r01, + _LOAD_FAST_7_r12, + _LOAD_FAST_7_r23, + _LOAD_FAST_AND_CLEAR_r01, + _LOAD_FAST_AND_CLEAR_r12, + _LOAD_FAST_AND_CLEAR_r23, + _LOAD_FAST_BORROW_r01, + _LOAD_FAST_BORROW_r12, + _LOAD_FAST_BORROW_r23, + _LOAD_FAST_BORROW_0_r01, + _LOAD_FAST_BORROW_0_r12, + _LOAD_FAST_BORROW_0_r23, + _LOAD_FAST_BORROW_1_r01, + _LOAD_FAST_BORROW_1_r12, + _LOAD_FAST_BORROW_1_r23, + _LOAD_FAST_BORROW_2_r01, + _LOAD_FAST_BORROW_2_r12, + _LOAD_FAST_BORROW_2_r23, + _LOAD_FAST_BORROW_3_r01, + _LOAD_FAST_BORROW_3_r12, + _LOAD_FAST_BORROW_3_r23, + _LOAD_FAST_BORROW_4_r01, + _LOAD_FAST_BORROW_4_r12, + _LOAD_FAST_BORROW_4_r23, + _LOAD_FAST_BORROW_5_r01, + _LOAD_FAST_BORROW_5_r12, + _LOAD_FAST_BORROW_5_r23, + _LOAD_FAST_BORROW_6_r01, + _LOAD_FAST_BORROW_6_r12, + _LOAD_FAST_BORROW_6_r23, + _LOAD_FAST_BORROW_7_r01, + _LOAD_FAST_BORROW_7_r12, + _LOAD_FAST_BORROW_7_r23, + _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02, + _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13, + _LOAD_FAST_CHECK_r01, + _LOAD_FAST_CHECK_r12, + _LOAD_FAST_CHECK_r23, + _LOAD_FAST_LOAD_FAST_r02, + _LOAD_FAST_LOAD_FAST_r13, + _LOAD_FROM_DICT_OR_DEREF_r11, + _LOAD_FROM_DICT_OR_GLOBALS_r11, + _LOAD_GLOBAL_r00, + _LOAD_GLOBAL_BUILTINS_r01, + _LOAD_GLOBAL_MODULE_r01, + _LOAD_LOCALS_r01, + _LOAD_LOCALS_r12, + _LOAD_LOCALS_r23, + _LOAD_NAME_r01, + _LOAD_SMALL_INT_r01, + _LOAD_SMALL_INT_r12, + _LOAD_SMALL_INT_r23, + _LOAD_SMALL_INT_0_r01, + _LOAD_SMALL_INT_0_r12, + _LOAD_SMALL_INT_0_r23, + _LOAD_SMALL_INT_1_r01, + _LOAD_SMALL_INT_1_r12, + _LOAD_SMALL_INT_1_r23, + _LOAD_SMALL_INT_2_r01, + _LOAD_SMALL_INT_2_r12, + _LOAD_SMALL_INT_2_r23, + _LOAD_SMALL_INT_3_r01, + _LOAD_SMALL_INT_3_r12, + _LOAD_SMALL_INT_3_r23, + _LOAD_SPECIAL_r00, + _LOAD_SUPER_ATTR_ATTR_r31, + _LOAD_SUPER_ATTR_METHOD_r32, + _LOCK_OBJECT_r01, + _LOCK_OBJECT_r11, + _LOCK_OBJECT_r22, + _LOCK_OBJECT_r33, + _MAKE_CALLARGS_A_TUPLE_r33, + _MAKE_CELL_r00, + _MAKE_FUNCTION_r12, + _MAKE_HEAP_SAFE_r01, + _MAKE_HEAP_SAFE_r11, + _MAKE_HEAP_SAFE_r22, + _MAKE_HEAP_SAFE_r33, + _MAKE_WARM_r00, + _MAKE_WARM_r11, + _MAKE_WARM_r22, + _MAKE_WARM_r33, + _MAP_ADD_r20, + _MATCH_CLASS_r33, + _MATCH_KEYS_r23, + _MATCH_MAPPING_r02, + _MATCH_MAPPING_r12, + _MATCH_MAPPING_r23, + _MATCH_SEQUENCE_r02, + _MATCH_SEQUENCE_r12, + _MATCH_SEQUENCE_r23, + _MAYBE_EXPAND_METHOD_r00, + _MAYBE_EXPAND_METHOD_KW_r11, + _MONITOR_CALL_r00, + _MONITOR_CALL_KW_r11, + _MONITOR_JUMP_BACKWARD_r00, + _MONITOR_JUMP_BACKWARD_r11, + _MONITOR_JUMP_BACKWARD_r22, + _MONITOR_JUMP_BACKWARD_r33, + _MONITOR_RESUME_r00, + _NOP_r00, + _NOP_r11, + _NOP_r22, + _NOP_r33, + _POP_EXCEPT_r10, + _POP_ITER_r20, + _POP_JUMP_IF_FALSE_r00, + _POP_JUMP_IF_FALSE_r10, + _POP_JUMP_IF_FALSE_r21, + _POP_JUMP_IF_FALSE_r32, + _POP_JUMP_IF_TRUE_r00, + _POP_JUMP_IF_TRUE_r10, + _POP_JUMP_IF_TRUE_r21, + _POP_JUMP_IF_TRUE_r32, + _POP_TOP_r10, + _POP_TOP_FLOAT_r00, + _POP_TOP_FLOAT_r10, + _POP_TOP_FLOAT_r21, + _POP_TOP_FLOAT_r32, + _POP_TOP_INT_r00, + _POP_TOP_INT_r10, + _POP_TOP_INT_r21, + _POP_TOP_INT_r32, + _POP_TOP_NOP_r00, + _POP_TOP_NOP_r10, + _POP_TOP_NOP_r21, + _POP_TOP_NOP_r32, + _POP_TOP_OPARG_r00, + _POP_TOP_UNICODE_r00, + _POP_TOP_UNICODE_r10, + _POP_TOP_UNICODE_r21, + _POP_TOP_UNICODE_r32, + _PUSH_EXC_INFO_r02, + _PUSH_EXC_INFO_r12, + _PUSH_EXC_INFO_r23, + _PUSH_FRAME_r10, + _PUSH_NULL_r01, + _PUSH_NULL_r12, + _PUSH_NULL_r23, + _PUSH_NULL_CONDITIONAL_r00, + _PUSH_TAGGED_ZERO_r01, + _PUSH_TAGGED_ZERO_r12, + _PUSH_TAGGED_ZERO_r23, + _PY_FRAME_EX_r31, + _PY_FRAME_GENERAL_r01, + _PY_FRAME_KW_r11, + _REPLACE_WITH_TRUE_r02, + _REPLACE_WITH_TRUE_r12, + _REPLACE_WITH_TRUE_r23, + _RESUME_CHECK_r00, + _RESUME_CHECK_r11, + _RESUME_CHECK_r22, + _RESUME_CHECK_r33, + _RETURN_GENERATOR_r01, + _RETURN_VALUE_r11, + _RROT_3_r03, + _RROT_3_r13, + _RROT_3_r23, + _RROT_3_r33, + _SAVE_RETURN_OFFSET_r00, + _SAVE_RETURN_OFFSET_r11, + _SAVE_RETURN_OFFSET_r22, + _SAVE_RETURN_OFFSET_r33, + _SEND_ASYNC_GEN_r33, + _SEND_ASYNC_GEN_TIER_TWO_r33, + _SEND_GEN_FRAME_r33, + _SEND_VIRTUAL_r33, + _SEND_VIRTUAL_TIER_TWO_r03, + _SEND_VIRTUAL_TIER_TWO_r13, + _SEND_VIRTUAL_TIER_TWO_r23, + _SEND_VIRTUAL_TIER_TWO_r33, + _SETUP_ANNOTATIONS_r00, + _SET_ADD_r10, + _SET_FUNCTION_ATTRIBUTE_r01, + _SET_FUNCTION_ATTRIBUTE_r11, + _SET_FUNCTION_ATTRIBUTE_r21, + _SET_FUNCTION_ATTRIBUTE_r32, + _SET_IP_r00, + _SET_IP_r11, + _SET_IP_r22, + _SET_IP_r33, + _SET_UPDATE_r11, + _SPILL_OR_RELOAD_r01, + _SPILL_OR_RELOAD_r02, + _SPILL_OR_RELOAD_r03, + _SPILL_OR_RELOAD_r10, + _SPILL_OR_RELOAD_r12, + _SPILL_OR_RELOAD_r13, + _SPILL_OR_RELOAD_r20, + _SPILL_OR_RELOAD_r21, + _SPILL_OR_RELOAD_r23, + _SPILL_OR_RELOAD_r30, + _SPILL_OR_RELOAD_r31, + _SPILL_OR_RELOAD_r32, + _START_EXECUTOR_r00, + _STORE_ATTR_r20, + _STORE_ATTR_INSTANCE_VALUE_r21, + _STORE_ATTR_SLOT_r21, + _STORE_ATTR_WITH_HINT_r21, + _STORE_DEREF_r10, + _STORE_FAST_LOAD_FAST_r11, + _STORE_FAST_STORE_FAST_r20, + _STORE_GLOBAL_r10, + _STORE_NAME_r10, + _STORE_SLICE_r30, + _STORE_SUBSCR_r30, + _STORE_SUBSCR_DICT_r31, + _STORE_SUBSCR_DICT_KNOWN_HASH_r31, + _STORE_SUBSCR_LIST_INT_r32, + _SWAP_r11, + _SWAP_2_r02, + _SWAP_2_r12, + _SWAP_2_r22, + _SWAP_2_r33, + _SWAP_3_r03, + _SWAP_3_r13, + _SWAP_3_r23, + _SWAP_3_r33, + _SWAP_FAST_r01, + _SWAP_FAST_r11, + _SWAP_FAST_r22, + _SWAP_FAST_r33, + _SWAP_FAST_0_r01, + _SWAP_FAST_0_r11, + _SWAP_FAST_0_r22, + _SWAP_FAST_0_r33, + _SWAP_FAST_1_r01, + _SWAP_FAST_1_r11, + _SWAP_FAST_1_r22, + _SWAP_FAST_1_r33, + _SWAP_FAST_2_r01, + _SWAP_FAST_2_r11, + _SWAP_FAST_2_r22, + _SWAP_FAST_2_r33, + _SWAP_FAST_3_r01, + _SWAP_FAST_3_r11, + _SWAP_FAST_3_r22, + _SWAP_FAST_3_r33, + _SWAP_FAST_4_r01, + _SWAP_FAST_4_r11, + _SWAP_FAST_4_r22, + _SWAP_FAST_4_r33, + _SWAP_FAST_5_r01, + _SWAP_FAST_5_r11, + _SWAP_FAST_5_r22, + _SWAP_FAST_5_r33, + _SWAP_FAST_6_r01, + _SWAP_FAST_6_r11, + _SWAP_FAST_6_r22, + _SWAP_FAST_6_r33, + _SWAP_FAST_7_r01, + _SWAP_FAST_7_r11, + _SWAP_FAST_7_r22, + _SWAP_FAST_7_r33, + _TIER2_RESUME_CHECK_r00, + _TIER2_RESUME_CHECK_r11, + _TIER2_RESUME_CHECK_r22, + _TIER2_RESUME_CHECK_r33, + _TO_BOOL_r11, + _TO_BOOL_BOOL_r01, + _TO_BOOL_BOOL_r11, + _TO_BOOL_BOOL_r22, + _TO_BOOL_BOOL_r33, + _TO_BOOL_INT_r02, + _TO_BOOL_INT_r12, + _TO_BOOL_INT_r23, + _TO_BOOL_LIST_r02, + _TO_BOOL_LIST_r12, + _TO_BOOL_LIST_r23, + _TO_BOOL_NONE_r01, + _TO_BOOL_NONE_r11, + _TO_BOOL_NONE_r22, + _TO_BOOL_NONE_r33, + _TO_BOOL_STR_r02, + _TO_BOOL_STR_r12, + _TO_BOOL_STR_r23, + _TRACE_RECORD_r00, + _UNARY_INVERT_r12, + _UNARY_NEGATIVE_r12, + _UNARY_NEGATIVE_FLOAT_INPLACE_r02, + _UNARY_NEGATIVE_FLOAT_INPLACE_r12, + _UNARY_NEGATIVE_FLOAT_INPLACE_r23, + _UNARY_NOT_r01, + _UNARY_NOT_r11, + _UNARY_NOT_r22, + _UNARY_NOT_r33, + _UNPACK_EX_r10, + _UNPACK_SEQUENCE_r10, + _UNPACK_SEQUENCE_LIST_r10, + _UNPACK_SEQUENCE_TUPLE_r10, + _UNPACK_SEQUENCE_TWO_TUPLE_r12, + _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03, + _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13, + _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10, + _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02, + _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12, + _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23, + _WITH_EXCEPT_START_r33, + _YIELD_VALUE_r11, +}; +#define MAX_UOP_REGS_ID 1650 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 2f2729419052dad..f464effe23bc0b3 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -79,7 +79,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_POP_TOP_INT] = 0, [_POP_TOP_FLOAT] = 0, [_POP_TOP_UNICODE] = 0, - [_POP_TWO] = HAS_ESCAPES_FLAG, + [_POP_TOP_OPARG] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PUSH_NULL] = HAS_PURE_FLAG, [_END_FOR] = HAS_ESCAPES_FLAG | HAS_NO_SAVE_IP_FLAG, [_POP_ITER] = HAS_ESCAPES_FLAG, @@ -103,6 +103,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_NOS_INT] = HAS_EXIT_FLAG, [_GUARD_TOS_INT] = HAS_EXIT_FLAG, + [_GUARD_TOS_EXACT_INT] = HAS_EXIT_FLAG, [_GUARD_NOS_OVERFLOWED] = HAS_EXIT_FLAG, [_GUARD_TOS_OVERFLOWED] = HAS_EXIT_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_EXIT_FLAG | HAS_PURE_FLAG, @@ -125,8 +126,13 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_ADD_FLOAT_INPLACE_RIGHT] = 0, [_BINARY_OP_MULTIPLY_FLOAT_INPLACE_RIGHT] = 0, [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT] = 0, + [_BINARY_OP_TRUEDIV_FLOAT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_BINARY_OP_EXTEND_LHS] = HAS_EXIT_FLAG, + [_GUARD_BINARY_OP_EXTEND_RHS] = HAS_EXIT_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_EXTEND] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -139,8 +145,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_TOS_TUPLE] = HAS_EXIT_FLAG, [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = HAS_EXIT_FLAG, [_BINARY_OP_SUBSCR_TUPLE_INT] = 0, - [_GUARD_NOS_DICT] = HAS_EXIT_FLAG, - [_GUARD_NOS_ANY_DICT] = HAS_EXIT_FLAG, + [_GUARD_NOS_DICT_SUBSCRIPT] = HAS_DEOPT_FLAG, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = HAS_DEOPT_FLAG, [_GUARD_TOS_ANY_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_DICT] = HAS_EXIT_FLAG, [_GUARD_TOS_FROZENDICT] = HAS_EXIT_FLAG, @@ -158,17 +164,21 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_INTRINSIC_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_INTRINSIC_2] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_HEAP_SAFE] = 0, - [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_VALUE] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_GUARD_TOS_IS_NONE] = HAS_EXIT_FLAG, + [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, + [_SEND_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG, + [_GUARD_3OS_ASYNC_GEN_ASEND] = HAS_EXIT_FLAG, + [_SEND_ASYNC_GEN_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE] = 0, @@ -178,9 +188,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_DELETE_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_LOCALS] = HAS_ERROR_FLAG, [_LOAD_NAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_LOAD_GLOBAL] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -192,7 +200,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MAKE_CELL] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_DELETE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_LOAD_FROM_DICT_OR_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_LOAD_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_DEREF] = HAS_ARG_FLAG | HAS_FREE_FLAG | HAS_ESCAPES_FLAG, [_COPY_FREE_VARS] = HAS_ARG_FLAG, [_BUILD_STRING] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -215,6 +223,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_TYPE_VERSION] = HAS_EXIT_FLAG, [_GUARD_TYPE_VERSION_LOCKED] = HAS_EXIT_FLAG, + [_GUARD_TYPE] = HAS_EXIT_FLAG, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = HAS_EXIT_FLAG, [_LOAD_ATTR_INSTANCE_VALUE] = HAS_DEOPT_FLAG, [_LOAD_ATTR_MODULE] = HAS_EXIT_FLAG, @@ -223,6 +232,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_ATTR_CLASS] = HAS_EXIT_FLAG, [_LOAD_ATTR_CLASS] = HAS_ESCAPES_FLAG, [_LOAD_ATTR_PROPERTY_FRAME] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_EXIT_FLAG, [_GUARD_DORV_NO_DICT] = HAS_EXIT_FLAG, [_STORE_ATTR_INSTANCE_VALUE] = HAS_ESCAPES_FLAG, [_LOCK_OBJECT] = HAS_DEOPT_FLAG, @@ -250,7 +260,16 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_MATCH_SEQUENCE] = 0, [_MATCH_KEYS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ITER] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_ITERATOR] = HAS_EXIT_FLAG, + [_GUARD_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_PUSH_TAGGED_ZERO] = 0, + [_GET_ITER_TRAD] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FOR_ITER_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_TYPE_ITER] = HAS_EXIT_FLAG, + [_ITER_NEXT_INLINE] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_NOS_ITER_VIRTUAL] = HAS_EXIT_FLAG, + [_GUARD_TOS_NOT_NULL] = HAS_EXIT_FLAG, + [_FOR_ITER_VIRTUAL_TIER_TWO] = HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_ITER_CHECK_LIST] = HAS_EXIT_FLAG, [_GUARD_NOT_EXHAUSTED_LIST] = HAS_EXIT_FLAG, [_ITER_NEXT_LIST_TIER_TWO] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -265,8 +284,6 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_SPECIAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_WITH_EXCEPT_START] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_PUSH_EXC_INFO] = 0, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = HAS_EXIT_FLAG, - [_GUARD_KEYS_VERSION] = HAS_EXIT_FLAG, [_LOAD_ATTR_METHOD_WITH_VALUES] = HAS_ARG_FLAG, [_LOAD_ATTR_METHOD_NO_DICT] = HAS_ARG_FLAG, [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, @@ -295,7 +312,6 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_GUARD_NOS_NULL] = HAS_EXIT_FLAG, - [_GUARD_NOS_NOT_NULL] = HAS_EXIT_FLAG, [_GUARD_THIRD_NULL] = HAS_EXIT_FLAG, [_GUARD_CALLABLE_TYPE_1] = HAS_EXIT_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG, @@ -303,16 +319,18 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_STR_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_TUPLE_1] = HAS_EXIT_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CHECK_AND_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_ALLOCATE_OBJECT] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CREATE_INIT_FRAME] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_EXIT_INIT_CHECK] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, - [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_GUARD_CALLABLE_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_BUILTIN_CLASS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_O] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_LEN] = HAS_EXIT_FLAG, [_CALL_LEN] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_ISINSTANCE] = HAS_EXIT_FLAG, @@ -324,14 +342,14 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_RECURSION_LIMIT] = HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_O_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_EXIT_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAYBE_EXPAND_METHOD_KW] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_PY_FRAME_KW] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_FUNCTION_VERSION_KW] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -344,9 +362,9 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_PY_FRAME_EX] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_CHECK_IS_NOT_PY_CALLABLE_EX] = HAS_EXIT_FLAG, [_CALL_FUNCTION_EX_NON_PY_GENERAL] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, - [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_NEEDS_GUARD_IP_FLAG, + [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG | HAS_NEEDS_GUARD_IP_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_FORMAT_SIMPLE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -381,21 +399,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_POP_TOP_LOAD_CONST_INLINE] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_POP_CALL] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE] = HAS_ESCAPES_FLAG, - [_POP_CALL_TWO] = HAS_ESCAPES_FLAG, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, - [_INSERT_1_LOAD_CONST_INLINE] = 0, - [_INSERT_1_LOAD_CONST_INLINE_BORROW] = 0, - [_INSERT_2_LOAD_CONST_INLINE_BORROW] = 0, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = 0, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = 0, - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, + [_RROT_3] = HAS_PURE_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, @@ -404,7 +409,7 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_ERROR_POP_N] = HAS_ARG_FLAG | HAS_SYNC_SP_FLAG, [_SPILL_OR_RELOAD] = 0, [_TIER2_RESUME_CHECK] = HAS_PERIODIC_FLAG, - [_COLD_EXIT] = HAS_SYNC_SP_FLAG, + [_COLD_EXIT] = HAS_ESCAPES_FLAG | HAS_SYNC_SP_FLAG, [_COLD_DYNAMIC_EXIT] = HAS_SYNC_SP_FLAG, [_GUARD_CODE_VERSION__PUSH_FRAME] = HAS_EXIT_FLAG, [_GUARD_CODE_VERSION_YIELD_VALUE] = HAS_EXIT_FLAG, @@ -417,10 +422,12 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_RECORD_TOS] = HAS_RECORDS_VALUE_FLAG, [_RECORD_TOS_TYPE] = HAS_RECORDS_VALUE_FLAG, [_RECORD_NOS] = HAS_RECORDS_VALUE_FLAG, + [_RECORD_NOS_TYPE] = HAS_RECORDS_VALUE_FLAG, [_RECORD_NOS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, [_RECORD_3OS_GEN_FUNC] = HAS_RECORDS_VALUE_FLAG, [_RECORD_4OS] = HAS_RECORDS_VALUE_FLAG, [_RECORD_CALLABLE] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, + [_RECORD_CALLABLE_KW] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, [_RECORD_BOUND_METHOD] = HAS_ARG_FLAG | HAS_RECORDS_VALUE_FLAG, [_RECORD_CODE] = HAS_RECORDS_VALUE_FLAG, }; @@ -834,12 +841,12 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 2, 3, _POP_TOP_UNICODE_r32 }, }, }, - [_POP_TWO] = { - .best = { 2, 2, 2, 2 }, + [_POP_TOP_OPARG] = { + .best = { 0, 0, 0, 0 }, .entries = { + { 0, 0, _POP_TOP_OPARG_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, - { 0, 2, _POP_TWO_r20 }, { -1, -1, -1 }, }, }, @@ -1050,6 +1057,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TOS_INT_r33 }, }, }, + [_GUARD_TOS_EXACT_INT] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_EXACT_INT_r01 }, + { 1, 1, _GUARD_TOS_EXACT_INT_r11 }, + { 2, 2, _GUARD_TOS_EXACT_INT_r22 }, + { 3, 3, _GUARD_TOS_EXACT_INT_r33 }, + }, + }, [_GUARD_NOS_OVERFLOWED] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -1248,6 +1264,33 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_BINARY_OP_TRUEDIV_FLOAT] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23 }, + { -1, -1, -1 }, + }, + }, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 3, 0, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03 }, + { 3, 1, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13 }, + { 3, 2, _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23 }, + { -1, -1, -1 }, + }, + }, [_BINARY_OP_ADD_UNICODE] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -1266,6 +1309,24 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_BINARY_OP_EXTEND_LHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_LHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_LHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_LHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_LHS_r33 }, + }, + }, + [_GUARD_BINARY_OP_EXTEND_RHS] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_BINARY_OP_EXTEND_RHS_r02 }, + { 2, 1, _GUARD_BINARY_OP_EXTEND_RHS_r12 }, + { 2, 2, _GUARD_BINARY_OP_EXTEND_RHS_r22 }, + { 3, 3, _GUARD_BINARY_OP_EXTEND_RHS_r33 }, + }, + }, [_GUARD_BINARY_OP_EXTEND] = { .best = { 2, 2, 2, 2 }, .entries = { @@ -1374,22 +1435,22 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_GUARD_NOS_DICT] = { + [_GUARD_NOS_DICT_SUBSCRIPT] = { .best = { 0, 1, 2, 3 }, .entries = { - { 2, 0, _GUARD_NOS_DICT_r02 }, - { 2, 1, _GUARD_NOS_DICT_r12 }, - { 2, 2, _GUARD_NOS_DICT_r22 }, - { 3, 3, _GUARD_NOS_DICT_r33 }, + { 2, 0, _GUARD_NOS_DICT_SUBSCRIPT_r02 }, + { 2, 1, _GUARD_NOS_DICT_SUBSCRIPT_r12 }, + { 2, 2, _GUARD_NOS_DICT_SUBSCRIPT_r22 }, + { 3, 3, _GUARD_NOS_DICT_SUBSCRIPT_r33 }, }, }, - [_GUARD_NOS_ANY_DICT] = { + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = { .best = { 0, 1, 2, 3 }, .entries = { - { 2, 0, _GUARD_NOS_ANY_DICT_r02 }, - { 2, 1, _GUARD_NOS_ANY_DICT_r12 }, - { 2, 2, _GUARD_NOS_ANY_DICT_r22 }, - { 3, 3, _GUARD_NOS_ANY_DICT_r33 }, + { 3, 0, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r03 }, + { 3, 1, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r13 }, + { 3, 2, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r23 }, + { 3, 3, _GUARD_NOS_DICT_STORE_SUBSCRIPT_r33 }, }, }, [_GUARD_TOS_ANY_DICT] = { @@ -1590,6 +1651,51 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _SEND_GEN_FRAME_r33 }, }, }, + [_GUARD_TOS_IS_NONE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_IS_NONE_r01 }, + { 1, 1, _GUARD_TOS_IS_NONE_r11 }, + { 2, 2, _GUARD_TOS_IS_NONE_r22 }, + { 3, 3, _GUARD_TOS_IS_NONE_r33 }, + }, + }, + [_GUARD_NOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, + { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, + { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, + }, + }, + [_SEND_VIRTUAL_TIER_TWO] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _SEND_VIRTUAL_TIER_TWO_r03 }, + { 3, 1, _SEND_VIRTUAL_TIER_TWO_r13 }, + { 3, 2, _SEND_VIRTUAL_TIER_TWO_r23 }, + { 3, 3, _SEND_VIRTUAL_TIER_TWO_r33 }, + }, + }, + [_GUARD_3OS_ASYNC_GEN_ASEND] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 3, 0, _GUARD_3OS_ASYNC_GEN_ASEND_r03 }, + { 3, 1, _GUARD_3OS_ASYNC_GEN_ASEND_r13 }, + { 3, 2, _GUARD_3OS_ASYNC_GEN_ASEND_r23 }, + { 3, 3, _GUARD_3OS_ASYNC_GEN_ASEND_r33 }, + }, + }, + [_SEND_ASYNC_GEN_TIER_TWO] = { + .best = { 3, 3, 3, 3 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 3, _SEND_ASYNC_GEN_TIER_TWO_r33 }, + }, + }, [_YIELD_VALUE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -1635,15 +1741,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_DELETE_NAME] = { - .best = { 0, 0, 0, 0 }, - .entries = { - { 0, 0, _DELETE_NAME_r00 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - }, - }, [_UNPACK_SEQUENCE] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -1725,15 +1822,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_DELETE_ATTR] = { - .best = { 1, 1, 1, 1 }, - .entries = { - { -1, -1, -1 }, - { 0, 1, _DELETE_ATTR_r10 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - }, - }, [_STORE_GLOBAL] = { .best = { 1, 1, 1, 1 }, .entries = { @@ -1743,15 +1831,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_DELETE_GLOBAL] = { - .best = { 0, 0, 0, 0 }, - .entries = { - { 0, 0, _DELETE_GLOBAL_r00 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - }, - }, [_LOAD_LOCALS] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -2058,6 +2137,15 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_TYPE_VERSION_LOCKED_r33 }, }, }, + [_GUARD_TYPE] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TYPE_r01 }, + { 1, 1, _GUARD_TYPE_r11 }, + { 2, 2, _GUARD_TYPE_r22 }, + { 3, 3, _GUARD_TYPE_r33 }, + }, + }, [_CHECK_MANAGED_OBJECT_HAS_VALUES] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -2122,10 +2210,19 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_ATTR_PROPERTY_FRAME] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _LOAD_ATTR_PROPERTY_FRAME_r01 }, + { 1, 1, _LOAD_ATTR_PROPERTY_FRAME_r11 }, + { 2, 2, _LOAD_ATTR_PROPERTY_FRAME_r22 }, + { 3, 3, _LOAD_ATTR_PROPERTY_FRAME_r33 }, + }, + }, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 1, _LOAD_ATTR_PROPERTY_FRAME_r11 }, + { 1, 1, _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -2373,6 +2470,42 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_ITERATOR] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITERATOR_r01 }, + { 1, 1, _GUARD_ITERATOR_r11 }, + { 2, 2, _GUARD_ITERATOR_r22 }, + { 3, 3, _GUARD_ITERATOR_r33 }, + }, + }, + [_GUARD_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_ITER_VIRTUAL_r01 }, + { 1, 1, _GUARD_ITER_VIRTUAL_r11 }, + { 2, 2, _GUARD_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_ITER_VIRTUAL_r33 }, + }, + }, + [_PUSH_TAGGED_ZERO] = { + .best = { 0, 1, 2, 2 }, + .entries = { + { 1, 0, _PUSH_TAGGED_ZERO_r01 }, + { 2, 1, _PUSH_TAGGED_ZERO_r12 }, + { 3, 2, _PUSH_TAGGED_ZERO_r23 }, + { -1, -1, -1 }, + }, + }, + [_GET_ITER_TRAD] = { + .best = { 1, 1, 1, 1 }, + .entries = { + { -1, -1, -1 }, + { 2, 1, _GET_ITER_TRAD_r12 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_FOR_ITER_TIER_TWO] = { .best = { 2, 2, 2, 2 }, .entries = { @@ -2382,6 +2515,51 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_TYPE_ITER] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_TYPE_ITER_r02 }, + { 2, 1, _GUARD_TYPE_ITER_r12 }, + { 2, 2, _GUARD_TYPE_ITER_r22 }, + { 3, 3, _GUARD_TYPE_ITER_r33 }, + }, + }, + [_ITER_NEXT_INLINE] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _ITER_NEXT_INLINE_r23 }, + { -1, -1, -1 }, + }, + }, + [_GUARD_NOS_ITER_VIRTUAL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 2, 0, _GUARD_NOS_ITER_VIRTUAL_r02 }, + { 2, 1, _GUARD_NOS_ITER_VIRTUAL_r12 }, + { 2, 2, _GUARD_NOS_ITER_VIRTUAL_r22 }, + { 3, 3, _GUARD_NOS_ITER_VIRTUAL_r33 }, + }, + }, + [_GUARD_TOS_NOT_NULL] = { + .best = { 0, 1, 2, 3 }, + .entries = { + { 1, 0, _GUARD_TOS_NOT_NULL_r01 }, + { 1, 1, _GUARD_TOS_NOT_NULL_r11 }, + { 2, 2, _GUARD_TOS_NOT_NULL_r22 }, + { 3, 3, _GUARD_TOS_NOT_NULL_r33 }, + }, + }, + [_FOR_ITER_VIRTUAL_TIER_TWO] = { + .best = { 2, 2, 2, 2 }, + .entries = { + { -1, -1, -1 }, + { -1, -1, -1 }, + { 3, 2, _FOR_ITER_VIRTUAL_TIER_TWO_r23 }, + { -1, -1, -1 }, + }, + }, [_ITER_CHECK_LIST] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -2508,24 +2686,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 1, 0, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01 }, - { 1, 1, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11 }, - { 2, 2, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22 }, - { 3, 3, _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33 }, - }, - }, - [_GUARD_KEYS_VERSION] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 1, 0, _GUARD_KEYS_VERSION_r01 }, - { 1, 1, _GUARD_KEYS_VERSION_r11 }, - { 2, 2, _GUARD_KEYS_VERSION_r22 }, - { 3, 3, _GUARD_KEYS_VERSION_r33 }, - }, - }, [_LOAD_ATTR_METHOD_WITH_VALUES] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -2778,15 +2938,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 3, 3, _GUARD_NOS_NULL_r33 }, }, }, - [_GUARD_NOS_NOT_NULL] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 2, 0, _GUARD_NOS_NOT_NULL_r02 }, - { 2, 1, _GUARD_NOS_NOT_NULL_r12 }, - { 2, 2, _GUARD_NOS_NOT_NULL_r22 }, - { 3, 3, _GUARD_NOS_NOT_NULL_r33 }, - }, - }, [_GUARD_THIRD_NULL] = { .best = { 0, 1, 2, 3 }, .entries = { @@ -2850,10 +3001,19 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { 2, 3, _CALL_TUPLE_1_r32 }, }, }, - [_CHECK_AND_ALLOCATE_OBJECT] = { + [_CHECK_OBJECT] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _CHECK_OBJECT_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, + [_ALLOCATE_OBJECT] = { .best = { 0, 0, 0, 0 }, .entries = { - { 0, 0, _CHECK_AND_ALLOCATE_OBJECT_r00 }, + { 0, 0, _ALLOCATE_OBJECT_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -2877,10 +3037,19 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, + [_GUARD_CALLABLE_BUILTIN_CLASS] = { + .best = { 0, 0, 0, 0 }, + .entries = { + { 0, 0, _GUARD_CALLABLE_BUILTIN_CLASS_r00 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + }, + }, [_CALL_BUILTIN_CLASS] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_BUILTIN_CLASS_r01 }, + { 0, 0, _CALL_BUILTIN_CLASS_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -2916,7 +3085,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_BUILTIN_FAST] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_BUILTIN_FAST_r01 }, + { 0, 0, _CALL_BUILTIN_FAST_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -2934,7 +3103,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_BUILTIN_FAST_WITH_KEYWORDS_r01 }, + { 0, 0, _CALL_BUILTIN_FAST_WITH_KEYWORDS_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3042,7 +3211,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3051,7 +3220,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3096,7 +3265,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3105,7 +3274,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01 }, + { 0, 0, _CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3223,7 +3392,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { .best = { 1, 1, 1, 1 }, .entries = { { -1, -1, -1 }, - { 1, 1, _MAKE_FUNCTION_r11 }, + { 2, 1, _MAKE_FUNCTION_r12 }, { -1, -1, -1 }, { -1, -1, -1 }, }, @@ -3240,7 +3409,7 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { [_RETURN_GENERATOR] = { .best = { 0, 0, 0, 0 }, .entries = { - { 1, 0, _RETURN_GENERATOR_r01 }, + { 1, 1, _RETURN_GENERATOR_r01 }, { -1, -1, -1 }, { -1, -1, -1 }, { -1, -1, -1 }, @@ -3552,15 +3721,6 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_POP_TOP_LOAD_CONST_INLINE] = { - .best = { 1, 1, 1, 1 }, - .entries = { - { -1, -1, -1 }, - { 1, 1, _POP_TOP_LOAD_CONST_INLINE_r11 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - }, - }, [_LOAD_CONST_INLINE_BORROW] = { .best = { 0, 1, 2, 2 }, .entries = { @@ -3570,121 +3730,13 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { { -1, -1, -1 }, }, }, - [_POP_CALL] = { - .best = { 2, 2, 2, 2 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { 0, 2, _POP_CALL_r20 }, - { -1, -1, -1 }, - }, - }, - [_POP_CALL_ONE] = { - .best = { 3, 3, 3, 3 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - { 0, 3, _POP_CALL_ONE_r30 }, - }, - }, - [_POP_CALL_TWO] = { - .best = { 3, 3, 3, 3 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - { 0, 3, _POP_CALL_TWO_r30 }, - }, - }, - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = { - .best = { 1, 1, 1, 1 }, - .entries = { - { -1, -1, -1 }, - { 1, 1, _POP_TOP_LOAD_CONST_INLINE_BORROW_r11 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - }, - }, - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = { - .best = { 2, 2, 2, 2 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { 1, 2, _POP_TWO_LOAD_CONST_INLINE_BORROW_r21 }, - { -1, -1, -1 }, - }, - }, - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = { - .best = { 2, 2, 2, 2 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { 1, 2, _POP_CALL_LOAD_CONST_INLINE_BORROW_r21 }, - { -1, -1, -1 }, - }, - }, - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = { - .best = { 3, 3, 3, 3 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - { 1, 3, _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31 }, - }, - }, - [_INSERT_1_LOAD_CONST_INLINE] = { - .best = { 0, 1, 2, 2 }, - .entries = { - { 2, 0, _INSERT_1_LOAD_CONST_INLINE_r02 }, - { 2, 1, _INSERT_1_LOAD_CONST_INLINE_r12 }, - { 3, 2, _INSERT_1_LOAD_CONST_INLINE_r23 }, - { -1, -1, -1 }, - }, - }, - [_INSERT_1_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, - .entries = { - { 2, 0, _INSERT_1_LOAD_CONST_INLINE_BORROW_r02 }, - { 2, 1, _INSERT_1_LOAD_CONST_INLINE_BORROW_r12 }, - { 3, 2, _INSERT_1_LOAD_CONST_INLINE_BORROW_r23 }, - { -1, -1, -1 }, - }, - }, - [_INSERT_2_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, - .entries = { - { 3, 0, _INSERT_2_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _INSERT_2_LOAD_CONST_INLINE_BORROW_r13 }, - { 3, 2, _INSERT_2_LOAD_CONST_INLINE_BORROW_r23 }, - { -1, -1, -1 }, - }, - }, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 3 }, - .entries = { - { 2, 0, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02 }, - { 2, 1, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12 }, - { 2, 2, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22 }, - { 2, 3, _SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32 }, - }, - }, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = { + [_RROT_3] = { .best = { 0, 1, 2, 3 }, .entries = { - { 3, 0, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 }, - { 3, 2, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 }, - { 3, 3, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 }, - }, - }, - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = { - .best = { 3, 3, 3, 3 }, - .entries = { - { -1, -1, -1 }, - { -1, -1, -1 }, - { -1, -1, -1 }, - { 1, 3, _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31 }, + { 3, 0, _RROT_3_r03 }, + { 3, 1, _RROT_3_r13 }, + { 3, 2, _RROT_3_r23 }, + { 3, 3, _RROT_3_r33 }, }, }, [_START_EXECUTOR] = { @@ -3984,7 +4036,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_POP_TOP_UNICODE_r10] = _POP_TOP_UNICODE, [_POP_TOP_UNICODE_r21] = _POP_TOP_UNICODE, [_POP_TOP_UNICODE_r32] = _POP_TOP_UNICODE, - [_POP_TWO_r20] = _POP_TWO, + [_POP_TOP_OPARG_r00] = _POP_TOP_OPARG, [_PUSH_NULL_r01] = _PUSH_NULL, [_PUSH_NULL_r12] = _PUSH_NULL, [_PUSH_NULL_r23] = _PUSH_NULL, @@ -4053,6 +4105,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_INT_r11] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r22] = _GUARD_TOS_INT, [_GUARD_TOS_INT_r33] = _GUARD_TOS_INT, + [_GUARD_TOS_EXACT_INT_r01] = _GUARD_TOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r11] = _GUARD_TOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r22] = _GUARD_TOS_EXACT_INT, + [_GUARD_TOS_EXACT_INT_r33] = _GUARD_TOS_EXACT_INT, [_GUARD_NOS_OVERFLOWED_r02] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r12] = _GUARD_NOS_OVERFLOWED, [_GUARD_NOS_OVERFLOWED_r22] = _GUARD_NOS_OVERFLOWED, @@ -4123,10 +4179,25 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, [_BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_r23] = _BINARY_OP_TRUEDIV_FLOAT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT, [_BINARY_OP_ADD_UNICODE_r03] = _BINARY_OP_ADD_UNICODE, [_BINARY_OP_ADD_UNICODE_r13] = _BINARY_OP_ADD_UNICODE, [_BINARY_OP_ADD_UNICODE_r23] = _BINARY_OP_ADD_UNICODE, [_BINARY_OP_INPLACE_ADD_UNICODE_r21] = _BINARY_OP_INPLACE_ADD_UNICODE, + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = _GUARD_BINARY_OP_EXTEND_LHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = _GUARD_BINARY_OP_EXTEND_RHS, + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = _GUARD_BINARY_OP_EXTEND_RHS, [_GUARD_BINARY_OP_EXTEND_r22] = _GUARD_BINARY_OP_EXTEND, [_BINARY_OP_EXTEND_r23] = _BINARY_OP_EXTEND, [_BINARY_SLICE_r31] = _BINARY_SLICE, @@ -4150,14 +4221,14 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBSCR_TUPLE_INT_r03] = _BINARY_OP_SUBSCR_TUPLE_INT, [_BINARY_OP_SUBSCR_TUPLE_INT_r13] = _BINARY_OP_SUBSCR_TUPLE_INT, [_BINARY_OP_SUBSCR_TUPLE_INT_r23] = _BINARY_OP_SUBSCR_TUPLE_INT, - [_GUARD_NOS_DICT_r02] = _GUARD_NOS_DICT, - [_GUARD_NOS_DICT_r12] = _GUARD_NOS_DICT, - [_GUARD_NOS_DICT_r22] = _GUARD_NOS_DICT, - [_GUARD_NOS_DICT_r33] = _GUARD_NOS_DICT, - [_GUARD_NOS_ANY_DICT_r02] = _GUARD_NOS_ANY_DICT, - [_GUARD_NOS_ANY_DICT_r12] = _GUARD_NOS_ANY_DICT, - [_GUARD_NOS_ANY_DICT_r22] = _GUARD_NOS_ANY_DICT, - [_GUARD_NOS_ANY_DICT_r33] = _GUARD_NOS_ANY_DICT, + [_GUARD_NOS_DICT_SUBSCRIPT_r02] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r12] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r22] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_SUBSCRIPT_r33] = _GUARD_NOS_DICT_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33] = _GUARD_NOS_DICT_STORE_SUBSCRIPT, [_GUARD_TOS_ANY_DICT_r01] = _GUARD_TOS_ANY_DICT, [_GUARD_TOS_ANY_DICT_r11] = _GUARD_TOS_ANY_DICT, [_GUARD_TOS_ANY_DICT_r22] = _GUARD_TOS_ANY_DICT, @@ -4195,6 +4266,23 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GET_ANEXT_r12] = _GET_ANEXT, [_GET_AWAITABLE_r11] = _GET_AWAITABLE, [_SEND_GEN_FRAME_r33] = _SEND_GEN_FRAME, + [_GUARD_TOS_IS_NONE_r01] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r11] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r22] = _GUARD_TOS_IS_NONE, + [_GUARD_TOS_IS_NONE_r33] = _GUARD_TOS_IS_NONE, + [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, + [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, + [_SEND_VIRTUAL_TIER_TWO_r03] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r13] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r23] = _SEND_VIRTUAL_TIER_TWO, + [_SEND_VIRTUAL_TIER_TWO_r33] = _SEND_VIRTUAL_TIER_TWO, + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = _GUARD_3OS_ASYNC_GEN_ASEND, + [_SEND_ASYNC_GEN_TIER_TWO_r33] = _SEND_ASYNC_GEN_TIER_TWO, [_YIELD_VALUE_r11] = _YIELD_VALUE, [_POP_EXCEPT_r10] = _POP_EXCEPT, [_LOAD_COMMON_CONSTANT_r01] = _LOAD_COMMON_CONSTANT, @@ -4202,7 +4290,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_COMMON_CONSTANT_r23] = _LOAD_COMMON_CONSTANT, [_LOAD_BUILD_CLASS_r01] = _LOAD_BUILD_CLASS, [_STORE_NAME_r10] = _STORE_NAME, - [_DELETE_NAME_r00] = _DELETE_NAME, [_UNPACK_SEQUENCE_r10] = _UNPACK_SEQUENCE, [_UNPACK_SEQUENCE_TWO_TUPLE_r12] = _UNPACK_SEQUENCE_TWO_TUPLE, [_UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02] = _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE, @@ -4215,9 +4302,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_UNPACK_SEQUENCE_LIST_r10] = _UNPACK_SEQUENCE_LIST, [_UNPACK_EX_r10] = _UNPACK_EX, [_STORE_ATTR_r20] = _STORE_ATTR, - [_DELETE_ATTR_r10] = _DELETE_ATTR, [_STORE_GLOBAL_r10] = _STORE_GLOBAL, - [_DELETE_GLOBAL_r00] = _DELETE_GLOBAL, [_LOAD_LOCALS_r01] = _LOAD_LOCALS, [_LOAD_LOCALS_r12] = _LOAD_LOCALS, [_LOAD_LOCALS_r23] = _LOAD_LOCALS, @@ -4272,6 +4357,10 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_TYPE_VERSION_LOCKED_r11] = _GUARD_TYPE_VERSION_LOCKED, [_GUARD_TYPE_VERSION_LOCKED_r22] = _GUARD_TYPE_VERSION_LOCKED, [_GUARD_TYPE_VERSION_LOCKED_r33] = _GUARD_TYPE_VERSION_LOCKED, + [_GUARD_TYPE_r01] = _GUARD_TYPE, + [_GUARD_TYPE_r11] = _GUARD_TYPE, + [_GUARD_TYPE_r22] = _GUARD_TYPE, + [_GUARD_TYPE_r33] = _GUARD_TYPE, [_CHECK_MANAGED_OBJECT_HAS_VALUES_r01] = _CHECK_MANAGED_OBJECT_HAS_VALUES, [_CHECK_MANAGED_OBJECT_HAS_VALUES_r11] = _CHECK_MANAGED_OBJECT_HAS_VALUES, [_CHECK_MANAGED_OBJECT_HAS_VALUES_r22] = _CHECK_MANAGED_OBJECT_HAS_VALUES, @@ -4289,7 +4378,11 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CHECK_ATTR_CLASS_r22] = _CHECK_ATTR_CLASS, [_CHECK_ATTR_CLASS_r33] = _CHECK_ATTR_CLASS, [_LOAD_ATTR_CLASS_r11] = _LOAD_ATTR_CLASS, + [_LOAD_ATTR_PROPERTY_FRAME_r01] = _LOAD_ATTR_PROPERTY_FRAME, [_LOAD_ATTR_PROPERTY_FRAME_r11] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r22] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_PROPERTY_FRAME_r33] = _LOAD_ATTR_PROPERTY_FRAME, + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME, [_GUARD_DORV_NO_DICT_r01] = _GUARD_DORV_NO_DICT, [_GUARD_DORV_NO_DICT_r11] = _GUARD_DORV_NO_DICT, [_GUARD_DORV_NO_DICT_r22] = _GUARD_DORV_NO_DICT, @@ -4340,7 +4433,33 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_MATCH_SEQUENCE_r23] = _MATCH_SEQUENCE, [_MATCH_KEYS_r23] = _MATCH_KEYS, [_GET_ITER_r12] = _GET_ITER, + [_GUARD_ITERATOR_r01] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r11] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r22] = _GUARD_ITERATOR, + [_GUARD_ITERATOR_r33] = _GUARD_ITERATOR, + [_GUARD_ITER_VIRTUAL_r01] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r11] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r22] = _GUARD_ITER_VIRTUAL, + [_GUARD_ITER_VIRTUAL_r33] = _GUARD_ITER_VIRTUAL, + [_PUSH_TAGGED_ZERO_r01] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r12] = _PUSH_TAGGED_ZERO, + [_PUSH_TAGGED_ZERO_r23] = _PUSH_TAGGED_ZERO, + [_GET_ITER_TRAD_r12] = _GET_ITER_TRAD, [_FOR_ITER_TIER_TWO_r23] = _FOR_ITER_TIER_TWO, + [_GUARD_TYPE_ITER_r02] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r12] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r22] = _GUARD_TYPE_ITER, + [_GUARD_TYPE_ITER_r33] = _GUARD_TYPE_ITER, + [_ITER_NEXT_INLINE_r23] = _ITER_NEXT_INLINE, + [_GUARD_NOS_ITER_VIRTUAL_r02] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r12] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r22] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_NOS_ITER_VIRTUAL_r33] = _GUARD_NOS_ITER_VIRTUAL, + [_GUARD_TOS_NOT_NULL_r01] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r11] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r22] = _GUARD_TOS_NOT_NULL, + [_GUARD_TOS_NOT_NULL_r33] = _GUARD_TOS_NOT_NULL, + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = _FOR_ITER_VIRTUAL_TIER_TWO, [_ITER_CHECK_LIST_r02] = _ITER_CHECK_LIST, [_ITER_CHECK_LIST_r12] = _ITER_CHECK_LIST, [_ITER_CHECK_LIST_r22] = _ITER_CHECK_LIST, @@ -4381,14 +4500,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_PUSH_EXC_INFO_r02] = _PUSH_EXC_INFO, [_PUSH_EXC_INFO_r12] = _PUSH_EXC_INFO, [_PUSH_EXC_INFO_r23] = _PUSH_EXC_INFO, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT, - [_GUARD_KEYS_VERSION_r01] = _GUARD_KEYS_VERSION, - [_GUARD_KEYS_VERSION_r11] = _GUARD_KEYS_VERSION, - [_GUARD_KEYS_VERSION_r22] = _GUARD_KEYS_VERSION, - [_GUARD_KEYS_VERSION_r33] = _GUARD_KEYS_VERSION, [_LOAD_ATTR_METHOD_WITH_VALUES_r02] = _LOAD_ATTR_METHOD_WITH_VALUES, [_LOAD_ATTR_METHOD_WITH_VALUES_r12] = _LOAD_ATTR_METHOD_WITH_VALUES, [_LOAD_ATTR_METHOD_WITH_VALUES_r23] = _LOAD_ATTR_METHOD_WITH_VALUES, @@ -4438,10 +4549,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_NULL_r12] = _GUARD_NOS_NULL, [_GUARD_NOS_NULL_r22] = _GUARD_NOS_NULL, [_GUARD_NOS_NULL_r33] = _GUARD_NOS_NULL, - [_GUARD_NOS_NOT_NULL_r02] = _GUARD_NOS_NOT_NULL, - [_GUARD_NOS_NOT_NULL_r12] = _GUARD_NOS_NOT_NULL, - [_GUARD_NOS_NOT_NULL_r22] = _GUARD_NOS_NOT_NULL, - [_GUARD_NOS_NOT_NULL_r33] = _GUARD_NOS_NOT_NULL, [_GUARD_THIRD_NULL_r03] = _GUARD_THIRD_NULL, [_GUARD_THIRD_NULL_r13] = _GUARD_THIRD_NULL, [_GUARD_THIRD_NULL_r23] = _GUARD_THIRD_NULL, @@ -4464,16 +4571,18 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_GUARD_CALLABLE_TUPLE_1_r23] = _GUARD_CALLABLE_TUPLE_1, [_GUARD_CALLABLE_TUPLE_1_r33] = _GUARD_CALLABLE_TUPLE_1, [_CALL_TUPLE_1_r32] = _CALL_TUPLE_1, - [_CHECK_AND_ALLOCATE_OBJECT_r00] = _CHECK_AND_ALLOCATE_OBJECT, + [_CHECK_OBJECT_r00] = _CHECK_OBJECT, + [_ALLOCATE_OBJECT_r00] = _ALLOCATE_OBJECT, [_CREATE_INIT_FRAME_r01] = _CREATE_INIT_FRAME, [_EXIT_INIT_CHECK_r10] = _EXIT_INIT_CHECK, - [_CALL_BUILTIN_CLASS_r01] = _CALL_BUILTIN_CLASS, + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = _GUARD_CALLABLE_BUILTIN_CLASS, + [_CALL_BUILTIN_CLASS_r00] = _CALL_BUILTIN_CLASS, [_GUARD_CALLABLE_BUILTIN_O_r00] = _GUARD_CALLABLE_BUILTIN_O, [_CALL_BUILTIN_O_r03] = _CALL_BUILTIN_O, [_GUARD_CALLABLE_BUILTIN_FAST_r00] = _GUARD_CALLABLE_BUILTIN_FAST, - [_CALL_BUILTIN_FAST_r01] = _CALL_BUILTIN_FAST, + [_CALL_BUILTIN_FAST_r00] = _CALL_BUILTIN_FAST, [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS, - [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = _CALL_BUILTIN_FAST_WITH_KEYWORDS, [_GUARD_CALLABLE_LEN_r03] = _GUARD_CALLABLE_LEN, [_GUARD_CALLABLE_LEN_r13] = _GUARD_CALLABLE_LEN, [_GUARD_CALLABLE_LEN_r23] = _GUARD_CALLABLE_LEN, @@ -4500,14 +4609,14 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CHECK_RECURSION_LIMIT_r33] = _CHECK_RECURSION_LIMIT, [_CALL_METHOD_DESCRIPTOR_O_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_O_INLINE, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS, [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS, [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE_r03] = _CALL_METHOD_DESCRIPTOR_NOARGS_INLINE, [_GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_r00] = _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST, - [_CALL_METHOD_DESCRIPTOR_FAST_r01] = _CALL_METHOD_DESCRIPTOR_FAST, - [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = _CALL_METHOD_DESCRIPTOR_FAST, + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = _CALL_METHOD_DESCRIPTOR_FAST_INLINE, [_MAYBE_EXPAND_METHOD_KW_r11] = _MAYBE_EXPAND_METHOD_KW, [_PY_FRAME_KW_r11] = _PY_FRAME_KW, [_CHECK_FUNCTION_VERSION_KW_r11] = _CHECK_FUNCTION_VERSION_KW, @@ -4526,7 +4635,7 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_CHECK_IS_NOT_PY_CALLABLE_EX_r23] = _CHECK_IS_NOT_PY_CALLABLE_EX, [_CHECK_IS_NOT_PY_CALLABLE_EX_r33] = _CHECK_IS_NOT_PY_CALLABLE_EX, [_CALL_FUNCTION_EX_NON_PY_GENERAL_r31] = _CALL_FUNCTION_EX_NON_PY_GENERAL, - [_MAKE_FUNCTION_r11] = _MAKE_FUNCTION, + [_MAKE_FUNCTION_r12] = _MAKE_FUNCTION, [_SET_FUNCTION_ATTRIBUTE_r01] = _SET_FUNCTION_ATTRIBUTE, [_SET_FUNCTION_ATTRIBUTE_r11] = _SET_FUNCTION_ATTRIBUTE, [_SET_FUNCTION_ATTRIBUTE_r21] = _SET_FUNCTION_ATTRIBUTE, @@ -4638,35 +4747,13 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r01] = _LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, - [_POP_TOP_LOAD_CONST_INLINE_r11] = _POP_TOP_LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, - [_POP_CALL_r20] = _POP_CALL, - [_POP_CALL_ONE_r30] = _POP_CALL_ONE, - [_POP_CALL_TWO_r30] = _POP_CALL_TWO, - [_POP_TOP_LOAD_CONST_INLINE_BORROW_r11] = _POP_TOP_LOAD_CONST_INLINE_BORROW, - [_POP_TWO_LOAD_CONST_INLINE_BORROW_r21] = _POP_TWO_LOAD_CONST_INLINE_BORROW, - [_POP_CALL_LOAD_CONST_INLINE_BORROW_r21] = _POP_CALL_LOAD_CONST_INLINE_BORROW, - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW, - [_INSERT_1_LOAD_CONST_INLINE_r02] = _INSERT_1_LOAD_CONST_INLINE, - [_INSERT_1_LOAD_CONST_INLINE_r12] = _INSERT_1_LOAD_CONST_INLINE, - [_INSERT_1_LOAD_CONST_INLINE_r23] = _INSERT_1_LOAD_CONST_INLINE, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r02] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r12] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r23] = _INSERT_1_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r03] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r13] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r23] = _INSERT_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32] = _SHUFFLE_2_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW, + [_RROT_3_r03] = _RROT_3, + [_RROT_3_r13] = _RROT_3, + [_RROT_3_r23] = _RROT_3, + [_RROT_3_r33] = _RROT_3, [_START_EXECUTOR_r00] = _START_EXECUTOR, [_MAKE_WARM_r00] = _MAKE_WARM, [_MAKE_WARM_r11] = _MAKE_WARM, @@ -4753,6 +4840,8 @@ const uint16_t _PyUop_SpillsAndReloads[4][4] = { }; const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { + [_ALLOCATE_OBJECT] = "_ALLOCATE_OBJECT", + [_ALLOCATE_OBJECT_r00] = "_ALLOCATE_OBJECT_r00", [_BINARY_OP] = "_BINARY_OP", [_BINARY_OP_r23] = "_BINARY_OP_r23", [_BINARY_OP_ADD_FLOAT] = "_BINARY_OP_ADD_FLOAT", @@ -4858,6 +4947,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r03", [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r13", [_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23] = "_BINARY_OP_SUBTRACT_INT_INPLACE_RIGHT_r23", + [_BINARY_OP_TRUEDIV_FLOAT] = "_BINARY_OP_TRUEDIV_FLOAT", + [_BINARY_OP_TRUEDIV_FLOAT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_r23", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r03", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r13", + [_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23] = "_BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT_r23", [_BINARY_SLICE] = "_BINARY_SLICE", [_BINARY_SLICE_r31] = "_BINARY_SLICE_r31", [_BUILD_INTERPOLATION] = "_BUILD_INTERPOLATION", @@ -4877,11 +4976,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_BUILD_TUPLE] = "_BUILD_TUPLE", [_BUILD_TUPLE_r01] = "_BUILD_TUPLE_r01", [_CALL_BUILTIN_CLASS] = "_CALL_BUILTIN_CLASS", - [_CALL_BUILTIN_CLASS_r01] = "_CALL_BUILTIN_CLASS_r01", + [_CALL_BUILTIN_CLASS_r00] = "_CALL_BUILTIN_CLASS_r00", [_CALL_BUILTIN_FAST] = "_CALL_BUILTIN_FAST", - [_CALL_BUILTIN_FAST_r01] = "_CALL_BUILTIN_FAST_r01", + [_CALL_BUILTIN_FAST_r00] = "_CALL_BUILTIN_FAST_r00", [_CALL_BUILTIN_FAST_WITH_KEYWORDS] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS", - [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r01", + [_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00] = "_CALL_BUILTIN_FAST_WITH_KEYWORDS_r00", [_CALL_BUILTIN_O] = "_CALL_BUILTIN_O", [_CALL_BUILTIN_O_r03] = "_CALL_BUILTIN_O_r03", [_CALL_FUNCTION_EX_NON_PY_GENERAL] = "_CALL_FUNCTION_EX_NON_PY_GENERAL", @@ -4902,13 +5001,13 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_LIST_APPEND_r23] = "_CALL_LIST_APPEND_r23", [_CALL_LIST_APPEND_r33] = "_CALL_LIST_APPEND_r33", [_CALL_METHOD_DESCRIPTOR_FAST] = "_CALL_METHOD_DESCRIPTOR_FAST", - [_CALL_METHOD_DESCRIPTOR_FAST_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_r00", [_CALL_METHOD_DESCRIPTOR_FAST_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE", - [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_r00", [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE", - [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r01", + [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE_r00", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_NOARGS_r03] = "_CALL_METHOD_DESCRIPTOR_NOARGS_r03", [_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE] = "_CALL_METHOD_DESCRIPTOR_NOARGS_INLINE", @@ -4928,8 +5027,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CALL_TYPE_1_r12] = "_CALL_TYPE_1_r12", [_CALL_TYPE_1_r22] = "_CALL_TYPE_1_r22", [_CALL_TYPE_1_r32] = "_CALL_TYPE_1_r32", - [_CHECK_AND_ALLOCATE_OBJECT] = "_CHECK_AND_ALLOCATE_OBJECT", - [_CHECK_AND_ALLOCATE_OBJECT_r00] = "_CHECK_AND_ALLOCATE_OBJECT_r00", [_CHECK_ATTR_CLASS] = "_CHECK_ATTR_CLASS", [_CHECK_ATTR_CLASS_r01] = "_CHECK_ATTR_CLASS_r01", [_CHECK_ATTR_CLASS_r11] = "_CHECK_ATTR_CLASS_r11", @@ -4980,6 +5077,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_CHECK_METHOD_VERSION_r00] = "_CHECK_METHOD_VERSION_r00", [_CHECK_METHOD_VERSION_KW] = "_CHECK_METHOD_VERSION_KW", [_CHECK_METHOD_VERSION_KW_r11] = "_CHECK_METHOD_VERSION_KW_r11", + [_CHECK_OBJECT] = "_CHECK_OBJECT", + [_CHECK_OBJECT_r00] = "_CHECK_OBJECT_r00", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PEP_523_r00] = "_CHECK_PEP_523_r00", [_CHECK_PEP_523_r11] = "_CHECK_PEP_523_r11", @@ -5055,16 +5154,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_COPY_FREE_VARS_r33] = "_COPY_FREE_VARS_r33", [_CREATE_INIT_FRAME] = "_CREATE_INIT_FRAME", [_CREATE_INIT_FRAME_r01] = "_CREATE_INIT_FRAME_r01", - [_DELETE_ATTR] = "_DELETE_ATTR", - [_DELETE_ATTR_r10] = "_DELETE_ATTR_r10", [_DELETE_DEREF] = "_DELETE_DEREF", [_DELETE_DEREF_r00] = "_DELETE_DEREF_r00", [_DELETE_FAST] = "_DELETE_FAST", [_DELETE_FAST_r00] = "_DELETE_FAST_r00", - [_DELETE_GLOBAL] = "_DELETE_GLOBAL", - [_DELETE_GLOBAL_r00] = "_DELETE_GLOBAL_r00", - [_DELETE_NAME] = "_DELETE_NAME", - [_DELETE_NAME_r00] = "_DELETE_NAME_r00", [_DELETE_SUBSCR] = "_DELETE_SUBSCR", [_DELETE_SUBSCR_r20] = "_DELETE_SUBSCR_r20", [_DEOPT] = "_DEOPT", @@ -5113,6 +5206,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_FOR_ITER_GEN_FRAME_r23] = "_FOR_ITER_GEN_FRAME_r23", [_FOR_ITER_TIER_TWO] = "_FOR_ITER_TIER_TWO", [_FOR_ITER_TIER_TWO_r23] = "_FOR_ITER_TIER_TWO_r23", + [_FOR_ITER_VIRTUAL_TIER_TWO] = "_FOR_ITER_VIRTUAL_TIER_TWO", + [_FOR_ITER_VIRTUAL_TIER_TWO_r23] = "_FOR_ITER_VIRTUAL_TIER_TWO_r23", [_GET_AITER] = "_GET_AITER", [_GET_AITER_r11] = "_GET_AITER_r11", [_GET_ANEXT] = "_GET_ANEXT", @@ -5121,10 +5216,27 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GET_AWAITABLE_r11] = "_GET_AWAITABLE_r11", [_GET_ITER] = "_GET_ITER", [_GET_ITER_r12] = "_GET_ITER_r12", + [_GET_ITER_TRAD] = "_GET_ITER_TRAD", + [_GET_ITER_TRAD_r12] = "_GET_ITER_TRAD_r12", [_GET_LEN] = "_GET_LEN", [_GET_LEN_r12] = "_GET_LEN_r12", + [_GUARD_3OS_ASYNC_GEN_ASEND] = "_GUARD_3OS_ASYNC_GEN_ASEND", + [_GUARD_3OS_ASYNC_GEN_ASEND_r03] = "_GUARD_3OS_ASYNC_GEN_ASEND_r03", + [_GUARD_3OS_ASYNC_GEN_ASEND_r13] = "_GUARD_3OS_ASYNC_GEN_ASEND_r13", + [_GUARD_3OS_ASYNC_GEN_ASEND_r23] = "_GUARD_3OS_ASYNC_GEN_ASEND_r23", + [_GUARD_3OS_ASYNC_GEN_ASEND_r33] = "_GUARD_3OS_ASYNC_GEN_ASEND_r33", [_GUARD_BINARY_OP_EXTEND] = "_GUARD_BINARY_OP_EXTEND", [_GUARD_BINARY_OP_EXTEND_r22] = "_GUARD_BINARY_OP_EXTEND_r22", + [_GUARD_BINARY_OP_EXTEND_LHS] = "_GUARD_BINARY_OP_EXTEND_LHS", + [_GUARD_BINARY_OP_EXTEND_LHS_r02] = "_GUARD_BINARY_OP_EXTEND_LHS_r02", + [_GUARD_BINARY_OP_EXTEND_LHS_r12] = "_GUARD_BINARY_OP_EXTEND_LHS_r12", + [_GUARD_BINARY_OP_EXTEND_LHS_r22] = "_GUARD_BINARY_OP_EXTEND_LHS_r22", + [_GUARD_BINARY_OP_EXTEND_LHS_r33] = "_GUARD_BINARY_OP_EXTEND_LHS_r33", + [_GUARD_BINARY_OP_EXTEND_RHS] = "_GUARD_BINARY_OP_EXTEND_RHS", + [_GUARD_BINARY_OP_EXTEND_RHS_r02] = "_GUARD_BINARY_OP_EXTEND_RHS_r02", + [_GUARD_BINARY_OP_EXTEND_RHS_r12] = "_GUARD_BINARY_OP_EXTEND_RHS_r12", + [_GUARD_BINARY_OP_EXTEND_RHS_r22] = "_GUARD_BINARY_OP_EXTEND_RHS_r22", + [_GUARD_BINARY_OP_EXTEND_RHS_r33] = "_GUARD_BINARY_OP_EXTEND_RHS_r33", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r02", [_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12] = "_GUARD_BINARY_OP_SUBSCR_TUPLE_INT_BOUNDS_r12", @@ -5180,6 +5292,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_BIT_IS_UNSET_POP_7_r10] = "_GUARD_BIT_IS_UNSET_POP_7_r10", [_GUARD_BIT_IS_UNSET_POP_7_r21] = "_GUARD_BIT_IS_UNSET_POP_7_r21", [_GUARD_BIT_IS_UNSET_POP_7_r32] = "_GUARD_BIT_IS_UNSET_POP_7_r32", + [_GUARD_CALLABLE_BUILTIN_CLASS] = "_GUARD_CALLABLE_BUILTIN_CLASS", + [_GUARD_CALLABLE_BUILTIN_CLASS_r00] = "_GUARD_CALLABLE_BUILTIN_CLASS_r00", [_GUARD_CALLABLE_BUILTIN_FAST] = "_GUARD_CALLABLE_BUILTIN_FAST", [_GUARD_CALLABLE_BUILTIN_FAST_r00] = "_GUARD_CALLABLE_BUILTIN_FAST_r00", [_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS] = "_GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS", @@ -5249,11 +5363,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_DORV_NO_DICT_r11] = "_GUARD_DORV_NO_DICT_r11", [_GUARD_DORV_NO_DICT_r22] = "_GUARD_DORV_NO_DICT_r22", [_GUARD_DORV_NO_DICT_r33] = "_GUARD_DORV_NO_DICT_r33", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r01", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r11", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r22", - [_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33] = "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT_r33", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_GLOBALS_VERSION_r00] = "_GUARD_GLOBALS_VERSION_r00", [_GUARD_GLOBALS_VERSION_r11] = "_GUARD_GLOBALS_VERSION_r11", @@ -5296,31 +5405,36 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_IS_TRUE_POP_r10] = "_GUARD_IS_TRUE_POP_r10", [_GUARD_IS_TRUE_POP_r21] = "_GUARD_IS_TRUE_POP_r21", [_GUARD_IS_TRUE_POP_r32] = "_GUARD_IS_TRUE_POP_r32", - [_GUARD_KEYS_VERSION] = "_GUARD_KEYS_VERSION", - [_GUARD_KEYS_VERSION_r01] = "_GUARD_KEYS_VERSION_r01", - [_GUARD_KEYS_VERSION_r11] = "_GUARD_KEYS_VERSION_r11", - [_GUARD_KEYS_VERSION_r22] = "_GUARD_KEYS_VERSION_r22", - [_GUARD_KEYS_VERSION_r33] = "_GUARD_KEYS_VERSION_r33", + [_GUARD_ITERATOR] = "_GUARD_ITERATOR", + [_GUARD_ITERATOR_r01] = "_GUARD_ITERATOR_r01", + [_GUARD_ITERATOR_r11] = "_GUARD_ITERATOR_r11", + [_GUARD_ITERATOR_r22] = "_GUARD_ITERATOR_r22", + [_GUARD_ITERATOR_r33] = "_GUARD_ITERATOR_r33", + [_GUARD_ITER_VIRTUAL] = "_GUARD_ITER_VIRTUAL", + [_GUARD_ITER_VIRTUAL_r01] = "_GUARD_ITER_VIRTUAL_r01", + [_GUARD_ITER_VIRTUAL_r11] = "_GUARD_ITER_VIRTUAL_r11", + [_GUARD_ITER_VIRTUAL_r22] = "_GUARD_ITER_VIRTUAL_r22", + [_GUARD_ITER_VIRTUAL_r33] = "_GUARD_ITER_VIRTUAL_r33", [_GUARD_LOAD_SUPER_ATTR_METHOD] = "_GUARD_LOAD_SUPER_ATTR_METHOD", [_GUARD_LOAD_SUPER_ATTR_METHOD_r03] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r03", [_GUARD_LOAD_SUPER_ATTR_METHOD_r13] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r13", [_GUARD_LOAD_SUPER_ATTR_METHOD_r23] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r23", [_GUARD_LOAD_SUPER_ATTR_METHOD_r33] = "_GUARD_LOAD_SUPER_ATTR_METHOD_r33", - [_GUARD_NOS_ANY_DICT] = "_GUARD_NOS_ANY_DICT", - [_GUARD_NOS_ANY_DICT_r02] = "_GUARD_NOS_ANY_DICT_r02", - [_GUARD_NOS_ANY_DICT_r12] = "_GUARD_NOS_ANY_DICT_r12", - [_GUARD_NOS_ANY_DICT_r22] = "_GUARD_NOS_ANY_DICT_r22", - [_GUARD_NOS_ANY_DICT_r33] = "_GUARD_NOS_ANY_DICT_r33", [_GUARD_NOS_COMPACT_ASCII] = "_GUARD_NOS_COMPACT_ASCII", [_GUARD_NOS_COMPACT_ASCII_r02] = "_GUARD_NOS_COMPACT_ASCII_r02", [_GUARD_NOS_COMPACT_ASCII_r12] = "_GUARD_NOS_COMPACT_ASCII_r12", [_GUARD_NOS_COMPACT_ASCII_r22] = "_GUARD_NOS_COMPACT_ASCII_r22", [_GUARD_NOS_COMPACT_ASCII_r33] = "_GUARD_NOS_COMPACT_ASCII_r33", - [_GUARD_NOS_DICT] = "_GUARD_NOS_DICT", - [_GUARD_NOS_DICT_r02] = "_GUARD_NOS_DICT_r02", - [_GUARD_NOS_DICT_r12] = "_GUARD_NOS_DICT_r12", - [_GUARD_NOS_DICT_r22] = "_GUARD_NOS_DICT_r22", - [_GUARD_NOS_DICT_r33] = "_GUARD_NOS_DICT_r33", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r03", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r13", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r23", + [_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_STORE_SUBSCRIPT_r33", + [_GUARD_NOS_DICT_SUBSCRIPT] = "_GUARD_NOS_DICT_SUBSCRIPT", + [_GUARD_NOS_DICT_SUBSCRIPT_r02] = "_GUARD_NOS_DICT_SUBSCRIPT_r02", + [_GUARD_NOS_DICT_SUBSCRIPT_r12] = "_GUARD_NOS_DICT_SUBSCRIPT_r12", + [_GUARD_NOS_DICT_SUBSCRIPT_r22] = "_GUARD_NOS_DICT_SUBSCRIPT_r22", + [_GUARD_NOS_DICT_SUBSCRIPT_r33] = "_GUARD_NOS_DICT_SUBSCRIPT_r33", [_GUARD_NOS_FLOAT] = "_GUARD_NOS_FLOAT", [_GUARD_NOS_FLOAT_r02] = "_GUARD_NOS_FLOAT_r02", [_GUARD_NOS_FLOAT_r12] = "_GUARD_NOS_FLOAT_r12", @@ -5331,6 +5445,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_NOS_INT_r12] = "_GUARD_NOS_INT_r12", [_GUARD_NOS_INT_r22] = "_GUARD_NOS_INT_r22", [_GUARD_NOS_INT_r33] = "_GUARD_NOS_INT_r33", + [_GUARD_NOS_ITER_VIRTUAL] = "_GUARD_NOS_ITER_VIRTUAL", + [_GUARD_NOS_ITER_VIRTUAL_r02] = "_GUARD_NOS_ITER_VIRTUAL_r02", + [_GUARD_NOS_ITER_VIRTUAL_r12] = "_GUARD_NOS_ITER_VIRTUAL_r12", + [_GUARD_NOS_ITER_VIRTUAL_r22] = "_GUARD_NOS_ITER_VIRTUAL_r22", + [_GUARD_NOS_ITER_VIRTUAL_r33] = "_GUARD_NOS_ITER_VIRTUAL_r33", [_GUARD_NOS_LIST] = "_GUARD_NOS_LIST", [_GUARD_NOS_LIST_r02] = "_GUARD_NOS_LIST_r02", [_GUARD_NOS_LIST_r12] = "_GUARD_NOS_LIST_r12", @@ -5401,6 +5520,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_DICT_r11] = "_GUARD_TOS_DICT_r11", [_GUARD_TOS_DICT_r22] = "_GUARD_TOS_DICT_r22", [_GUARD_TOS_DICT_r33] = "_GUARD_TOS_DICT_r33", + [_GUARD_TOS_EXACT_INT] = "_GUARD_TOS_EXACT_INT", + [_GUARD_TOS_EXACT_INT_r01] = "_GUARD_TOS_EXACT_INT_r01", + [_GUARD_TOS_EXACT_INT_r11] = "_GUARD_TOS_EXACT_INT_r11", + [_GUARD_TOS_EXACT_INT_r22] = "_GUARD_TOS_EXACT_INT_r22", + [_GUARD_TOS_EXACT_INT_r33] = "_GUARD_TOS_EXACT_INT_r33", [_GUARD_TOS_FLOAT] = "_GUARD_TOS_FLOAT", [_GUARD_TOS_FLOAT_r01] = "_GUARD_TOS_FLOAT_r01", [_GUARD_TOS_FLOAT_r11] = "_GUARD_TOS_FLOAT_r11", @@ -5421,11 +5545,21 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_INT_r11] = "_GUARD_TOS_INT_r11", [_GUARD_TOS_INT_r22] = "_GUARD_TOS_INT_r22", [_GUARD_TOS_INT_r33] = "_GUARD_TOS_INT_r33", + [_GUARD_TOS_IS_NONE] = "_GUARD_TOS_IS_NONE", + [_GUARD_TOS_IS_NONE_r01] = "_GUARD_TOS_IS_NONE_r01", + [_GUARD_TOS_IS_NONE_r11] = "_GUARD_TOS_IS_NONE_r11", + [_GUARD_TOS_IS_NONE_r22] = "_GUARD_TOS_IS_NONE_r22", + [_GUARD_TOS_IS_NONE_r33] = "_GUARD_TOS_IS_NONE_r33", [_GUARD_TOS_LIST] = "_GUARD_TOS_LIST", [_GUARD_TOS_LIST_r01] = "_GUARD_TOS_LIST_r01", [_GUARD_TOS_LIST_r11] = "_GUARD_TOS_LIST_r11", [_GUARD_TOS_LIST_r22] = "_GUARD_TOS_LIST_r22", [_GUARD_TOS_LIST_r33] = "_GUARD_TOS_LIST_r33", + [_GUARD_TOS_NOT_NULL] = "_GUARD_TOS_NOT_NULL", + [_GUARD_TOS_NOT_NULL_r01] = "_GUARD_TOS_NOT_NULL_r01", + [_GUARD_TOS_NOT_NULL_r11] = "_GUARD_TOS_NOT_NULL_r11", + [_GUARD_TOS_NOT_NULL_r22] = "_GUARD_TOS_NOT_NULL_r22", + [_GUARD_TOS_NOT_NULL_r33] = "_GUARD_TOS_NOT_NULL_r33", [_GUARD_TOS_OVERFLOWED] = "_GUARD_TOS_OVERFLOWED", [_GUARD_TOS_OVERFLOWED_r01] = "_GUARD_TOS_OVERFLOWED_r01", [_GUARD_TOS_OVERFLOWED_r11] = "_GUARD_TOS_OVERFLOWED_r11", @@ -5451,6 +5585,16 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_GUARD_TOS_UNICODE_r11] = "_GUARD_TOS_UNICODE_r11", [_GUARD_TOS_UNICODE_r22] = "_GUARD_TOS_UNICODE_r22", [_GUARD_TOS_UNICODE_r33] = "_GUARD_TOS_UNICODE_r33", + [_GUARD_TYPE] = "_GUARD_TYPE", + [_GUARD_TYPE_r01] = "_GUARD_TYPE_r01", + [_GUARD_TYPE_r11] = "_GUARD_TYPE_r11", + [_GUARD_TYPE_r22] = "_GUARD_TYPE_r22", + [_GUARD_TYPE_r33] = "_GUARD_TYPE_r33", + [_GUARD_TYPE_ITER] = "_GUARD_TYPE_ITER", + [_GUARD_TYPE_ITER_r02] = "_GUARD_TYPE_ITER_r02", + [_GUARD_TYPE_ITER_r12] = "_GUARD_TYPE_ITER_r12", + [_GUARD_TYPE_ITER_r22] = "_GUARD_TYPE_ITER_r22", + [_GUARD_TYPE_ITER_r33] = "_GUARD_TYPE_ITER_r33", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_GUARD_TYPE_VERSION_r01] = "_GUARD_TYPE_VERSION_r01", [_GUARD_TYPE_VERSION_r11] = "_GUARD_TYPE_VERSION_r11", @@ -5484,18 +5628,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_3_r01] = "_INIT_CALL_PY_EXACT_ARGS_3_r01", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", [_INIT_CALL_PY_EXACT_ARGS_4_r01] = "_INIT_CALL_PY_EXACT_ARGS_4_r01", - [_INSERT_1_LOAD_CONST_INLINE] = "_INSERT_1_LOAD_CONST_INLINE", - [_INSERT_1_LOAD_CONST_INLINE_r02] = "_INSERT_1_LOAD_CONST_INLINE_r02", - [_INSERT_1_LOAD_CONST_INLINE_r12] = "_INSERT_1_LOAD_CONST_INLINE_r12", - [_INSERT_1_LOAD_CONST_INLINE_r23] = "_INSERT_1_LOAD_CONST_INLINE_r23", - [_INSERT_1_LOAD_CONST_INLINE_BORROW] = "_INSERT_1_LOAD_CONST_INLINE_BORROW", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r02] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r02", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r12] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r12", - [_INSERT_1_LOAD_CONST_INLINE_BORROW_r23] = "_INSERT_1_LOAD_CONST_INLINE_BORROW_r23", - [_INSERT_2_LOAD_CONST_INLINE_BORROW] = "_INSERT_2_LOAD_CONST_INLINE_BORROW", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r03] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r03", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r13] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r13", - [_INSERT_2_LOAD_CONST_INLINE_BORROW_r23] = "_INSERT_2_LOAD_CONST_INLINE_BORROW_r23", [_INSERT_NULL] = "_INSERT_NULL", [_INSERT_NULL_r10] = "_INSERT_NULL_r10", [_IS_NONE] = "_IS_NONE", @@ -5519,6 +5651,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_ITER_CHECK_TUPLE_r12] = "_ITER_CHECK_TUPLE_r12", [_ITER_CHECK_TUPLE_r22] = "_ITER_CHECK_TUPLE_r22", [_ITER_CHECK_TUPLE_r33] = "_ITER_CHECK_TUPLE_r33", + [_ITER_NEXT_INLINE] = "_ITER_NEXT_INLINE", + [_ITER_NEXT_INLINE_r23] = "_ITER_NEXT_INLINE_r23", [_ITER_NEXT_LIST_TIER_TWO] = "_ITER_NEXT_LIST_TIER_TWO", [_ITER_NEXT_LIST_TIER_TWO_r23] = "_ITER_NEXT_LIST_TIER_TWO_r23", [_ITER_NEXT_RANGE] = "_ITER_NEXT_RANGE", @@ -5539,6 +5673,8 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_ATTR_r10] = "_LOAD_ATTR_r10", [_LOAD_ATTR_CLASS] = "_LOAD_ATTR_CLASS", [_LOAD_ATTR_CLASS_r11] = "_LOAD_ATTR_CLASS_r11", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME", + [_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11] = "_LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME_r11", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", [_LOAD_ATTR_INSTANCE_VALUE_r02] = "_LOAD_ATTR_INSTANCE_VALUE_r02", [_LOAD_ATTR_INSTANCE_VALUE_r12] = "_LOAD_ATTR_INSTANCE_VALUE_r12", @@ -5562,7 +5698,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", [_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11] = "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES_r11", [_LOAD_ATTR_PROPERTY_FRAME] = "_LOAD_ATTR_PROPERTY_FRAME", + [_LOAD_ATTR_PROPERTY_FRAME_r01] = "_LOAD_ATTR_PROPERTY_FRAME_r01", [_LOAD_ATTR_PROPERTY_FRAME_r11] = "_LOAD_ATTR_PROPERTY_FRAME_r11", + [_LOAD_ATTR_PROPERTY_FRAME_r22] = "_LOAD_ATTR_PROPERTY_FRAME_r22", + [_LOAD_ATTR_PROPERTY_FRAME_r33] = "_LOAD_ATTR_PROPERTY_FRAME_r33", [_LOAD_ATTR_SLOT] = "_LOAD_ATTR_SLOT", [_LOAD_ATTR_SLOT_r02] = "_LOAD_ATTR_SLOT_r02", [_LOAD_ATTR_SLOT_r12] = "_LOAD_ATTR_SLOT_r12", @@ -5719,7 +5858,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_MAKE_CELL] = "_MAKE_CELL", [_MAKE_CELL_r00] = "_MAKE_CELL_r00", [_MAKE_FUNCTION] = "_MAKE_FUNCTION", - [_MAKE_FUNCTION_r11] = "_MAKE_FUNCTION_r11", + [_MAKE_FUNCTION_r12] = "_MAKE_FUNCTION_r12", [_MAKE_HEAP_SAFE] = "_MAKE_HEAP_SAFE", [_MAKE_HEAP_SAFE_r01] = "_MAKE_HEAP_SAFE_r01", [_MAKE_HEAP_SAFE_r11] = "_MAKE_HEAP_SAFE_r11", @@ -5753,18 +5892,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_NOP_r11] = "_NOP_r11", [_NOP_r22] = "_NOP_r22", [_NOP_r33] = "_NOP_r33", - [_POP_CALL] = "_POP_CALL", - [_POP_CALL_r20] = "_POP_CALL_r20", - [_POP_CALL_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_LOAD_CONST_INLINE_BORROW_r21] = "_POP_CALL_LOAD_CONST_INLINE_BORROW_r21", - [_POP_CALL_ONE] = "_POP_CALL_ONE", - [_POP_CALL_ONE_r30] = "_POP_CALL_ONE_r30", - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31] = "_POP_CALL_ONE_LOAD_CONST_INLINE_BORROW_r31", - [_POP_CALL_TWO] = "_POP_CALL_TWO", - [_POP_CALL_TWO_r30] = "_POP_CALL_TWO_r30", - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW", - [_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31] = "_POP_CALL_TWO_LOAD_CONST_INLINE_BORROW_r31", [_POP_EXCEPT] = "_POP_EXCEPT", [_POP_EXCEPT_r10] = "_POP_EXCEPT_r10", [_POP_ITER] = "_POP_ITER", @@ -5781,24 +5908,18 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_POP_TOP_INT_r10] = "_POP_TOP_INT_r10", [_POP_TOP_INT_r21] = "_POP_TOP_INT_r21", [_POP_TOP_INT_r32] = "_POP_TOP_INT_r32", - [_POP_TOP_LOAD_CONST_INLINE] = "_POP_TOP_LOAD_CONST_INLINE", - [_POP_TOP_LOAD_CONST_INLINE_r11] = "_POP_TOP_LOAD_CONST_INLINE_r11", - [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", - [_POP_TOP_LOAD_CONST_INLINE_BORROW_r11] = "_POP_TOP_LOAD_CONST_INLINE_BORROW_r11", [_POP_TOP_NOP] = "_POP_TOP_NOP", [_POP_TOP_NOP_r00] = "_POP_TOP_NOP_r00", [_POP_TOP_NOP_r10] = "_POP_TOP_NOP_r10", [_POP_TOP_NOP_r21] = "_POP_TOP_NOP_r21", [_POP_TOP_NOP_r32] = "_POP_TOP_NOP_r32", + [_POP_TOP_OPARG] = "_POP_TOP_OPARG", + [_POP_TOP_OPARG_r00] = "_POP_TOP_OPARG_r00", [_POP_TOP_UNICODE] = "_POP_TOP_UNICODE", [_POP_TOP_UNICODE_r00] = "_POP_TOP_UNICODE_r00", [_POP_TOP_UNICODE_r10] = "_POP_TOP_UNICODE_r10", [_POP_TOP_UNICODE_r21] = "_POP_TOP_UNICODE_r21", [_POP_TOP_UNICODE_r32] = "_POP_TOP_UNICODE_r32", - [_POP_TWO] = "_POP_TWO", - [_POP_TWO_r20] = "_POP_TWO_r20", - [_POP_TWO_LOAD_CONST_INLINE_BORROW] = "_POP_TWO_LOAD_CONST_INLINE_BORROW", - [_POP_TWO_LOAD_CONST_INLINE_BORROW_r21] = "_POP_TWO_LOAD_CONST_INLINE_BORROW_r21", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_EXC_INFO_r02] = "_PUSH_EXC_INFO_r02", [_PUSH_EXC_INFO_r12] = "_PUSH_EXC_INFO_r12", @@ -5811,6 +5932,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_PUSH_NULL_r23] = "_PUSH_NULL_r23", [_PUSH_NULL_CONDITIONAL] = "_PUSH_NULL_CONDITIONAL", [_PUSH_NULL_CONDITIONAL_r00] = "_PUSH_NULL_CONDITIONAL_r00", + [_PUSH_TAGGED_ZERO] = "_PUSH_TAGGED_ZERO", + [_PUSH_TAGGED_ZERO_r01] = "_PUSH_TAGGED_ZERO_r01", + [_PUSH_TAGGED_ZERO_r12] = "_PUSH_TAGGED_ZERO_r12", + [_PUSH_TAGGED_ZERO_r23] = "_PUSH_TAGGED_ZERO_r23", [_PY_FRAME_EX] = "_PY_FRAME_EX", [_PY_FRAME_EX_r31] = "_PY_FRAME_EX_r31", [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", @@ -5821,9 +5946,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_RECORD_4OS] = "_RECORD_4OS", [_RECORD_BOUND_METHOD] = "_RECORD_BOUND_METHOD", [_RECORD_CALLABLE] = "_RECORD_CALLABLE", + [_RECORD_CALLABLE_KW] = "_RECORD_CALLABLE_KW", [_RECORD_CODE] = "_RECORD_CODE", [_RECORD_NOS] = "_RECORD_NOS", [_RECORD_NOS_GEN_FUNC] = "_RECORD_NOS_GEN_FUNC", + [_RECORD_NOS_TYPE] = "_RECORD_NOS_TYPE", [_RECORD_TOS] = "_RECORD_TOS", [_RECORD_TOS_TYPE] = "_RECORD_TOS_TYPE", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", @@ -5839,13 +5966,25 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_RETURN_GENERATOR_r01] = "_RETURN_GENERATOR_r01", [_RETURN_VALUE] = "_RETURN_VALUE", [_RETURN_VALUE_r11] = "_RETURN_VALUE_r11", + [_RROT_3] = "_RROT_3", + [_RROT_3_r03] = "_RROT_3_r03", + [_RROT_3_r13] = "_RROT_3_r13", + [_RROT_3_r23] = "_RROT_3_r23", + [_RROT_3_r33] = "_RROT_3_r33", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_SAVE_RETURN_OFFSET_r00] = "_SAVE_RETURN_OFFSET_r00", [_SAVE_RETURN_OFFSET_r11] = "_SAVE_RETURN_OFFSET_r11", [_SAVE_RETURN_OFFSET_r22] = "_SAVE_RETURN_OFFSET_r22", [_SAVE_RETURN_OFFSET_r33] = "_SAVE_RETURN_OFFSET_r33", + [_SEND_ASYNC_GEN_TIER_TWO] = "_SEND_ASYNC_GEN_TIER_TWO", + [_SEND_ASYNC_GEN_TIER_TWO_r33] = "_SEND_ASYNC_GEN_TIER_TWO_r33", [_SEND_GEN_FRAME] = "_SEND_GEN_FRAME", [_SEND_GEN_FRAME_r33] = "_SEND_GEN_FRAME_r33", + [_SEND_VIRTUAL_TIER_TWO] = "_SEND_VIRTUAL_TIER_TWO", + [_SEND_VIRTUAL_TIER_TWO_r03] = "_SEND_VIRTUAL_TIER_TWO_r03", + [_SEND_VIRTUAL_TIER_TWO_r13] = "_SEND_VIRTUAL_TIER_TWO_r13", + [_SEND_VIRTUAL_TIER_TWO_r23] = "_SEND_VIRTUAL_TIER_TWO_r23", + [_SEND_VIRTUAL_TIER_TWO_r33] = "_SEND_VIRTUAL_TIER_TWO_r33", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", [_SETUP_ANNOTATIONS_r00] = "_SETUP_ANNOTATIONS_r00", [_SET_ADD] = "_SET_ADD", @@ -5862,16 +6001,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SET_IP_r33] = "_SET_IP_r33", [_SET_UPDATE] = "_SET_UPDATE", [_SET_UPDATE_r11] = "_SET_UPDATE_r11", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r02", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r12", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r22", - [_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32] = "_SHUFFLE_2_LOAD_CONST_INLINE_BORROW_r32", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33", [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", [_SPILL_OR_RELOAD_r02] = "_SPILL_OR_RELOAD_r02", @@ -6125,8 +6254,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _POP_TOP_UNICODE: return 1; - case _POP_TWO: - return 2; + case _POP_TOP_OPARG: + return oparg; case _PUSH_NULL: return 0; case _END_FOR: @@ -6173,6 +6302,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TOS_INT: return 0; + case _GUARD_TOS_EXACT_INT: + return 0; case _GUARD_NOS_OVERFLOWED: return 0; case _GUARD_TOS_OVERFLOWED: @@ -6217,10 +6348,20 @@ int _PyUop_num_popped(int opcode, int oparg) return 2; case _BINARY_OP_SUBTRACT_FLOAT_INPLACE_RIGHT: return 2; + case _BINARY_OP_TRUEDIV_FLOAT: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE: + return 2; + case _BINARY_OP_TRUEDIV_FLOAT_INPLACE_RIGHT: + return 2; case _BINARY_OP_ADD_UNICODE: return 2; case _BINARY_OP_INPLACE_ADD_UNICODE: return 2; + case _GUARD_BINARY_OP_EXTEND_LHS: + return 0; + case _GUARD_BINARY_OP_EXTEND_RHS: + return 0; case _GUARD_BINARY_OP_EXTEND: return 0; case _BINARY_OP_EXTEND: @@ -6245,9 +6386,9 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _BINARY_OP_SUBSCR_TUPLE_INT: return 2; - case _GUARD_NOS_DICT: + case _GUARD_NOS_DICT_SUBSCRIPT: return 0; - case _GUARD_NOS_ANY_DICT: + case _GUARD_NOS_DICT_STORE_SUBSCRIPT: return 0; case _GUARD_TOS_ANY_DICT: return 0; @@ -6293,6 +6434,16 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _SEND_GEN_FRAME: return 1; + case _GUARD_TOS_IS_NONE: + return 0; + case _GUARD_NOS_NOT_NULL: + return 0; + case _SEND_VIRTUAL_TIER_TWO: + return 1; + case _GUARD_3OS_ASYNC_GEN_ASEND: + return 0; + case _SEND_ASYNC_GEN_TIER_TWO: + return 3; case _YIELD_VALUE: return 1; case _POP_EXCEPT: @@ -6303,8 +6454,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _STORE_NAME: return 1; - case _DELETE_NAME: - return 0; case _UNPACK_SEQUENCE: return 1; case _UNPACK_SEQUENCE_TWO_TUPLE: @@ -6323,12 +6472,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _STORE_ATTR: return 2; - case _DELETE_ATTR: - return 1; case _STORE_GLOBAL: return 1; - case _DELETE_GLOBAL: - return 0; case _LOAD_LOCALS: return 0; case _LOAD_NAME: @@ -6397,6 +6542,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GUARD_TYPE_VERSION_LOCKED: return 0; + case _GUARD_TYPE: + return 0; case _CHECK_MANAGED_OBJECT_HAS_VALUES: return 0; case _LOAD_ATTR_INSTANCE_VALUE: @@ -6413,6 +6560,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _LOAD_ATTR_PROPERTY_FRAME: return 1; + case _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN_FRAME: + return 1; case _GUARD_DORV_NO_DICT: return 0; case _STORE_ATTR_INSTANCE_VALUE: @@ -6467,8 +6616,26 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _GET_ITER: return 1; + case _GUARD_ITERATOR: + return 0; + case _GUARD_ITER_VIRTUAL: + return 0; + case _PUSH_TAGGED_ZERO: + return 0; + case _GET_ITER_TRAD: + return 1; case _FOR_ITER_TIER_TWO: return 0; + case _GUARD_TYPE_ITER: + return 0; + case _ITER_NEXT_INLINE: + return 0; + case _GUARD_NOS_ITER_VIRTUAL: + return 0; + case _GUARD_TOS_NOT_NULL: + return 0; + case _FOR_ITER_VIRTUAL_TIER_TWO: + return 0; case _ITER_CHECK_LIST: return 0; case _GUARD_NOT_EXHAUSTED_LIST: @@ -6497,10 +6664,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _PUSH_EXC_INFO: return 1; - case _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT: - return 0; - case _GUARD_KEYS_VERSION: - return 0; case _LOAD_ATTR_METHOD_WITH_VALUES: return 1; case _LOAD_ATTR_METHOD_NO_DICT: @@ -6557,8 +6720,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GUARD_NOS_NULL: return 0; - case _GUARD_NOS_NOT_NULL: - return 0; case _GUARD_THIRD_NULL: return 0; case _GUARD_CALLABLE_TYPE_1: @@ -6573,14 +6734,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _CALL_TUPLE_1: return 3; - case _CHECK_AND_ALLOCATE_OBJECT: + case _CHECK_OBJECT: + return 0; + case _ALLOCATE_OBJECT: return 0; case _CREATE_INIT_FRAME: return 2 + oparg; case _EXIT_INIT_CHECK: return 1; + case _GUARD_CALLABLE_BUILTIN_CLASS: + return 0; case _CALL_BUILTIN_CLASS: - return 2 + oparg; + return 0; case _GUARD_CALLABLE_BUILTIN_O: return 0; case _CALL_BUILTIN_O: @@ -6588,11 +6753,11 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_BUILTIN_FAST: return 0; case _CALL_BUILTIN_FAST: - return 2 + oparg; + return 0; case _GUARD_CALLABLE_BUILTIN_FAST_WITH_KEYWORDS: return 0; case _CALL_BUILTIN_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; case _GUARD_CALLABLE_LEN: return 0; case _CALL_LEN: @@ -6616,9 +6781,9 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: - return 2 + oparg; + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_INLINE: - return 1 + oparg; + return 0; case _GUARD_CALLABLE_METHOD_DESCRIPTOR_NOARGS: return 0; case _CALL_METHOD_DESCRIPTOR_NOARGS: @@ -6628,9 +6793,9 @@ int _PyUop_num_popped(int opcode, int oparg) case _GUARD_CALLABLE_METHOD_DESCRIPTOR_FAST: return 0; case _CALL_METHOD_DESCRIPTOR_FAST: - return 2 + oparg; + return 0; case _CALL_METHOD_DESCRIPTOR_FAST_INLINE: - return 1 + oparg; + return 0; case _MAYBE_EXPAND_METHOD_KW: return 0; case _PY_FRAME_KW: @@ -6729,36 +6894,10 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_CONST_INLINE: return 0; - case _POP_TOP_LOAD_CONST_INLINE: - return 1; case _LOAD_CONST_INLINE_BORROW: return 0; - case _POP_CALL: - return 2; - case _POP_CALL_ONE: - return 3; - case _POP_CALL_TWO: - return 4; - case _POP_TOP_LOAD_CONST_INLINE_BORROW: - return 1; - case _POP_TWO_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_LOAD_CONST_INLINE_BORROW: - return 2; - case _POP_CALL_ONE_LOAD_CONST_INLINE_BORROW: - return 3; - case _INSERT_1_LOAD_CONST_INLINE: - return 1; - case _INSERT_1_LOAD_CONST_INLINE_BORROW: - return 1; - case _INSERT_2_LOAD_CONST_INLINE_BORROW: - return 2; - case _SHUFFLE_2_LOAD_CONST_INLINE_BORROW: - return 3; - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW: - return 3; - case _POP_CALL_TWO_LOAD_CONST_INLINE_BORROW: - return 4; + case _RROT_3: + return 0; case _START_EXECUTOR: return 0; case _MAKE_WARM: @@ -6801,6 +6940,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _RECORD_NOS: return 0; + case _RECORD_NOS_TYPE: + return 0; case _RECORD_NOS_GEN_FUNC: return 0; case _RECORD_3OS_GEN_FUNC: @@ -6809,6 +6950,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _RECORD_CALLABLE: return 0; + case _RECORD_CALLABLE_KW: + return 0; case _RECORD_BOUND_METHOD: return 0; case _RECORD_CODE: diff --git a/Include/moduleobject.h b/Include/moduleobject.h index c2fb1f85165f7d2..88c66672ff164a8 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -73,31 +73,6 @@ struct PyModuleDef_Slot { void *value; }; -#define Py_mod_create 1 -#define Py_mod_exec 2 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 -# define Py_mod_multiple_interpreters 3 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 -# define Py_mod_gil 4 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -# define Py_mod_abi 5 -# define Py_mod_name 6 -# define Py_mod_doc 7 -# define Py_mod_state_size 8 -# define Py_mod_methods 9 -# define Py_mod_state_traverse 10 -# define Py_mod_state_clear 11 -# define Py_mod_state_free 12 -# define Py_mod_token 13 -#endif - - -#ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 13 -#endif - #endif /* New in 3.5 */ /* for Py_mod_multiple_interpreters: */ @@ -120,7 +95,7 @@ PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) -PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PyModuleDef_Slot *slots, +PyAPI_FUNC(PyObject *) PyModule_FromSlotsAndSpec(const PySlot *slots, PyObject *spec); PyAPI_FUNC(int) PyModule_Exec(PyObject *module); PyAPI_FUNC(int) PyModule_GetStateSize(PyObject *module, Py_ssize_t *result); diff --git a/Include/object.h b/Include/object.h index e33f92295fb7711..549b91ebecf4f22 100644 --- a/Include/object.h +++ b/Include/object.h @@ -306,6 +306,11 @@ typedef Py_hash_t (*hashfunc)(PyObject *); typedef PyObject *(*richcmpfunc) (PyObject *, PyObject *, int); typedef PyObject *(*getiterfunc) (PyObject *); typedef PyObject *(*iternextfunc) (PyObject *); +typedef struct { + PyObject *object; + Py_ssize_t index; +} _PyObjectIndexPair; +typedef _PyObjectIndexPair (*_Py_iteritemfunc) (PyObject *, Py_ssize_t index); typedef PyObject *(*descrgetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*initproc)(PyObject *, PyObject *, PyObject *); @@ -359,6 +364,9 @@ PyAPI_FUNC(Py_ssize_t) PyType_GetTypeDataSize(PyTypeObject *cls); PyAPI_FUNC(int) PyType_GetBaseByToken(PyTypeObject *, void *, PyTypeObject **); #define Py_TP_USE_SPEC NULL #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(PyObject *) PyType_FromSlots(struct PySlot *slots); +#endif /* Generic type check */ PyAPI_FUNC(int) PyType_IsSubtype(PyTypeObject *, PyTypeObject *); @@ -482,7 +490,7 @@ given type object has a specified feature. #define Py_TPFLAGS_SEQUENCE (1 << 5) /* Set if instances of the type object are treated as mappings for pattern matching */ #define Py_TPFLAGS_MAPPING (1 << 6) -#endif +#endif // Py_LIMITED_API /* Disallow creating instances of the type: set tp_new to NULL and don't create * the "__new__" key in the type dictionary. */ @@ -777,6 +785,7 @@ PyAPI_FUNC(int) PyType_Freeze(PyTypeObject *type); #endif #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +PyAPI_FUNC(int) PyObject_CallFinalizerFromDealloc(PyObject *); PyAPI_FUNC(PyObject *) PyType_GetModuleByToken(PyTypeObject *type, const void *token); PyAPI_FUNC(void *) PyObject_GetTypeData_DuringGC(PyObject *obj, diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index ac6d4d964d3b5e8..11342ae451b9f6c 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -69,67 +69,64 @@ extern "C" { #define CONVERT_VALUE 56 #define COPY 57 #define COPY_FREE_VARS 58 -#define DELETE_ATTR 59 -#define DELETE_DEREF 60 -#define DELETE_FAST 61 -#define DELETE_GLOBAL 62 -#define DELETE_NAME 63 -#define DICT_MERGE 64 -#define DICT_UPDATE 65 -#define END_ASYNC_FOR 66 -#define EXTENDED_ARG 67 -#define FOR_ITER 68 -#define GET_AWAITABLE 69 -#define GET_ITER 70 -#define IMPORT_FROM 71 -#define IMPORT_NAME 72 -#define IS_OP 73 -#define JUMP_BACKWARD 74 -#define JUMP_BACKWARD_NO_INTERRUPT 75 -#define JUMP_FORWARD 76 -#define LIST_APPEND 77 -#define LIST_EXTEND 78 -#define LOAD_ATTR 79 -#define LOAD_COMMON_CONSTANT 80 -#define LOAD_CONST 81 -#define LOAD_DEREF 82 -#define LOAD_FAST 83 -#define LOAD_FAST_AND_CLEAR 84 -#define LOAD_FAST_BORROW 85 -#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 86 -#define LOAD_FAST_CHECK 87 -#define LOAD_FAST_LOAD_FAST 88 -#define LOAD_FROM_DICT_OR_DEREF 89 -#define LOAD_FROM_DICT_OR_GLOBALS 90 -#define LOAD_GLOBAL 91 -#define LOAD_NAME 92 -#define LOAD_SMALL_INT 93 -#define LOAD_SPECIAL 94 -#define LOAD_SUPER_ATTR 95 -#define MAKE_CELL 96 -#define MAP_ADD 97 -#define MATCH_CLASS 98 -#define POP_JUMP_IF_FALSE 99 -#define POP_JUMP_IF_NONE 100 -#define POP_JUMP_IF_NOT_NONE 101 -#define POP_JUMP_IF_TRUE 102 -#define RAISE_VARARGS 103 -#define RERAISE 104 -#define SEND 105 -#define SET_ADD 106 -#define SET_FUNCTION_ATTRIBUTE 107 -#define SET_UPDATE 108 -#define STORE_ATTR 109 -#define STORE_DEREF 110 -#define STORE_FAST 111 -#define STORE_FAST_LOAD_FAST 112 -#define STORE_FAST_STORE_FAST 113 -#define STORE_GLOBAL 114 -#define STORE_NAME 115 -#define SWAP 116 -#define UNPACK_EX 117 -#define UNPACK_SEQUENCE 118 -#define YIELD_VALUE 119 +#define DELETE_DEREF 59 +#define DELETE_FAST 60 +#define DICT_MERGE 61 +#define DICT_UPDATE 62 +#define END_ASYNC_FOR 63 +#define EXTENDED_ARG 64 +#define FOR_ITER 65 +#define GET_AWAITABLE 66 +#define GET_ITER 67 +#define IMPORT_FROM 68 +#define IMPORT_NAME 69 +#define IS_OP 70 +#define JUMP_BACKWARD 71 +#define JUMP_BACKWARD_NO_INTERRUPT 72 +#define JUMP_FORWARD 73 +#define LIST_APPEND 74 +#define LIST_EXTEND 75 +#define LOAD_ATTR 76 +#define LOAD_COMMON_CONSTANT 77 +#define LOAD_CONST 78 +#define LOAD_DEREF 79 +#define LOAD_FAST 80 +#define LOAD_FAST_AND_CLEAR 81 +#define LOAD_FAST_BORROW 82 +#define LOAD_FAST_BORROW_LOAD_FAST_BORROW 83 +#define LOAD_FAST_CHECK 84 +#define LOAD_FAST_LOAD_FAST 85 +#define LOAD_FROM_DICT_OR_DEREF 86 +#define LOAD_FROM_DICT_OR_GLOBALS 87 +#define LOAD_GLOBAL 88 +#define LOAD_NAME 89 +#define LOAD_SMALL_INT 90 +#define LOAD_SPECIAL 91 +#define LOAD_SUPER_ATTR 92 +#define MAKE_CELL 93 +#define MAP_ADD 94 +#define MATCH_CLASS 95 +#define POP_JUMP_IF_FALSE 96 +#define POP_JUMP_IF_NONE 97 +#define POP_JUMP_IF_NOT_NONE 98 +#define POP_JUMP_IF_TRUE 99 +#define RAISE_VARARGS 100 +#define RERAISE 101 +#define SEND 102 +#define SET_ADD 103 +#define SET_FUNCTION_ATTRIBUTE 104 +#define SET_UPDATE 105 +#define STORE_ATTR 106 +#define STORE_DEREF 107 +#define STORE_FAST 108 +#define STORE_FAST_LOAD_FAST 109 +#define STORE_FAST_STORE_FAST 110 +#define STORE_GLOBAL 111 +#define STORE_NAME 112 +#define SWAP 113 +#define UNPACK_EX 114 +#define UNPACK_SEQUENCE 115 +#define YIELD_VALUE 116 #define RESUME 128 #define BINARY_OP_ADD_FLOAT 129 #define BINARY_OP_ADD_INT 130 @@ -180,42 +177,47 @@ extern "C" { #define FOR_ITER_LIST 175 #define FOR_ITER_RANGE 176 #define FOR_ITER_TUPLE 177 -#define JUMP_BACKWARD_JIT 178 -#define JUMP_BACKWARD_NO_JIT 179 -#define LOAD_ATTR_CLASS 180 -#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 181 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 182 -#define LOAD_ATTR_INSTANCE_VALUE 183 -#define LOAD_ATTR_METHOD_LAZY_DICT 184 -#define LOAD_ATTR_METHOD_NO_DICT 185 -#define LOAD_ATTR_METHOD_WITH_VALUES 186 -#define LOAD_ATTR_MODULE 187 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 188 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 189 -#define LOAD_ATTR_PROPERTY 190 -#define LOAD_ATTR_SLOT 191 -#define LOAD_ATTR_WITH_HINT 192 -#define LOAD_GLOBAL_BUILTIN 193 -#define LOAD_GLOBAL_MODULE 194 -#define LOAD_SUPER_ATTR_ATTR 195 -#define LOAD_SUPER_ATTR_METHOD 196 -#define RESUME_CHECK 197 -#define RESUME_CHECK_JIT 198 -#define SEND_GEN 199 -#define STORE_ATTR_INSTANCE_VALUE 200 -#define STORE_ATTR_SLOT 201 -#define STORE_ATTR_WITH_HINT 202 -#define STORE_SUBSCR_DICT 203 -#define STORE_SUBSCR_LIST_INT 204 -#define TO_BOOL_ALWAYS_TRUE 205 -#define TO_BOOL_BOOL 206 -#define TO_BOOL_INT 207 -#define TO_BOOL_LIST 208 -#define TO_BOOL_NONE 209 -#define TO_BOOL_STR 210 -#define UNPACK_SEQUENCE_LIST 211 -#define UNPACK_SEQUENCE_TUPLE 212 -#define UNPACK_SEQUENCE_TWO_TUPLE 213 +#define FOR_ITER_VIRTUAL 178 +#define GET_ITER_SELF 179 +#define GET_ITER_VIRTUAL 180 +#define JUMP_BACKWARD_JIT 181 +#define JUMP_BACKWARD_NO_JIT 182 +#define LOAD_ATTR_CLASS 183 +#define LOAD_ATTR_CLASS_WITH_METACLASS_CHECK 184 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 185 +#define LOAD_ATTR_INSTANCE_VALUE 186 +#define LOAD_ATTR_METHOD_LAZY_DICT 187 +#define LOAD_ATTR_METHOD_NO_DICT 188 +#define LOAD_ATTR_METHOD_WITH_VALUES 189 +#define LOAD_ATTR_MODULE 190 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 191 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 192 +#define LOAD_ATTR_PROPERTY 193 +#define LOAD_ATTR_SLOT 194 +#define LOAD_ATTR_WITH_HINT 195 +#define LOAD_GLOBAL_BUILTIN 196 +#define LOAD_GLOBAL_MODULE 197 +#define LOAD_SUPER_ATTR_ATTR 198 +#define LOAD_SUPER_ATTR_METHOD 199 +#define RESUME_CHECK 200 +#define RESUME_CHECK_JIT 201 +#define SEND_ASYNC_GEN 202 +#define SEND_GEN 203 +#define SEND_VIRTUAL 204 +#define STORE_ATTR_INSTANCE_VALUE 205 +#define STORE_ATTR_SLOT 206 +#define STORE_ATTR_WITH_HINT 207 +#define STORE_SUBSCR_DICT 208 +#define STORE_SUBSCR_LIST_INT 209 +#define TO_BOOL_ALWAYS_TRUE 210 +#define TO_BOOL_BOOL 211 +#define TO_BOOL_INT 212 +#define TO_BOOL_LIST 213 +#define TO_BOOL_NONE 214 +#define TO_BOOL_STR 215 +#define UNPACK_SEQUENCE_LIST 216 +#define UNPACK_SEQUENCE_TUPLE 217 +#define UNPACK_SEQUENCE_TWO_TUPLE 218 #define INSTRUMENTED_END_FOR 233 #define INSTRUMENTED_POP_ITER 234 #define INSTRUMENTED_END_SEND 235 diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 9f5c36230a7e45d..d82d52584069413 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -21,13 +21,13 @@ /* Version parsed out into numeric values */ /*--start constants--*/ #define PY_MAJOR_VERSION 3 -#define PY_MINOR_VERSION 15 +#define PY_MINOR_VERSION 16 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 8 +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.15.0a8+" +#define PY_VERSION "3.16.0a0" /*--end constants--*/ @@ -61,32 +61,4 @@ #define PYTHON_ABI_VERSION 3 #define PYTHON_ABI_STRING "3" - -/* Stable ABI for free-threaded builds (introduced in PEP 803) - is enabled by one of: - - Py_TARGET_ABI3T, or - - Py_LIMITED_API and Py_GIL_DISABLED. - "Output" macros to be used internally: - - Py_LIMITED_API (defines the subset of API we expose) - - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between - free-threaded & GIL) - (Don't use Py_TARGET_ABI3T directly: it's currently only used to set these - 2 macros. It's also available for users' convenience.) - */ -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ - && !defined(Py_TARGET_ABI3T) -# define Py_TARGET_ABI3T Py_LIMITED_API -#endif -#if defined(Py_TARGET_ABI3T) -# define _Py_OPAQUE_PYOBJECT -# if !defined(Py_LIMITED_API) -# define Py_LIMITED_API Py_TARGET_ABI3T -# elif Py_LIMITED_API > Py_TARGET_ABI3T - // if both are defined, use the *lower* version, - // i.e. maximum compatibility -# undef Py_LIMITED_API -# define Py_LIMITED_API Py_TARGET_ABI3T -# endif -#endif - #endif //_Py_PATCHLEVEL_H diff --git a/Include/py_curses.h b/Include/py_curses.h index 0948aabedd49939..85a540086c83844 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -36,6 +36,22 @@ #define NCURSES_OPAQUE 0 #endif +/* PDCurses exposes its ncurses-compatible mouse API, the one this module uses, + only when this is defined before the curses header is included below. + Ignored by other curses implementations. */ +#ifndef PDC_NCMOUSE +# define PDC_NCMOUSE +#endif + +/* On Solaris/illumos, the SVr4 does "typedef char bool;", which + clashes with C's bool from . Define _BOOL to suppress it, and + include for the bool the header then needs. ncurses ignores + _BOOL. */ +#if defined(__sun) && !defined(_BOOL) +# include +# define _BOOL +#endif + #if defined(HAVE_NCURSESW_NCURSES_H) # include #elif defined(HAVE_NCURSESW_CURSES_H) @@ -80,8 +96,18 @@ typedef struct PyCursesWindowObject { WINDOW *win; char *encoding; struct PyCursesWindowObject *orig; + PyObject *screen; /* the screen the window belongs to, or NULL, + kept alive for the lifetime of the window */ } PyCursesWindowObject; +typedef struct { + PyObject_HEAD + SCREEN *screen; /* NULL after the screen has been deleted */ + FILE *outfp; /* owned output stream, or NULL */ + FILE *infp; /* owned input stream, or NULL */ + PyObject *stdscr_win; /* the screen's standard window, or NULL */ +} PyCursesScreenObject; + #define PyCurses_CAPSULE_NAME "_curses._C_API" diff --git a/Include/pyabi.h b/Include/pyabi.h new file mode 100644 index 000000000000000..21a6ab0c1ee6ea2 --- /dev/null +++ b/Include/pyabi.h @@ -0,0 +1,123 @@ +/* Macros that restrict available definitions and select implementations + * to match an ABI stability promise: + * + * - internal API/ABI (may change at any time) -- Py_BUILD_CORE* + * - general CPython API/ABI (may change in 3.x.0) -- default + * - Stable ABI: abi3, abi3t (long-term stable) -- Py_LIMITED_API, + * Py_TARGET_ABI3T, _Py_OPAQUE_PYOBJECT + * - Free-threading (incompatible with non-free-threading builds) + * -- Py_GIL_DISABLED + */ + +#ifndef _Py_PYABI_H +#define _Py_PYABI_H + +/* Defines to build Python and its standard library: + * + * - Py_BUILD_CORE: Build Python core. Gives access to Python internals; should + * not be used by third-party modules. + * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. + * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. + * + * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. + * + * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas + * Py_BUILD_CORE_BUILTIN does not. + */ +#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif +#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) +# define Py_BUILD_CORE +#endif + +/* Check valid values for target ABI macros. + */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 3 + // Empty Py_LIMITED_API used to work; redefine to + // Python 3.2 to be explicit. +# undef Py_LIMITED_API +# define Py_LIMITED_API 0x03020000 +#endif +#if defined(Py_TARGET_ABI3T) && Py_TARGET_ABI3T+0 < 0x030f0000 +# error "Py_TARGET_ABI3T must be 0x030f0000 (3.15) or above" +#endif + +/* Stable ABI for free-threaded builds (abi3t, introduced in PEP 803) + * is enabled by one of: + * - Py_TARGET_ABI3T, or + * - Py_LIMITED_API and Py_GIL_DISABLED. + * + * These affect set the following, which Python.h should use internally: + * - Py_LIMITED_API (defines the subset of API we expose) + * - _Py_OPAQUE_PYOBJECT (additionally hides what's ABI-incompatible between + * free-threaded & GIL) + * + * (Don't use Py_TARGET_ABI3T directly. It's currently only used to set these + * 2 macros, and defined for users' convenience.) + * + * This logic is currently partially duplicated in PC/pyconfig.h. + */ +#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) \ + && !defined(Py_TARGET_ABI3T) +# define Py_TARGET_ABI3T Py_LIMITED_API +#endif +#if defined(Py_TARGET_ABI3T) +# define _Py_OPAQUE_PYOBJECT +# if !defined(Py_LIMITED_API) +# define Py_LIMITED_API Py_TARGET_ABI3T +# elif Py_LIMITED_API > Py_TARGET_ABI3T + // if both are defined, use the *lower* version, + // i.e. maximum compatibility +# undef Py_LIMITED_API +# define Py_LIMITED_API Py_TARGET_ABI3T +# endif +#else +# ifdef _Py_OPAQUE_PYOBJECT + // _Py_OPAQUE_PYOBJECT is a private macro; do not define it directly. +# error "Define Py_TARGET_ABI3T to target abi3t." +# endif +#endif + +#if defined(Py_TARGET_ABI3T) +# if !defined(Py_GIL_DISABLED) + // Define Py_GIL_DISABLED for users' needs. Users check this macro to see + // whether they need extra synchronization. +# define Py_GIL_DISABLED +# endif +# if defined(_Py_IS_TESTCEXT) + // When compiling for abi3t, contents of Python.h should not depend + // on Py_GIL_DISABLED. + // We ask GCC to error if it sees the macro from this point on. + // Since users are free to the macro, and there's no way to undo the + // poisoning at the end of Python.h, we only do this in a test module + // (test_cext). + // + // Clang's poisoning is stricter than GCC's: it looks in `#elif` + // expressions after matching `#if`s. We disable it for now. + // We also provide an undocumented, unsupported opt-out macro to help + // porting to other compilers. Consider reaching out if you use it. +# if defined(__GNUC__) && !defined(__clang__) && !defined(_Py_NO_GCC_POISON) +# undef Py_GIL_DISABLED +# pragma GCC poison Py_GIL_DISABLED +# endif +# endif +#endif + +/* The internal C API must not be used with the limited C API: make sure + * that Py_BUILD_CORE* macros are not defined in this case. + * But, keep the "original" values, under different names, for "exports.h" + */ +#ifdef Py_BUILD_CORE +# define _PyEXPORTS_CORE +#endif +#ifdef Py_BUILD_CORE_MODULE +# define _PyEXPORTS_CORE_MODULE +#endif +#ifdef Py_LIMITED_API +# undef Py_BUILD_CORE +# undef Py_BUILD_CORE_BUILTIN +# undef Py_BUILD_CORE_MODULE +#endif + +#endif // _Py_PYABI_H diff --git a/Include/pyexpat.h b/Include/pyexpat.h index f523f8bb273983a..a676e16a7a457ea 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -62,6 +62,9 @@ struct PyExpat_CAPI XML_Parser parser, unsigned long long activationThresholdBytes); XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Include/pyport.h b/Include/pyport.h index 62cba4c1421f99f..73a3e6cdaf09200 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -58,34 +58,6 @@ #endif -/* Defines to build Python and its standard library: - * - * - Py_BUILD_CORE: Build Python core. Give access to Python internals, but - * should not be used by third-party modules. - * - Py_BUILD_CORE_BUILTIN: Build a Python stdlib module as a built-in module. - * - Py_BUILD_CORE_MODULE: Build a Python stdlib module as a dynamic library. - * - * Py_BUILD_CORE_BUILTIN and Py_BUILD_CORE_MODULE imply Py_BUILD_CORE. - * - * On Windows, Py_BUILD_CORE_MODULE exports "PyInit_xxx" symbol, whereas - * Py_BUILD_CORE_BUILTIN does not. - */ -#if defined(Py_BUILD_CORE_BUILTIN) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif -#if defined(Py_BUILD_CORE_MODULE) && !defined(Py_BUILD_CORE) -# define Py_BUILD_CORE -#endif - -#if defined(Py_TARGET_ABI3T) -# if !defined(Py_GIL_DISABLED) -// Define Py_GIL_DISABLED for users' needs. This macro is used to enable -// locking needed in for free-threaded interpreters builds. -# define Py_GIL_DISABLED -# endif -#endif - - /************************************************************************** Symbols and macros to supply platform-independent interfaces to basic C language & library operations whose spellings vary across platforms. @@ -393,17 +365,6 @@ extern "C" { # define Py_NO_INLINE #endif -#include "exports.h" - -#ifdef Py_LIMITED_API - // The internal C API must not be used with the limited C API: make sure - // that Py_BUILD_CORE macro is not defined in this case. These 3 macros are - // used by exports.h, so only undefine them afterwards. -# undef Py_BUILD_CORE -# undef Py_BUILD_CORE_BUILTIN -# undef Py_BUILD_CORE_MODULE -#endif - /* limits.h constants that may be missing */ #ifndef INT_MAX @@ -592,6 +553,7 @@ extern "C" { # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory)) +# define _Py_MSAN_UNPOISON(PTR, SIZE) (__msan_unpoison(PTR, SIZE)) # endif # endif # if __has_feature(address_sanitizer) @@ -630,6 +592,9 @@ extern "C" { #ifndef _Py_NO_SANITIZE_MEMORY # define _Py_NO_SANITIZE_MEMORY #endif +#ifndef _Py_MSAN_UNPOISON +# define _Py_MSAN_UNPOISON(PTR, SIZE) +#endif /* AIX has __bool__ redefined in it's system header file. */ #if defined(_AIX) && defined(__bool__) diff --git a/Include/pystate.h b/Include/pystate.h index 727b8fbfffe0e67..8dad748238f4f39 100644 --- a/Include/pystate.h +++ b/Include/pystate.h @@ -120,6 +120,29 @@ PyAPI_FUNC(void) PyGILState_Release(PyGILState_STATE); PyAPI_FUNC(PyThreadState *) PyGILState_GetThisThreadState(void); +/* PEP 788 -- Protection against interpreter finalization */ + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) + +typedef struct PyInterpreterGuard PyInterpreterGuard; +typedef struct PyInterpreterView PyInterpreterView; + +typedef void PyThreadStateToken; + +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterGuard_Close(PyInterpreterGuard *guard); +PyAPI_FUNC(PyInterpreterGuard *) PyInterpreterGuard_FromView(PyInterpreterView *view); + +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromCurrent(void); +PyAPI_FUNC(void) PyInterpreterView_Close(PyInterpreterView *view); +PyAPI_FUNC(PyInterpreterView *) PyInterpreterView_FromMain(void); + +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_Ensure(PyInterpreterGuard *guard); +PyAPI_FUNC(PyThreadStateToken *) PyThreadState_EnsureFromView(PyInterpreterView *view); +PyAPI_FUNC(void) PyThreadState_Release(PyThreadStateToken *tstate); + +#endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYSTATE_H # include "cpython/pystate.h" diff --git a/Include/pytypedefs.h b/Include/pytypedefs.h index e78ed56a3b67cd1..1d0b4e3e85ef74a 100644 --- a/Include/pytypedefs.h +++ b/Include/pytypedefs.h @@ -14,6 +14,7 @@ typedef struct PyModuleDef_Slot PyModuleDef_Slot; typedef struct PyMethodDef PyMethodDef; typedef struct PyGetSetDef PyGetSetDef; typedef struct PyMemberDef PyMemberDef; +typedef struct PySlot PySlot; typedef struct _object PyObject; typedef struct _longobject PyLongObject; diff --git a/Include/sliceobject.h b/Include/sliceobject.h index 00c70a6e911b417..9d6a16da95fe2f5 100644 --- a/Include/sliceobject.h +++ b/Include/sliceobject.h @@ -45,9 +45,9 @@ PyAPI_FUNC(Py_ssize_t) PySlice_AdjustIndices(Py_ssize_t length, #endif #ifndef Py_LIMITED_API -# define Py_CPYTHON_SLICEOBJECT_H +# define _Py_CPYTHON_SLICEOBJECT_H # include "cpython/sliceobject.h" -# undef Py_CPYTHON_SLICEOBJECT_H +# undef _Py_CPYTHON_SLICEOBJECT_H #endif #ifdef __cplusplus diff --git a/Include/slots.h b/Include/slots.h new file mode 100644 index 000000000000000..4bf7bda0208a8df --- /dev/null +++ b/Include/slots.h @@ -0,0 +1,56 @@ +#ifndef _Py_HAVE_SLOTS_H +#define _Py_HAVE_SLOTS_H + +typedef void (*_Py_funcptr_t)(void); + +struct PySlot { + uint16_t sl_id; + uint16_t sl_flags; + _Py_ANONYMOUS union { + uint32_t sl_reserved; // must be 0 + }; + _Py_ANONYMOUS union { + void *sl_ptr; + _Py_funcptr_t sl_func; + Py_ssize_t sl_size; + int64_t sl_int64; + uint64_t sl_uint64; + }; +}; + +#define PySlot_OPTIONAL 0x0001 +#define PySlot_STATIC 0x0002 +#define PySlot_INTPTR 0x0004 + +#define Py_slot_invalid 0xffff + +#define PySlot_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_INTPTR, .sl_ptr=(void*)(VALUE)} + +#define PySlot_FUNC(NAME, VALUE) \ + {.sl_id=(NAME), .sl_func=(_Py_funcptr_t)(VALUE)} + +#define PySlot_SIZE(NAME, VALUE) \ + {.sl_id=(NAME), .sl_size=(Py_ssize_t)(VALUE)} + +#define PySlot_INT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_int64=(int64_t)(VALUE)} + +#define PySlot_UINT64(NAME, VALUE) \ + {.sl_id=(NAME), .sl_uint64=(uint64_t)(VALUE)} + +#define PySlot_STATIC_DATA(NAME, VALUE) \ + {.sl_id=(NAME), .sl_flags=PySlot_STATIC, .sl_ptr=(VALUE)} + +#define PySlot_END {0} + + +// Macros without designated initializers (for C++11 and below): + +#define PySlot_PTR(NAME, VALUE) \ + {(NAME), PySlot_INTPTR, {0}, {(void*)(VALUE)}} + +#define PySlot_PTR_STATIC(NAME, VALUE) \ + {(NAME), PySlot_INTPTR | PySlot_STATIC, {0}, {(void*)(VALUE)}} + +#endif // _Py_HAVE_SLOTS_H diff --git a/Include/slots_generated.h b/Include/slots_generated.h new file mode 100644 index 000000000000000..42edd3ad4c69ffd --- /dev/null +++ b/Include/slots_generated.h @@ -0,0 +1,121 @@ +/* Generated by Tools/build/generate_slots.py */ + +#ifndef _PY_HAVE_SLOTS_GENERATED_H +#define _PY_HAVE_SLOTS_GENERATED_H + +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= _Py_PACK_VERSION(3, 15) +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) NEW +#else +#define _Py_SLOT_COMPAT_VALUE(OLD, NEW) OLD +#endif + +#define Py_slot_end 0 +#define Py_mp_subscript 5 +#define Py_nb_absolute 6 +#define Py_nb_add 7 +#define Py_nb_and 8 +#define Py_nb_bool 9 +#define Py_nb_divmod 10 +#define Py_nb_float 11 +#define Py_nb_floor_divide 12 +#define Py_nb_index 13 +#define Py_nb_inplace_add 14 +#define Py_nb_inplace_and 15 +#define Py_nb_inplace_floor_divide 16 +#define Py_nb_inplace_lshift 17 +#define Py_nb_inplace_multiply 18 +#define Py_nb_inplace_or 19 +#define Py_nb_inplace_power 20 +#define Py_nb_inplace_remainder 21 +#define Py_nb_inplace_rshift 22 +#define Py_nb_inplace_subtract 23 +#define Py_nb_inplace_true_divide 24 +#define Py_nb_inplace_xor 25 +#define Py_nb_int 26 +#define Py_nb_invert 27 +#define Py_nb_lshift 28 +#define Py_nb_multiply 29 +#define Py_nb_negative 30 +#define Py_nb_or 31 +#define Py_nb_positive 32 +#define Py_nb_power 33 +#define Py_nb_remainder 34 +#define Py_nb_rshift 35 +#define Py_nb_subtract 36 +#define Py_nb_true_divide 37 +#define Py_nb_xor 38 +#define Py_sq_ass_item 39 +#define Py_sq_concat 40 +#define Py_sq_contains 41 +#define Py_sq_inplace_concat 42 +#define Py_sq_inplace_repeat 43 +#define Py_sq_item 44 +#define Py_sq_length 45 +#define Py_sq_repeat 46 +#define Py_tp_alloc 47 +#define Py_tp_base 48 +#define Py_tp_bases 49 +#define Py_tp_call 50 +#define Py_tp_clear 51 +#define Py_tp_dealloc 52 +#define Py_tp_del 53 +#define Py_tp_descr_get 54 +#define Py_tp_descr_set 55 +#define Py_tp_doc 56 +#define Py_tp_getattr 57 +#define Py_tp_getattro 58 +#define Py_tp_hash 59 +#define Py_tp_init 60 +#define Py_tp_is_gc 61 +#define Py_tp_iter 62 +#define Py_tp_iternext 63 +#define Py_tp_methods 64 +#define Py_tp_new 65 +#define Py_tp_repr 66 +#define Py_tp_richcompare 67 +#define Py_tp_setattr 68 +#define Py_tp_setattro 69 +#define Py_tp_str 70 +#define Py_tp_traverse 71 +#define Py_tp_members 72 +#define Py_tp_getset 73 +#define Py_tp_free 74 +#define Py_nb_matrix_multiply 75 +#define Py_nb_inplace_matrix_multiply 76 +#define Py_am_await 77 +#define Py_am_aiter 78 +#define Py_am_anext 79 +#define Py_tp_finalize 80 +#define Py_am_send 81 +#define Py_tp_vectorcall 82 +#define Py_tp_token 83 +#define Py_mod_create _Py_SLOT_COMPAT_VALUE(1, 84) +#define Py_mod_exec _Py_SLOT_COMPAT_VALUE(2, 85) +#define Py_mod_multiple_interpreters _Py_SLOT_COMPAT_VALUE(3, 86) +#define Py_mod_gil _Py_SLOT_COMPAT_VALUE(4, 87) +#define Py_bf_getbuffer _Py_SLOT_COMPAT_VALUE(1, 88) +#define Py_bf_releasebuffer _Py_SLOT_COMPAT_VALUE(2, 89) +#define Py_mp_ass_subscript _Py_SLOT_COMPAT_VALUE(3, 90) +#define Py_mp_length _Py_SLOT_COMPAT_VALUE(4, 91) +#define Py_slot_subslots 92 +#define Py_tp_slots 93 +#define Py_mod_slots 94 +#define Py_tp_name 95 +#define Py_tp_basicsize 96 +#define Py_tp_extra_basicsize 97 +#define Py_tp_itemsize 98 +#define Py_tp_flags 99 +#define Py_mod_name 100 +#define Py_mod_doc 101 +#define Py_mod_state_size 102 +#define Py_mod_methods 103 +#define Py_mod_state_traverse 104 +#define Py_mod_state_clear 105 +#define Py_mod_state_free 106 +#define Py_tp_metaclass 107 +#define Py_tp_module 108 +#define Py_mod_abi 109 +#define Py_mod_token 110 + +#define _Py_slot_COUNT 111 +#endif /* _PY_HAVE_SLOTS_GENERATED_H */ diff --git a/Include/structseq.h b/Include/structseq.h index e52d6188030af9d..e5da785f13d46b8 100644 --- a/Include/structseq.h +++ b/Include/structseq.h @@ -29,9 +29,9 @@ PyAPI_FUNC(void) PyStructSequence_SetItem(PyObject*, Py_ssize_t, PyObject*); PyAPI_FUNC(PyObject*) PyStructSequence_GetItem(PyObject*, Py_ssize_t); #ifndef Py_LIMITED_API -# define Py_CPYTHON_STRUCTSEQ_H +# define _Py_CPYTHON_STRUCTSEQ_H # include "cpython/structseq.h" -# undef Py_CPYTHON_STRUCTSEQ_H +# undef _Py_CPYTHON_STRUCTSEQ_H #endif #ifdef __cplusplus diff --git a/Include/typeslots.h b/Include/typeslots.h deleted file mode 100644 index a7f3017ec02e92f..000000000000000 --- a/Include/typeslots.h +++ /dev/null @@ -1,96 +0,0 @@ -/* Do not renumber the file; these numbers are part of the stable ABI. */ -#define Py_bf_getbuffer 1 -#define Py_bf_releasebuffer 2 -#define Py_mp_ass_subscript 3 -#define Py_mp_length 4 -#define Py_mp_subscript 5 -#define Py_nb_absolute 6 -#define Py_nb_add 7 -#define Py_nb_and 8 -#define Py_nb_bool 9 -#define Py_nb_divmod 10 -#define Py_nb_float 11 -#define Py_nb_floor_divide 12 -#define Py_nb_index 13 -#define Py_nb_inplace_add 14 -#define Py_nb_inplace_and 15 -#define Py_nb_inplace_floor_divide 16 -#define Py_nb_inplace_lshift 17 -#define Py_nb_inplace_multiply 18 -#define Py_nb_inplace_or 19 -#define Py_nb_inplace_power 20 -#define Py_nb_inplace_remainder 21 -#define Py_nb_inplace_rshift 22 -#define Py_nb_inplace_subtract 23 -#define Py_nb_inplace_true_divide 24 -#define Py_nb_inplace_xor 25 -#define Py_nb_int 26 -#define Py_nb_invert 27 -#define Py_nb_lshift 28 -#define Py_nb_multiply 29 -#define Py_nb_negative 30 -#define Py_nb_or 31 -#define Py_nb_positive 32 -#define Py_nb_power 33 -#define Py_nb_remainder 34 -#define Py_nb_rshift 35 -#define Py_nb_subtract 36 -#define Py_nb_true_divide 37 -#define Py_nb_xor 38 -#define Py_sq_ass_item 39 -#define Py_sq_concat 40 -#define Py_sq_contains 41 -#define Py_sq_inplace_concat 42 -#define Py_sq_inplace_repeat 43 -#define Py_sq_item 44 -#define Py_sq_length 45 -#define Py_sq_repeat 46 -#define Py_tp_alloc 47 -#define Py_tp_base 48 -#define Py_tp_bases 49 -#define Py_tp_call 50 -#define Py_tp_clear 51 -#define Py_tp_dealloc 52 -#define Py_tp_del 53 -#define Py_tp_descr_get 54 -#define Py_tp_descr_set 55 -#define Py_tp_doc 56 -#define Py_tp_getattr 57 -#define Py_tp_getattro 58 -#define Py_tp_hash 59 -#define Py_tp_init 60 -#define Py_tp_is_gc 61 -#define Py_tp_iter 62 -#define Py_tp_iternext 63 -#define Py_tp_methods 64 -#define Py_tp_new 65 -#define Py_tp_repr 66 -#define Py_tp_richcompare 67 -#define Py_tp_setattr 68 -#define Py_tp_setattro 69 -#define Py_tp_str 70 -#define Py_tp_traverse 71 -#define Py_tp_members 72 -#define Py_tp_getset 73 -#define Py_tp_free 74 -#define Py_nb_matrix_multiply 75 -#define Py_nb_inplace_matrix_multiply 76 -#define Py_am_await 77 -#define Py_am_aiter 78 -#define Py_am_anext 79 -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 -/* New in 3.5 */ -#define Py_tp_finalize 80 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 -/* New in 3.10 */ -#define Py_am_send 81 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ -#define Py_tp_vectorcall 82 -#endif -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030E0000 -/* New in 3.14 */ -#define Py_tp_token 83 -#endif diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index a91a7043c1b8d41..98fa22d66a923c2 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -48,7 +48,7 @@ Note that traceback objects don't store all this information -- they store the s number, for backward compatibility, and the "last instruction" value. The rest can be computed from the last instruction (`tb_lasti`) with the help of the locations table. For Python code, there is a convenience method -(`codeobject.co_positions`)[https://docs.python.org/dev/reference/datamodel.html#codeobject.co_positions] +[`codeobject.co_positions`](https://docs.python.org/dev/reference/datamodel.html#codeobject.co_positions) which returns an iterator of `({line}, {endline}, {column}, {endcolumn})` tuples, one per instruction. There is also `co_lines()` which returns an iterator of `({start}, {end}, {line})` tuples, @@ -70,14 +70,6 @@ The `co_linetable` bytes object of code objects contains a compact representation of the source code positions of instructions, which are returned by the `co_positions()` iterator. -> [!NOTE] -> `co_linetable` is not to be confused with `co_lnotab`. -> For backwards compatibility, `co_lnotab` exposes the format -> as it existed in Python 3.10 and lower: this older format -> stores only the start line for each instruction. -> It is lazily created from `co_linetable` when accessed. -> See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. - `co_linetable` consists of a sequence of location entries. Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 742af5efcf5ce50..9ed4d0eb65a0bdd 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -359,19 +359,19 @@ in [Python/compile.c](../Python/compile.c) into a sequence of pseudo instruction These are similar to bytecode, but in some cases they are more abstract, and are resolved later into actual bytecode. The construction of this instruction sequence is handled by several functions that break the task down by various AST node types. -The functions are all named `compiler_visit_{xx}` where *xx* is the name of the node +The functions are all named `codegen_visit_{xx}` where *xx* is the name of the node type (such as `stmt`, `expr`, etc.). Each function receives a `struct compiler *` and `{xx}_ty` where *xx* is the AST node type. Typically these functions consist of a large 'switch' statement, branching based on the kind of node type passed to it. Simple things are handled inline in the 'switch' statement with more complex transformations farmed out to other -functions named `compiler_{xx}` with *xx* being a descriptive name of what is +functions named `codegen_{xx}` with *xx* being a descriptive name of what is being handled. When transforming an arbitrary AST node, use the `VISIT()` macro. -The appropriate `compiler_visit_{xx}` function is called, based on the value +The appropriate `codegen_visit_{xx}` function is called, based on the value passed in for (so `VISIT({c}, expr, {node})` calls -`compiler_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, +`codegen_visit_expr({c}, {node})`). The `VISIT_SEQ()` macro is very similar, but is called on AST node sequences (those values that were created as arguments to a node that used the '*' modifier). @@ -414,8 +414,8 @@ which is added at the end of a function is not associated with any line in the source code. There are several helper functions that will emit pseudo-instructions -and are named `compiler_{xx}()` where *xx* is what the function helps -with (`list`, `boolop`, etc.). A rather useful one is `compiler_nameop()`. +and are named `codegen_{xx}()` where *xx* is what the function helps +with (`list`, `boolop`, etc.). A rather useful one is `codegen_nameop()`. This function looks up the scope of a variable and, based on the expression context, emits the proper opcode to load, store, or delete the variable. diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 60ab2055afa7b1c..475ca75e28e1a5d 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -142,22 +142,26 @@ since the frame chain may have been in an inconsistent state due to concurrent u ### Remote Profiling Frame Cache -The `last_profiled_frame` field in `PyThreadState` supports an optimization for -remote profilers that sample call stacks from external processes. When a remote -profiler reads the call stack, it writes the current frame address to this field. -The eval loop then keeps this pointer valid by updating it to the parent frame -whenever a frame returns (in `_PyEval_FrameClearAndPop`). +The `last_profiled_frame` and `last_profiled_frame_seq` fields in +`PyThreadState` support an optimization for remote profilers that sample call +stacks from external processes. When a remote profiler reads the call stack, it +writes the current frame address to `last_profiled_frame`. The eval loop then +keeps this pointer valid by updating it to the parent frame whenever a frame +returns (in `_PyEval_FrameClearAndPop`) and increments the sequence. This creates a "high-water mark" that always points to a frame still on the stack. On subsequent samples, the profiler can walk from `current_frame` until it reaches -`last_profiled_frame`, knowing that frames from that point downward are unchanged -and can be retrieved from a cache. This significantly reduces the amount of remote -memory reads needed when call stacks are deep and stable at their base. - -The update in `_PyEval_FrameClearAndPop` is guarded: it only writes when -`last_profiled_frame` is non-NULL AND matches the frame being popped. This -prevents transient frames (called and returned between profiler samples) from -corrupting the cache pointer, while avoiding any overhead when profiling is inactive. +`last_profiled_frame`, then validate the pointer and sequence before using cached +callers. This prevents a later frame that reuses the same `_PyInterpreterFrame` +address from being mistaken for the sampled frame. The cache significantly +reduces the amount of remote memory reads needed when call stacks are deep and +stable at their base. + +The update in `_PyEval_FrameClearAndPop` is guarded: it only advances the +pointer and sequence when `last_profiled_frame` is non-NULL AND matches the +frame being popped. This prevents transient frames (called and returned between +profiler samples) from corrupting the cache anchor, while avoiding any overhead +when profiling is inactive. ### The Instruction Pointer diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 94e6fb05b68d6fc..0ef45ff8e02bc5f 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -107,7 +107,7 @@ As is explained later in the [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +that in the [Optimization: generations](#Optimization-generations) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -203,22 +203,22 @@ unreachable: ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -360,11 +360,12 @@ follows these steps in order: the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: incremental collection -==================================== +Optimization: generations +========================= -In order to bound the length of each garbage collection pause, the GC implementation -for the default build uses incremental collection with two generations. +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. Generational garbage collection takes advantage of what is known as the weak generational hypothesis: Most objects die young. @@ -372,76 +373,29 @@ This has proven to be very close to the reality of many Python programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -two generations: young and old. Every new object starts in the young generation. -Each garbage collection scans the entire young generation and part of the old generation. - -The time taken to scan the young generation can be controlled by controlling its -size, but the size of the old generation cannot be controlled. -In order to keep pause times down, scanning of the old generation of the heap -occurs in increments. - -To keep track of what has been scanned, the old generation contains two lists: - -* Those objects that have not yet been scanned, referred to as the `pending` list. -* Those objects that have been scanned, referred to as the `visited` list. - -To detect and collect all unreachable objects in the heap, the garbage collector -must scan the whole heap. This whole heap scan is called a full scavenge. - -Increments ----------- - -Each full scavenge is performed in a series of increments. -For each full scavenge, the combined increments will cover the whole heap. - -Each increment is made up of: - -* The young generation -* The old generation's least recently scanned objects -* All objects reachable from those objects that have not yet been scanned this full scavenge - -The surviving objects (those that are not collected) are moved to the back of the -`visited` list in the old generation. - -When a full scavenge starts, no objects in the heap are considered to have been scanned, -so all objects in the old generation must be in the `pending` space. -When all objects in the heap have been scanned a cycle ends, and all objects are moved -to the `pending` list again. To avoid having to traverse the entire list, which list is -`pending` and which is `visited` is determined by a field in the `GCState` struct. -The `visited` and `pending` lists can be swapped by toggling this bit. - -Correctness ------------ - -The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all -unreachable cycles in a list of objects, but will not find any cycles that are -even partly outside of that list. -Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, -each cycle must be fully contained within a single increment. - -To make sure that no partial cycles are included in the increment we perform a -[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) -over reachable, unscanned objects from the initial increment. -Since the transitive closure of objects reachable from an object must be a (non-strict) -superset of any unreachable cycle including that object, we are guaranteed that a -transitive closure cannot contain any partial cycles. -We can exclude scanned objects, as they must have been reachable when scanned. -If a scanned object becomes part of an unreachable cycle after being scanned, it will -not be collected at this time, but it will be collected in the next full scavenge. +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. > [!NOTE] -> The GC implementation for the free-threaded build does not use incremental collection. -> Every collection operates on the entire heap. +> The GC implementation for the free-threaded build does not use generational +> collection. Every collection operates on the entire heap. + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds `threshold0`, -collection starts. `threshold1` determines the fraction of the old -collection that is included in the increment. -The fraction is inversely proportional to `threshold1`, -as historically a larger `threshold1` meant that old generation -collections were performed less frequently. -`threshold2` is ignored. +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) @@ -450,7 +404,7 @@ function: ```pycon >>> import gc >>> gc.get_threshold() -(700, 10, 10) +(2000, 10, 10) ``` The content of these generations can be examined using the @@ -463,84 +417,61 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ... pass ... >>> # Move everything to the old generation so it's easier to inspect ->>> # the young generation. +>>> # the younger generation. >>> gc.collect() 0 >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> ->>> # Initially the object is in the young generation. +>>> +>>> # Initially the object is in the youngest generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object ->>> # moves to the old generation. +>>> # moves to the next generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) -[] ->>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + Optimization: excluding reachable objects ========================================= An object cannot be garbage if it can be reached. To avoid having to identify -reference cycles across the whole heap, we can reduce the amount of work done -considerably by first identifying objects reachable from objects known to be -alive. These objects are excluded from the normal cyclic detection process. - -The default and free-threaded build both implement this optimization but in -slightly different ways. - -Finding reachable objects for the default build GC --------------------------------------------------- - -This works by first moving most reachable objects to the `visited` space. -Empirically, most reachable objects can be reached from a small set of global -objects and local variables. This step does much less work per object, so -reduces the time spent performing garbage collection by at least half. - -> [!NOTE] -> Objects that are not determined to be reachable by this pass are not necessarily -> unreachable. We still need to perform the main algorithm to determine which objects -> are actually unreachable. -We use the same technique of forming a transitive closure as the incremental -collector does to find reachable objects, seeding the list with some global -objects and the currently executing frames. - -This phase moves objects to the `visited` space, as follows: - -1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` -module and all objects directly referred to from stack frames are added to a working -set of reachable objects. -2. Until this working set is empty: - 1. Pop an object from the set and move it to the `visited` space - 2. For each object directly reachable from that object: - * If it is not already in `visited` space and it is a GC object, - add it to the working set - - -Before each increment of collection is performed, the stacks are scanned -to check for any new stack frames that have been created since the last -increment. All objects directly referred to from those stack frames are -added to the working set. -Then the above algorithm is repeated, starting from step 2. - +reference cycles across the whole heap, the free-threaded build first identifies +objects reachable from objects known to be alive. These objects are excluded +from the normal cyclic detection process. Finding reachable objects for the free-threaded GC -------------------------------------------------- Within the `gc_free_threading.c` implementation, this is known as the "mark -alive" pass or phase. It is similar in concept to what is done for the default -build GC. Rather than moving objects between double-linked lists, the -free-threaded GC uses a flag in `ob_gc_bits` to track if an object is -found to be definitely alive (not garbage). +alive" pass or phase. The free-threaded GC uses a flag in `ob_gc_bits` to track +if an object is found to be definitely alive (not garbage). To find objects reachable from known alive objects, known as the "roots", the `gc_mark_alive_from_roots()` function is used. Root objects include @@ -771,6 +702,14 @@ tuples can be untracked. A tuple can be untracked if all of its contents are already not tracked. Tuples are examined for untracking in all garbage collection cycles. +Dictionaries are always tracked from creation and are not untracked by the +garbage collector. Earlier versions (up to 3.13) used lazy tracking: empty or +atomic-only dicts were untracked on creation and re-tracked when a trackable +value was inserted (via `MAINTAIN_TRACKING`), and full collections called +`_PyDict_MaybeUntrack` to prune dicts whose values had become atomic. That +machinery was removed in 3.14 (GH-127010) because the per-set-item cost of +checking the tracking invariant outweighed the savings on full collections. + The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the tracking status of the object. diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 7fc41a807dd5667..fa89996aa654291 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -307,9 +307,7 @@ With a new bytecode you must also change what is called the "magic number" for .pyc files: bump the value of the variable `MAGIC_NUMBER` in [`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py). Changing this number will lead to all .pyc files with the old `MAGIC_NUMBER` -to be recompiled by the interpreter on import. Whenever `MAGIC_NUMBER` is -changed, the ranges in the `magic_values` array in -[`PC/launcher.c`](../PC/launcher.c) may also need to be updated. Changes to +to be recompiled by the interpreter on import. Changes to [`Lib/importlib/_bootstrap_external.py`](../Lib/importlib/_bootstrap_external.py) will take effect only after running `make regen-importlib`. diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index 1345f2db8b34db9..8db5ba3c85ddda6 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -140,7 +140,7 @@ template file [`Tools/jit/template.c`](../Tools/jit/template.c). Each of the `.c` files is compiled by LLVM, to produce an object file that contains a function that executes the opcode. These compiled functions are used to generate the file -[`jit_stencils.h`](../jit_stencils.h), which contains the functions +`jit_stencils.h`, which contains the functions that the JIT can use to emit code for each of the bytecodes. For Python maintainers this means that changes to the bytecodes and diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 1d0ffe6d40d078a..a6de8d456b6f71c 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -545,7 +545,7 @@ by default** except for rules with the special marker `memo` after the rule name (and type, if present): ``` -rule_name[typr] (memo): +rule_name[return_type] (memo): ... ``` @@ -772,7 +772,7 @@ from them or to perform extra processing on the generated tree. > Actions must **never** be used to accept or reject rules. It may be tempting > in some situations to write a very generic rule and then check the generated > AST to decide whether it is valid or not, but this will render the -> (official grammar)[https://docs.python.org/3/reference/grammar.html] partially +> [official grammar](https://docs.python.org/3/reference/grammar.html) partially > incorrect (because it does not include actions) and will make it more difficult > for other Python implementations to adapt the grammar to their own needs. @@ -819,6 +819,13 @@ directory on the CPython repository and manually call the parser generator by ex $ python -m pegen python ``` +> [!CAUTION] +> Python's grammar (the `Grammar/python.gram` file) is written for the +> C backend. To experiment, you will need to write a grammar +> without C-specific parts like actions and the trailer. +> See [#133560](https://github.com/python/cpython/issues/133560) +> and [#96424](https://github.com/python/cpython/issues/96424) for more information. + This will generate a file called `parse.py` in the same directory that you can use to parse some input: @@ -853,7 +860,7 @@ $ python -d file_to_test.py This will print **a lot** of output to `stderr` so it is probably better to dump it to a file for further analysis. The output consists of trace lines with the -following structure:: +following structure: ``` ('>'|'-'|'+'|'!') []: ... diff --git a/InternalDocs/profiling_binary_format.md b/InternalDocs/profiling_binary_format.md index 7e4592a0d897055..51b1944aa5c8a00 100644 --- a/InternalDocs/profiling_binary_format.md +++ b/InternalDocs/profiling_binary_format.md @@ -34,7 +34,7 @@ by 10-50x compared to text formats while also enabling faster I/O. ## File Layout -The file consists of five sections: +The file consists of five required sections and one optional extension: ``` +------------------+ Offset 0 @@ -47,6 +47,8 @@ The file consists of five sections: | String Table | Variable size +------------------+ frame_table_offset | Frame Table | Variable size ++------------------+ file_size - 64 (when stats are present) +| Profile Stats | 32 bytes (optional) +------------------+ file_size - 32 | Footer | 32 bytes (fixed) +------------------+ file_size @@ -354,6 +356,38 @@ location. Zigzag encoding ensures these small negative values encode efficiently (−1 becomes 1, which is one byte) rather than requiring the maximum varint length. +## Profile Statistics + +New files can store measured duration, sampling rate, error rate, and missed +sample percentage in an optional 56-byte extension immediately before the +footer. Older readers ignore these +bytes after parsing the declared number of frame-table entries, and newer +readers treat a missing extension as unavailable statistics. + +``` + Offset Size Type Description ++--------+------+---------+----------------------------------------+ +| 0 | 8 | double | Measured duration (seconds) | +| 8 | 8 | double | Measured sample rate (samples/second) | +| 16 | 8 | double | Failed sample percentage | +| 24 | 8 | double | Missed sample percentage | +| 32 | 4 | uint32 | Optional field presence flags | +| 36 | 4 | uint32 | Reserved | +| 40 | 8 | bytes | Signature ("TACHSTAT") | +| 48 | 4 | uint32 | Extension version (1) | +| 52 | 4 | uint32 | Extension size (56) | ++--------+------+---------+----------------------------------------+ +``` + +Putting the signature, version, and size at the end lets readers discover +the extension from its fixed position relative to the footer while allowing +future versions to add fields before that trailer. Multi-byte values use the +same native byte order as the rest of the file and are byte-swapped by +cross-endian readers. + +Readers also accept the original 32-byte extension, which only contains the +duration and sampling rate. + ## Footer ``` @@ -448,8 +482,9 @@ compress less; higher levels (6+) compress more but slow down writing. Level 4. Flush remaining buffered data and finalize compression 5. Write the string table (length-prefixed strings in index order) 6. Write the frame table (varint-encoded entries in index order) -7. Write the footer with final counts -8. Seek to offset 0 and write the header with actual values +7. Write measured profile statistics, when available +8. Write the footer with final counts +9. Seek to offset 0 and write the header with actual values The writer maintains two dictionaries: one mapping strings to indices, one mapping (filename_idx, funcname_idx, lineno) tuples to frame indices. These @@ -461,12 +496,13 @@ enable O(1) lookup during interning. if the magic appears byte-swapped) 2. Validate version and read remaining header fields (byte-swapping if needed) 3. Seek to end − 32 and read the footer (byte-swapping counts if needed) -4. Allocate string array of `string_count` elements -5. Parse the string table, populating the array -6. Allocate frame array of `frame_count * 3` uint32 elements -7. Parse the frame table, populating the array -8. If compressed, decompress the sample data region -9. Iterate through samples, resolving indices to strings/frames +4. Read measured profile statistics when the optional extension is present +5. Allocate string array of `string_count` elements +6. Parse the string table, populating the array +7. Allocate the frame array +8. Parse the frame table, populating the array +9. If compressed, decompress the sample data region +10. Iterate through samples, resolving indices to strings/frames (byte-swapping thread_id and interpreter_id if needed) The reader builds lookup arrays rather than dictionaries since it only needs @@ -530,11 +566,10 @@ one write() call (or feeds through the compression stream). ## Future Considerations -The format reserves space for future extensions. The 12 reserved bytes in -the header could hold additional metadata. The 16-byte checksum field in -the footer is currently unused. The version field allows incompatible -changes with graceful rejection. New compression types could be added -(compression_type > 1). +The optional profile-statistics block provides an extensible metadata area. +The 16-byte checksum field in the footer is currently unused. The version +field allows incompatible changes with graceful rejection. New compression +types could be added (compression_type > 1). Any changes that alter the meaning of existing fields or the parsing logic should increment the version number to prevent older readers from diff --git a/InternalDocs/qsbr.md b/InternalDocs/qsbr.md index 1c4a79a7b44436a..d511396fdab6452 100644 --- a/InternalDocs/qsbr.md +++ b/InternalDocs/qsbr.md @@ -101,15 +101,39 @@ Periodically, a polling mechanism processes this deferred-free list: To reduce memory contention from frequent updates to the global `wr_seq`, its advancement is sometimes deferred. Instead of incrementing `wr_seq` on every -reclamation request, each thread tracks its number of deferrals locally. Once -the deferral count reaches a limit (QSBR_DEFERRED_LIMIT, currently 10), the -thread advances the global `wr_seq` and resets its local count. - -When an object is added to the deferred-free list, its qsbr_goal is set to -`wr_seq` + 2. By setting the goal to the next sequence value, we ensure it's safe -to defer the global counter advancement. This optimization improves runtime -speed but may increase peak memory usage by slightly delaying when memory can -be reclaimed. +reclamation request, the object's qsbr_goal is set to `wr_seq` + 2 (the value +the counter *would* take on its next advance) without actually advancing the +global counter. This is safe because the goal still corresponds to a future +sequence value that no thread has yet observed as quiescent. + +Whether to actually advance `wr_seq` is decided per request, based on how +much memory and how many items the calling thread has already deferred since +its last advance: + +* For deferred object frees (`_PyMem_FreeDelayed`), the thread tracks both a + count (`deferred_count`) and an estimate of the held memory + (`deferred_memory`). The global `wr_seq` is advanced when the freed block + is larger than `QSBR_FREE_MEM_LIMIT` (1 MiB), when the accumulated deferred + memory exceeds that limit, or when the count exceeds `QSBR_DEFERRED_LIMIT` + (127, sized so a chunk of work items is processed before it overflows). + Crossing any of these thresholds also sets a per-thread `should_process` + flag, signalling that the deferred-free list should be drained. + +* For mimalloc pages held by QSBR, the thread tracks `deferred_page_memory` + and advances `wr_seq` when either the individual page or the accumulated + page memory exceeds `QSBR_PAGE_MEM_LIMIT` (4096 * 20 bytes). Advancing + promptly here matters because a held page cannot be reused for a different + size class or by a different thread. + +Processing of the deferred-free list normally happens from the eval breaker +(rather than from inside `_PyMem_FreeDelayed`), which gives the global +`rd_seq` a better chance to have advanced far enough that items can actually +be freed. `_PyMem_ProcessDelayed` is still called from the free path as a +safety valve when a work-item chunk fills up. + +This optimization improves runtime speed but may increase peak memory usage +by slightly delaying when memory can be reclaimed; the size-based thresholds +above bound that extra memory. ## Limitations diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 0913b1a3471ef49..829a27654a37d34 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -16,8 +16,8 @@ dynamic interning. The 256 possible one-character latin-1 strings, which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, -`_PyRuntime.static_objects.strings.ascii` and -`_PyRuntime.static_objects.strings.latin1`. +`_PyRuntime.static_objects.singletons.strings.ascii` and +`_PyRuntime.static_objects.singletons.strings.latin1`. Longer singleton strings are marked in C source with `_Py_ID` (if the string is a valid C identifier fragment) or `_Py_STR` (if it needs a separate diff --git a/Lib/.ruff.toml b/Lib/.ruff.toml new file mode 100644 index 000000000000000..c9ae9b95641b318 --- /dev/null +++ b/Lib/.ruff.toml @@ -0,0 +1,21 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +# Unlike Tools/, stdlib can use newer syntax than PYTHON_FOR_REGEN +target-version = "py315" + +[lint] +select = [ + "F401", # Unused import +] + +[lint.per-file-ignores] +"ctypes/__init__.py" = ["F401"] # Re-exports from _ctypes +"ensurepip/__init__.py" = ["F401"] # `import zlib` availability check +"idlelib/idle_test/htest.py" = ["F401"] # Import for Windows DPI side effect +"idlelib/idle_test/test_iomenu.py" = ["F401"] # Imports checked for existence +"importlib/_abc.py" = ["F401"] # Bootstrap-sensitive _bootstrap import +"importlib/machinery.py" = ["F401"] # NamespacePath re-export +"importlib/metadata/__init__.py" = ["F401"] # Synced from importlib_metadata +"profiling/sampling/sample.py" = ["F401"] # Re-exports PROFILING_MODE_* constants +"ssl.py" = ["F401"] # Re-exports from _ssl +"warnings.py" = ["F401"] # Re-exports from _py_warnings diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 23cc6d8faae2dac..24d08605429d340 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -67,7 +67,6 @@ def _f(): pass # are not included on this list. bytes_iterator = type(iter(b'')) bytearray_iterator = type(iter(bytearray())) -#callable_iterator = ??? dict_keyiterator = type(iter({}.keys())) dict_valueiterator = type(iter({}.values())) dict_itemiterator = type(iter({}.items())) @@ -319,7 +318,6 @@ def __subclasshook__(cls, C): Iterator.register(bytes_iterator) Iterator.register(bytearray_iterator) -#Iterator.register(callable_iterator) Iterator.register(dict_keyiterator) Iterator.register(dict_valueiterator) Iterator.register(dict_itemiterator) @@ -461,8 +459,8 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened ``argtypes`` - followed by ``resulttype``. + This sets ``__args__`` to a tuple containing the flattened + ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. @@ -928,8 +926,9 @@ def __delitem__(self, key): __marker = object() def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise + KeyError is raised. ''' try: value = self[key] @@ -963,9 +962,12 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v + If E present and has a .keys() method, does: + for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: + for (k, v) in E: D[k] = v + In either case, this is followed by: + for k, v in F.items(): D[k] = v ''' if isinstance(other, Mapping): for key in other: @@ -1030,8 +1032,8 @@ def __reversed__(self): yield self[i] def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. + '''S.index(value, [start, [stop]]) -> integer -- return first index of + value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. @@ -1139,15 +1141,16 @@ def reverse(self): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + """S.extend(iterable) -- extend sequence by appending elements from the + iterable""" if values is self: values = list(values) for v in values: self.append(v) def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. + '''S.pop([index]) -> item -- remove and return item at index (default + last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 478f81894911e73..5f44a3aa05eb8f6 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,16 +1,15 @@ +import builtins import os import sys from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field, Field +lazy from typing import IO, Literal, Self, ClassVar COLORIZE = True - -# types -if False: - from typing import IO, Literal, Self, ClassVar - _theme: Theme +_theme: Theme +type BackgroundStyle = Literal["dark", "light"] class ANSIColors: @@ -188,6 +187,25 @@ class Argparse(ThemeSection): message: str = ANSIColors.MAGENTA +@dataclass(frozen=True, kw_only=True) +class Ast(ThemeSection): + node: str = ANSIColors.CYAN + field: str = ANSIColors.BLUE + attribute: str = ANSIColors.GREY + string: str = ANSIColors.GREEN + number: str = ANSIColors.YELLOW + keyword: str = ANSIColors.BOLD_BLUE + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Calendar(ThemeSection): + header: str = ANSIColors.BOLD + highlight: str = ANSIColors.BLACK + ANSIColors.BACKGROUND_YELLOW + weekday: str = ANSIColors.CYAN + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Difflib(ThemeSection): """A 'git diff'-like theme for `difflib.unified_diff`.""" @@ -202,25 +220,25 @@ class Difflib(ThemeSection): @dataclass(frozen=True, kw_only=True) class FancyCompleter(ThemeSection): # functions and methods - function: str = ANSIColors.BOLD_BLUE - builtin_function_or_method: str = ANSIColors.BOLD_BLUE - method: str = ANSIColors.BOLD_CYAN - method_wrapper: str = ANSIColors.BOLD_CYAN - wrapper_descriptor: str = ANSIColors.BOLD_CYAN - method_descriptor: str = ANSIColors.BOLD_CYAN + function: builtins.str = ANSIColors.BOLD_BLUE + builtin_function_or_method: builtins.str = ANSIColors.BOLD_BLUE + method: builtins.str = ANSIColors.BOLD_CYAN + method_wrapper: builtins.str = ANSIColors.BOLD_CYAN + wrapper_descriptor: builtins.str = ANSIColors.BOLD_CYAN + method_descriptor: builtins.str = ANSIColors.BOLD_CYAN # numbers - int: str = ANSIColors.BOLD_YELLOW - float: str = ANSIColors.BOLD_YELLOW - complex: str = ANSIColors.BOLD_YELLOW - bool: str = ANSIColors.BOLD_YELLOW + int: builtins.str = ANSIColors.BOLD_YELLOW + float: builtins.str = ANSIColors.BOLD_YELLOW + complex: builtins.str = ANSIColors.BOLD_YELLOW + bool: builtins.str = ANSIColors.BOLD_YELLOW # others - type: str = ANSIColors.BOLD_MAGENTA - module: str = ANSIColors.CYAN - NoneType: str = ANSIColors.GREY - bytes: str = ANSIColors.BOLD_GREEN - str: str = ANSIColors.BOLD_GREEN + type: builtins.str = ANSIColors.BOLD_MAGENTA + module: builtins.str = ANSIColors.CYAN + NoneType: builtins.str = ANSIColors.GREY + bytes: builtins.str = ANSIColors.BOLD_GREEN + str: builtins.str = ANSIColors.BOLD_GREEN @dataclass(frozen=True, kw_only=True) @@ -299,7 +317,7 @@ class LiveProfiler(ThemeSection): medal_bronze_fg: int = CursesColors.GREEN # Background style: 'dark' or 'light' - background_style: Literal["dark", "light"] = "dark" + background_style: BackgroundStyle = "dark" LiveProfilerLight = LiveProfiler( @@ -347,12 +365,47 @@ class LiveProfiler(ThemeSection): ) +@dataclass(frozen=True, kw_only=True) +class ProfilerDump(ThemeSection): + header: str = ANSIColors.BOLD_BLUE + interpreter: str = ANSIColors.GREY + thread: str = ANSIColors.BOLD_CYAN + status: str = ANSIColors.YELLOW + frame_index: str = ANSIColors.GREY + frame: str = ANSIColors.BOLD_GREEN + filename: str = ANSIColors.CYAN + line_no: str = ANSIColors.YELLOW + source: str = ANSIColors.WHITE + source_highlight: str = ANSIColors.BOLD_YELLOW + opcode: str = ANSIColors.GREY + warning: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + +@dataclass(frozen=True, kw_only=True) +class Pickletools(ThemeSection): + annotation: str = ANSIColors.GREY + arg_number: str = ANSIColors.YELLOW + arg_string: str = ANSIColors.GREEN + mark: str = ANSIColors.GREY + op_call: str = ANSIColors.GREEN + op_container: str = ANSIColors.INTENSE_BLUE + op_memo: str = ANSIColors.MAGENTA + op_meta: str = ANSIColors.GREY + op_stack: str = ANSIColors.BOLD_RED + opcode_code: str = ANSIColors.CYAN + position: str = ANSIColors.GREY + proto: str = ANSIColors.YELLOW + reset: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN + command: str = ANSIColors.BOLD_CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN number: str = ANSIColors.YELLOW @@ -374,6 +427,14 @@ class Timeit(ThemeSection): reset: str = ANSIColors.RESET +@dataclass(frozen=True, kw_only=True) +class Tokenize(ThemeSection): + whitespace: str = ANSIColors.GREY + error: str = ANSIColors.BOLD_RED + position: str = ANSIColors.GREY + delimiter: str = ANSIColors.RESET + + @dataclass(frozen=True, kw_only=True) class Traceback(ThemeSection): type: str = ANSIColors.BOLD_MAGENTA @@ -404,12 +465,17 @@ class Theme: below. """ argparse: Argparse = field(default_factory=Argparse) + ast: Ast = field(default_factory=Ast) + calendar: Calendar = field(default_factory=Calendar) difflib: Difflib = field(default_factory=Difflib) fancycompleter: FancyCompleter = field(default_factory=FancyCompleter) http_server: HttpServer = field(default_factory=HttpServer) live_profiler: LiveProfiler = field(default_factory=LiveProfiler) + pickletools: Pickletools = field(default_factory=Pickletools) + profiler_dump: ProfilerDump = field(default_factory=ProfilerDump) syntax: Syntax = field(default_factory=Syntax) timeit: Timeit = field(default_factory=Timeit) + tokenize: Tokenize = field(default_factory=Tokenize) traceback: Traceback = field(default_factory=Traceback) unittest: Unittest = field(default_factory=Unittest) @@ -417,12 +483,17 @@ def copy_with( self, *, argparse: Argparse | None = None, + ast: Ast | None = None, + calendar: Calendar | None = None, difflib: Difflib | None = None, fancycompleter: FancyCompleter | None = None, http_server: HttpServer | None = None, live_profiler: LiveProfiler | None = None, + pickletools: Pickletools | None = None, + profiler_dump: ProfilerDump | None = None, syntax: Syntax | None = None, timeit: Timeit | None = None, + tokenize: Tokenize | None = None, traceback: Traceback | None = None, unittest: Unittest | None = None, ) -> Self: @@ -433,12 +504,17 @@ def copy_with( """ return type(self)( argparse=argparse or self.argparse, + ast=ast or self.ast, + calendar=calendar or self.calendar, difflib=difflib or self.difflib, fancycompleter=fancycompleter or self.fancycompleter, http_server=http_server or self.http_server, live_profiler=live_profiler or self.live_profiler, + pickletools=pickletools or self.pickletools, + profiler_dump=profiler_dump or self.profiler_dump, syntax=syntax or self.syntax, timeit=timeit or self.timeit, + tokenize=tokenize or self.tokenize, traceback=traceback or self.traceback, unittest=unittest or self.unittest, ) @@ -453,12 +529,17 @@ def no_colors(cls) -> Self: """ return cls( argparse=Argparse.no_colors(), + ast=Ast.no_colors(), + calendar=Calendar.no_colors(), difflib=Difflib.no_colors(), fancycompleter=FancyCompleter.no_colors(), http_server=HttpServer.no_colors(), live_profiler=LiveProfiler.no_colors(), + pickletools=Pickletools.no_colors(), + profiler_dump=ProfilerDump.no_colors(), syntax=Syntax.no_colors(), timeit=Timeit.no_colors(), + tokenize=Tokenize.no_colors(), traceback=Traceback.no_colors(), unittest=Unittest.no_colors(), ) diff --git a/Lib/_ios_support.py b/Lib/_ios_support.py index 20467a7c2bcaeb0..cc51aa504916e36 100644 --- a/Lib/_ios_support.py +++ b/Lib/_ios_support.py @@ -10,12 +10,15 @@ else: # ctypes is available. Load the ObjC library, and wrap the objc_getClass, # sel_registerName methods - lib = util.find_library("objc") - if lib is None: - # Failed to load the objc library - raise ImportError("ObjC runtime library couldn't be loaded") + # find_library() resolves a system library through the dyld shared cache, + # which needs _dyld_shared_cache_contains_path(); that is unavailable before + # iOS 14, so fall back to the bare name, which dyld resolves by itself. + lib = util.find_library("objc") or "libobjc.dylib" - objc = cdll.LoadLibrary(lib) + try: + objc = cdll.LoadLibrary(lib) + except OSError: + raise ImportError("ObjC runtime library couldn't be loaded") objc.objc_getClass.restype = c_void_p objc.objc_getClass.argtypes = [c_char_p] objc.sel_registerName.restype = c_void_p diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index f5954e4372a980a..df92eae151d2488 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -39,6 +39,8 @@ ), SEND=( "SEND_GEN", + "SEND_VIRTUAL", + "SEND_ASYNC_GEN", ), UNPACK_SEQUENCE=( "UNPACK_SEQUENCE_TWO_TUPLE", @@ -86,11 +88,16 @@ "JUMP_BACKWARD_NO_JIT", "JUMP_BACKWARD_JIT", ), + GET_ITER=( + "GET_ITER_SELF", + "GET_ITER_VIRTUAL", + ), FOR_ITER=( "FOR_ITER_LIST", "FOR_ITER_TUPLE", "FOR_ITER_RANGE", "FOR_ITER_GEN", + "FOR_ITER_VIRTUAL", ), CALL=( "CALL_BOUND_METHOD_EXACT_ARGS", @@ -176,42 +183,47 @@ FOR_ITER_LIST=175, FOR_ITER_RANGE=176, FOR_ITER_TUPLE=177, - JUMP_BACKWARD_JIT=178, - JUMP_BACKWARD_NO_JIT=179, - LOAD_ATTR_CLASS=180, - LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=181, - LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=182, - LOAD_ATTR_INSTANCE_VALUE=183, - LOAD_ATTR_METHOD_LAZY_DICT=184, - LOAD_ATTR_METHOD_NO_DICT=185, - LOAD_ATTR_METHOD_WITH_VALUES=186, - LOAD_ATTR_MODULE=187, - LOAD_ATTR_NONDESCRIPTOR_NO_DICT=188, - LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=189, - LOAD_ATTR_PROPERTY=190, - LOAD_ATTR_SLOT=191, - LOAD_ATTR_WITH_HINT=192, - LOAD_GLOBAL_BUILTIN=193, - LOAD_GLOBAL_MODULE=194, - LOAD_SUPER_ATTR_ATTR=195, - LOAD_SUPER_ATTR_METHOD=196, - RESUME_CHECK=197, - RESUME_CHECK_JIT=198, - SEND_GEN=199, - STORE_ATTR_INSTANCE_VALUE=200, - STORE_ATTR_SLOT=201, - STORE_ATTR_WITH_HINT=202, - STORE_SUBSCR_DICT=203, - STORE_SUBSCR_LIST_INT=204, - TO_BOOL_ALWAYS_TRUE=205, - TO_BOOL_BOOL=206, - TO_BOOL_INT=207, - TO_BOOL_LIST=208, - TO_BOOL_NONE=209, - TO_BOOL_STR=210, - UNPACK_SEQUENCE_LIST=211, - UNPACK_SEQUENCE_TUPLE=212, - UNPACK_SEQUENCE_TWO_TUPLE=213, + FOR_ITER_VIRTUAL=178, + GET_ITER_SELF=179, + GET_ITER_VIRTUAL=180, + JUMP_BACKWARD_JIT=181, + JUMP_BACKWARD_NO_JIT=182, + LOAD_ATTR_CLASS=183, + LOAD_ATTR_CLASS_WITH_METACLASS_CHECK=184, + LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN=185, + LOAD_ATTR_INSTANCE_VALUE=186, + LOAD_ATTR_METHOD_LAZY_DICT=187, + LOAD_ATTR_METHOD_NO_DICT=188, + LOAD_ATTR_METHOD_WITH_VALUES=189, + LOAD_ATTR_MODULE=190, + LOAD_ATTR_NONDESCRIPTOR_NO_DICT=191, + LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES=192, + LOAD_ATTR_PROPERTY=193, + LOAD_ATTR_SLOT=194, + LOAD_ATTR_WITH_HINT=195, + LOAD_GLOBAL_BUILTIN=196, + LOAD_GLOBAL_MODULE=197, + LOAD_SUPER_ATTR_ATTR=198, + LOAD_SUPER_ATTR_METHOD=199, + RESUME_CHECK=200, + RESUME_CHECK_JIT=201, + SEND_ASYNC_GEN=202, + SEND_GEN=203, + SEND_VIRTUAL=204, + STORE_ATTR_INSTANCE_VALUE=205, + STORE_ATTR_SLOT=206, + STORE_ATTR_WITH_HINT=207, + STORE_SUBSCR_DICT=208, + STORE_SUBSCR_LIST_INT=209, + TO_BOOL_ALWAYS_TRUE=210, + TO_BOOL_BOOL=211, + TO_BOOL_INT=212, + TO_BOOL_LIST=213, + TO_BOOL_NONE=214, + TO_BOOL_STR=215, + UNPACK_SEQUENCE_LIST=216, + UNPACK_SEQUENCE_TUPLE=217, + UNPACK_SEQUENCE_TWO_TUPLE=218, ) opmap = frozendict( @@ -277,67 +289,64 @@ CONVERT_VALUE=56, COPY=57, COPY_FREE_VARS=58, - DELETE_ATTR=59, - DELETE_DEREF=60, - DELETE_FAST=61, - DELETE_GLOBAL=62, - DELETE_NAME=63, - DICT_MERGE=64, - DICT_UPDATE=65, - END_ASYNC_FOR=66, - EXTENDED_ARG=67, - FOR_ITER=68, - GET_AWAITABLE=69, - GET_ITER=70, - IMPORT_FROM=71, - IMPORT_NAME=72, - IS_OP=73, - JUMP_BACKWARD=74, - JUMP_BACKWARD_NO_INTERRUPT=75, - JUMP_FORWARD=76, - LIST_APPEND=77, - LIST_EXTEND=78, - LOAD_ATTR=79, - LOAD_COMMON_CONSTANT=80, - LOAD_CONST=81, - LOAD_DEREF=82, - LOAD_FAST=83, - LOAD_FAST_AND_CLEAR=84, - LOAD_FAST_BORROW=85, - LOAD_FAST_BORROW_LOAD_FAST_BORROW=86, - LOAD_FAST_CHECK=87, - LOAD_FAST_LOAD_FAST=88, - LOAD_FROM_DICT_OR_DEREF=89, - LOAD_FROM_DICT_OR_GLOBALS=90, - LOAD_GLOBAL=91, - LOAD_NAME=92, - LOAD_SMALL_INT=93, - LOAD_SPECIAL=94, - LOAD_SUPER_ATTR=95, - MAKE_CELL=96, - MAP_ADD=97, - MATCH_CLASS=98, - POP_JUMP_IF_FALSE=99, - POP_JUMP_IF_NONE=100, - POP_JUMP_IF_NOT_NONE=101, - POP_JUMP_IF_TRUE=102, - RAISE_VARARGS=103, - RERAISE=104, - SEND=105, - SET_ADD=106, - SET_FUNCTION_ATTRIBUTE=107, - SET_UPDATE=108, - STORE_ATTR=109, - STORE_DEREF=110, - STORE_FAST=111, - STORE_FAST_LOAD_FAST=112, - STORE_FAST_STORE_FAST=113, - STORE_GLOBAL=114, - STORE_NAME=115, - SWAP=116, - UNPACK_EX=117, - UNPACK_SEQUENCE=118, - YIELD_VALUE=119, + DELETE_DEREF=59, + DELETE_FAST=60, + DICT_MERGE=61, + DICT_UPDATE=62, + END_ASYNC_FOR=63, + EXTENDED_ARG=64, + FOR_ITER=65, + GET_AWAITABLE=66, + GET_ITER=67, + IMPORT_FROM=68, + IMPORT_NAME=69, + IS_OP=70, + JUMP_BACKWARD=71, + JUMP_BACKWARD_NO_INTERRUPT=72, + JUMP_FORWARD=73, + LIST_APPEND=74, + LIST_EXTEND=75, + LOAD_ATTR=76, + LOAD_COMMON_CONSTANT=77, + LOAD_CONST=78, + LOAD_DEREF=79, + LOAD_FAST=80, + LOAD_FAST_AND_CLEAR=81, + LOAD_FAST_BORROW=82, + LOAD_FAST_BORROW_LOAD_FAST_BORROW=83, + LOAD_FAST_CHECK=84, + LOAD_FAST_LOAD_FAST=85, + LOAD_FROM_DICT_OR_DEREF=86, + LOAD_FROM_DICT_OR_GLOBALS=87, + LOAD_GLOBAL=88, + LOAD_NAME=89, + LOAD_SMALL_INT=90, + LOAD_SPECIAL=91, + LOAD_SUPER_ATTR=92, + MAKE_CELL=93, + MAP_ADD=94, + MATCH_CLASS=95, + POP_JUMP_IF_FALSE=96, + POP_JUMP_IF_NONE=97, + POP_JUMP_IF_NOT_NONE=98, + POP_JUMP_IF_TRUE=99, + RAISE_VARARGS=100, + RERAISE=101, + SEND=102, + SET_ADD=103, + SET_FUNCTION_ATTRIBUTE=104, + SET_UPDATE=105, + STORE_ATTR=106, + STORE_DEREF=107, + STORE_FAST=108, + STORE_FAST_LOAD_FAST=109, + STORE_FAST_STORE_FAST=110, + STORE_GLOBAL=111, + STORE_NAME=112, + SWAP=113, + UNPACK_EX=114, + UNPACK_SEQUENCE=115, + YIELD_VALUE=116, INSTRUMENTED_END_FOR=233, INSTRUMENTED_POP_ITER=234, INSTRUMENTED_END_SEND=235, diff --git a/Lib/_py_abc.py b/Lib/_py_abc.py index c870ae9048b4f13..2bfb9489a7acdcb 100644 --- a/Lib/_py_abc.py +++ b/Lib/_py_abc.py @@ -92,7 +92,12 @@ def _abc_caches_clear(cls): def __instancecheck__(cls, instance): """Override for isinstance(instance, cls).""" # Inline the cache checking - subclass = instance.__class__ + try: + subclass = instance.__class__ + except AttributeError: + # Fall back to the type when the instance has no __class__, + # matching the behaviour of the built-in isinstance() (gh-153772). + subclass = type(instance) if subclass in cls._abc_cache: return True subtype = type(instance) diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index 81a386c4487d953..ab09913de6812dd 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -620,17 +620,18 @@ def warn_explicit(message, category, filename, lineno, linecache.getlines(filename, module_globals) # Print message and context - msg = _wm.WarningMessage(message, category, filename, lineno, source=source) + msg = _wm.WarningMessage(message, category, filename, lineno, + module=module, source=source) _wm._showwarnmsg(msg) class WarningMessage(object): _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file", - "line", "source") + "line", "source", "module") def __init__(self, message, category, filename, lineno, file=None, - line=None, source=None): + line=None, source=None, module=None): self.message = message self.category = category self.filename = filename @@ -638,12 +639,14 @@ def __init__(self, message, category, filename, lineno, file=None, self.file = file self.line = line self.source = source + self.module = module self._category_name = category.__name__ if category else None def __str__(self): - return ("{message : %r, category : %r, filename : %r, lineno : %s, " - "line : %r}" % (self.message, self._category_name, - self.filename, self.lineno, self.line)) + return ("{message : %r, category : %r, module : %r, " + "filename : %r, lineno : %s, line : %r}" % ( + self.message, self._category_name, self.module, + self.filename, self.lineno, self.line)) def __repr__(self): return f'<{type(self).__qualname__} {self}>' diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b6d68f2372850a7..6f53f45d524c991 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -55,7 +55,7 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] @@ -272,12 +272,12 @@ def _wrap_strftime(object, format, timetuple): newformat.append(Zreplace) # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so # year 1000 for %G can go on the fast path. - elif ((ch in 'YG' or ch in 'FC') and - object.year < 1000 and _need_normalize_century()): + elif (ch in 'YGFC' and timetuple[0] < 1000 and + _need_normalize_century()): if ch == 'G': year = int(_time.strftime("%G", timetuple)) else: - year = object.year + year = timetuple[0] if ch == 'C': push('{:02}'.format(year // 100)) else: @@ -355,18 +355,30 @@ def _find_isoformat_datetime_separator(dtstr): return 8 +def _read_isoformat_component(s, n): + # The caller has verified the string is ASCII, so isdigit() matches only + # the ASCII digits accepted by the C parser. + if len(s) != n or not s.isdigit(): + raise ValueError("Invalid isoformat string") + return int(s) + + def _parse_isoformat_date(dtstr): # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator - assert len(dtstr) in (7, 8, 10) - year = int(dtstr[0:4]) + if len(dtstr) not in (7, 8, 10): + raise ValueError("Invalid isoformat string") + if not dtstr.isascii(): + raise ValueError("Invalid isoformat string") + + year = _read_isoformat_component(dtstr[0:4], 4) has_sep = dtstr[4] == '-' pos = 4 + has_sep if dtstr[pos:pos + 1] == "W": # YYYY-?Www-?D? pos += 1 - weekno = int(dtstr[pos:pos + 2]) + weekno = _read_isoformat_component(dtstr[pos:pos + 2], 2) pos += 2 dayno = 1 @@ -376,17 +388,17 @@ def _parse_isoformat_date(dtstr): pos += has_sep - dayno = int(dtstr[pos:pos + 1]) + dayno = _read_isoformat_component(dtstr[pos:pos + 1], 1) return list(_isoweek_to_gregorian(year, weekno, dayno)) else: - month = int(dtstr[pos:pos + 2]) + month = _read_isoformat_component(dtstr[pos:pos + 2], 2) pos += 2 if (dtstr[pos:pos + 1] == "-") != has_sep: raise ValueError("Inconsistent use of dash separator") pos += has_sep - day = int(dtstr[pos:pos + 2]) + day = _read_isoformat_component(dtstr[pos:pos + 2], 2) return [year, month, day] @@ -401,10 +413,7 @@ def _parse_hh_mm_ss_ff(tstr): time_comps = [0, 0, 0, 0] pos = 0 for comp in range(0, 3): - if (len_str - pos) < 2: - raise ValueError("Incomplete time component") - - time_comps[comp] = int(tstr[pos:pos+2]) + time_comps[comp] = _read_isoformat_component(tstr[pos:pos+2], 2) pos += 2 next_char = tstr[pos:pos+1] @@ -425,7 +434,7 @@ def _parse_hh_mm_ss_ff(tstr): raise ValueError("Invalid microsecond separator") else: pos += 1 - if not all(map(_is_ascii_digit, tstr[pos:])): + if not tstr[pos:].isdigit(): raise ValueError("Non-digit values in fraction") len_remainder = len_str - pos @@ -446,6 +455,8 @@ def _parse_isoformat_time(tstr): len_str = len(tstr) if len_str < 2: raise ValueError("Isoformat time too short") + if not tstr.isascii(): + raise ValueError("Invalid isoformat string") # This is equivalent to re.search('[+-Z]', tstr), but faster tz_pos = (tstr.find('-') + 1 or tstr.find('+') + 1 or tstr.find('Z') + 1) @@ -1987,7 +1998,7 @@ def fromisoformat(cls, date_string): if became_next_day: year, month, day = date_components # Only wrap day/month when it was previously valid - if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): # Calculate midnight of the next day day += 1 if day > days_in_month: diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index ef889ea0cc834c1..57d6143daea43a3 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -107,8 +107,8 @@ class DecimalException(ArithmeticError): anything, though. handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after + trap_enabler is not set. First argument is self, second is + the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) @@ -225,11 +225,12 @@ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. + detected during an operation. This can occur if contexts are not + checked on creation and either the precision exceeds the capability of + the underlying concrete representation or an unknown or unsupported + rounding was specified. These aspects of the context need only be + checked when the values are required to be used. The result is + [0,qNaN]. """ def handle(self, context, *args): @@ -322,8 +323,9 @@ class FloatOperation(DecimalException, TypeError): Decimal.from_float() or context.create_decimal_from_float() do not set the flag. - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. + Otherwise (the signal is trapped), only equality comparisons and + explicit conversions are silent. All other mixed operations raise + FloatOperation. """ # List of public traps and flags @@ -1822,9 +1824,10 @@ def __round__(self, n=None): Decimal('123.46') >>> round(Decimal('123.456'), -2) Decimal('1E+2') - >>> round(Decimal('-Infinity'), 37) + >>> with localcontext(ExtendedContext): + ... round(Decimal('-Infinity'), 37) + ... round(Decimal('sNaN123'), 0) Decimal('NaN') - >>> round(Decimal('sNaN123'), 0) Decimal('NaN123') """ @@ -2901,8 +2904,8 @@ def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. """ other = _convert_other(other, raiseit=True) @@ -2973,7 +2976,8 @@ def compare_total(self, other, context=None): def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ other = _convert_other(other, raiseit=True) @@ -4002,6 +4006,20 @@ def copy(self): return nc __copy__ = copy + def __replace__(self, /, **changes): + """Returns a copy of self with the specified attributes replaced.""" + unexpected = changes.keys() - _context_attributes + if unexpected: + raise TypeError(f'__replace__() got an unexpected keyword ' + f'argument {min(unexpected)!r}') + nc = self.copy() + for name, value in changes.items(): + if name in ('flags', 'traps') and isinstance(value, list): + # As in the constructor, accept a list of signals. + value = dict((s, int(s in value)) for s in _signals + value) + setattr(nc, name, value) + return nc + def _raise_error(self, condition, explanation = None, *args): """Handles an error @@ -4110,9 +4128,9 @@ def create_decimal_from_float(self, f): def abs(self, a): """Returns the absolute value of the operand. - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. + If the operand is negative, the result is the same as using the + minus operation on the operand. Otherwise, the result is the same + as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') @@ -4168,16 +4186,17 @@ def canonical(self, a): def compare(self, a, b): """Compares values numerically. - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. + If the signs of the operands differ, a value representing each + operand ('-1' if the operand is less than zero, '0' if the operand + is zero or negative zero, or '1' if the operand is greater than + zero) is used in place of that operand for the comparison instead of + the actual operand. - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. + The comparison is then effected by subtracting the second operand + from the first and then returning a value according to the result of + the subtraction: '-1' if the result is less than zero, '0' if the + result is zero or negative zero, or '1' if the result is greater + than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') @@ -4240,8 +4259,8 @@ def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') @@ -4268,7 +4287,8 @@ def compare_total(self, a, b): def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) @@ -4926,8 +4946,8 @@ def multiply(self, a, b): If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. + ('long multiplication'), resulting in a number which may be as long + as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') @@ -5203,19 +5223,19 @@ def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). + operand. It may be rounded using the current rounding setting (if + the exponent is being increased), multiplied by a positive power of + ten (if the exponent is being decreased), or is unchanged (if the + exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. + operation condition is raised. This guarantees that, unless there + is an error condition, the exponent of the result of a quantize is + always equal to that of the right-hand operand. - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. + Also unlike other operations, quantize will never raise Underflow, + even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') @@ -5269,13 +5289,13 @@ def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. + calculating integer division as described for divide-integer, + rounded to precision digits if necessary. The sign of the result, + if non-zero, is the same as that of the original dividend. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') @@ -5309,9 +5329,9 @@ def remainder_near(self, a, b): is chosen). If the result is equal to 0 then its sign will be the sign of a. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') @@ -5369,8 +5389,8 @@ def rotate(self, a, b): def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. - The result is never affected by either the sign or the coefficient of - either operand. + The result is never affected by either the sign or the coefficient + of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False @@ -5442,8 +5462,8 @@ def shift(self, a, b): def sqrt(self, a): """Square root of a non-negative number to context precision. - If the result must be inexact, it is rounded using the round-half-even - algorithm. + If the result must be inexact, it is rounded using the + round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 3306c8a274760be..cf4ef04f37d26cc 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -10,7 +10,7 @@ import sys # Import _thread instead of threading to reduce startup cost from _thread import allocate_lock as Lock -if sys.platform in {'win32', 'cygwin'}: +if sys.platform == 'win32': from msvcrt import setmode as _setmode else: _setmode = None @@ -83,27 +83,28 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) - mode is an optional string that specifies the mode in which the file is - opened. It defaults to 'r' which means open for reading in text mode. Other - common values are 'w' for writing (truncating the file if it already - exists), 'x' for exclusive creation of a new file, and 'a' for appending - (which on some Unix systems, means that all writes append to the end of the - file regardless of the current seek position). In text mode, if encoding is - not specified the encoding used is platform dependent. (For reading and - writing raw bytes use binary mode and leave encoding unspecified.) The - available modes are: - - ========= =============================================================== + mode is an optional string that specifies the mode in which the file + is opened. It defaults to 'r' which means open for reading in text + mode. Other common values are 'w' for writing (truncating the file if + it already exists), 'x' for exclusive creation of a new file, and + 'a' for appending (which on some Unix systems, means that all writes + append to the end of the file regardless of the current seek position). + In text mode, if encoding is not specified the encoding used is platform + dependent. (For reading and writing raw bytes use binary mode and leave + encoding unspecified.) The available modes are: + + ========= ========================================================== Character Meaning - --------- --------------------------------------------------------------- + --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing - 'a' open for writing, appending to the end of the file if it exists + 'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - ========= =============================================================== + ========= ========================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while @@ -111,23 +112,23 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, - even when the underlying operating system doesn't. Files opened in + even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as - bytes objects without any decoding. In text mode (the default, or when + bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. - Pass 0 to switch buffering off (only allowed in binary mode), 1 to select - line buffering (only usable in text mode), and an integer > 1 to indicate - the size of a fixed-size chunk buffer. When no buffering argument is - given, the default buffering policy works as follows: + Pass 0 to switch buffering off (only allowed in binary mode), 1 to + select line buffering (only usable in text mode), and an integer > 1 to + indicate the size of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -147,8 +148,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, encoding error strings. newline is a string controlling how universal newlines works (it only - applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works - as follows: + applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It + works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and @@ -164,17 +165,17 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, other legal values, any '\n' characters written are translated to the given string. - closedfd is a bool. If closefd is False, the underlying file descriptor will - be kept open when the file is closed. This does not work when a file name is - given and must be True in that case. + closedfd is a bool. If closefd is False, the underlying file descriptor + will be kept open when the file is closed. This does not work when + a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as *opener*. The - underlying file descriptor for the file object is then obtained by calling - *opener* with (*file*, *flags*). *opener* must return an open file - descriptor (passing os.open as *opener* results in functionality similar to - passing None). + underlying file descriptor for the file object is then obtained by + calling *opener* with (*file*, *flags*). *opener* must return an open + file descriptor (passing os.open as *opener* results in functionality + similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing @@ -351,10 +352,12 @@ def seek(self, pos, whence=0): interpreted relative to the position indicated by whence. Values for whence are ints: - * 0 -- start of stream (the default); offset should be zero or positive + * 0 -- start of stream (the default); offset should be zero or + positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. + Some operating systems / file systems could provide additional + values. Return an int indicating the new absolute position. """ @@ -367,8 +370,8 @@ def tell(self): def truncate(self, pos=None): """Truncate file to size bytes. - Size defaults to the current IO position as reported by tell(). Return - the new size. + Size defaults to the current IO position as reported by tell(). + Return the new size. """ self._unsupported("truncate") @@ -492,7 +495,8 @@ def __exit__(self, *args): def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An OSError is raised if the IO object does not use a file + descriptor. """ self._unsupported("fileno") @@ -917,7 +921,7 @@ def getbuffer(self): def close(self): if self._buffer is not None: - self._buffer.clear() + self._buffer = bytearray() super().close() def read(self, size=-1): @@ -941,7 +945,7 @@ def read(self, size=-1): newpos = min(len(self._buffer), self._pos + size) b = self._buffer[self._pos : newpos] self._pos = newpos - return bytes(b) + return b.take_bytes() def read1(self, size=-1): """This is the same as read. @@ -996,6 +1000,23 @@ def tell(self): raise ValueError("tell on closed file") return self._pos + def peek(self, size=0): + if self.closed: + raise ValueError("peek on closed file") + try: + size_index = size.__index__ + except AttributeError: + raise TypeError(f"{size!r} is not an integer") + else: + size = size_index() + + if size < 1: + size = io.DEFAULT_BUFFER_SIZE + + with self._lock: + b = self._buffer[self._pos:self._pos + size] + return b.take_bytes() + def truncate(self, pos=None): if self.closed: raise ValueError("truncate on closed file") @@ -1505,17 +1526,22 @@ class FileIO(RawIOBase): _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). + """Open a file. + + The mode can be 'r' (default), 'w', 'x' or 'a' for reading, + writing, exclusive creation or appending. The file will be created + if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. A FileExistsError will be raised + if it already exists when opened for creating. Opening a file for + creating implies writing so this mode behaves in a similar way to + 'w'. Add a '+' to the mode to allow simultaneous reading and + writing. + + A custom opener can be used by passing a callable as *opener*. + The underlying file descriptor for the file object is then obtained + by calling opener with (*name*, *flags*). *opener* must return + an open file descriptor (passing os.open as *opener* results in + functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. @@ -1754,8 +1780,8 @@ def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. + The number of bytes actually written is returned. In non-blocking + mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() @@ -1767,11 +1793,12 @@ def write(self, b): def seek(self, pos, whence=SEEK_SET): """Move to new file position. - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). + Argument offset is a byte count. Optional argument whence defaults + to SEEK_SET or 0 (offset from start of file, offset should be >= 0); + other values are SEEK_CUR or 1 (move relative to current position, + positive or negative), and SEEK_END or 2 (move relative to end of + file, usually negative, although many platforms allow seeking beyond + the end of a file). Note that not all file objects are seekable. """ @@ -1804,8 +1831,8 @@ def truncate(self, size=None): def close(self): """Close the file. - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. + A closed file cannot be used for further I/O operations. + close() may be called more than once without error. """ if not self.closed: self._stat_atopen = None @@ -1903,8 +1930,8 @@ class TextIOBase(IOBase): def read(self, size=-1): """Read at most size characters from stream, where size is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have size characters or we hit + EOF. If size is negative or omitted, read until EOF. Returns a string. """ @@ -1918,7 +1945,7 @@ def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index a22b0297b24ea07..280a666e0c78f37 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import importlib import os import pkgutil @@ -13,13 +11,11 @@ from dataclasses import dataclass from itertools import chain from tokenize import TokenInfo +from .fancycompleter import safe_getattr -TYPE_CHECKING = False - -if TYPE_CHECKING: - from types import ModuleType - from typing import Any, Iterable, Iterator, Mapping - from .types import CompletionAction +lazy from types import ModuleType +lazy from typing import Any, Iterable, Iterator, Mapping +lazy from .types import CompletionAction HARDCODED_SUBMODULES = { @@ -71,41 +67,69 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self._curr_sys_path: list[str] = sys.path[:] self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> tuple[list[str], CompletionAction | None] | None: + def get_completions( + self, line: str, *, include_values: bool = True + ) -> tuple[list[str], list[Any], CompletionAction | None] | None: """Return the next possible import completions for 'line'. For attributes completion, if the module to complete from is not imported, also return an action (prompt + callback to run if the user press TAB again) to import the module. + + If *include_values* is false, the returned values list is empty and + attribute values are not resolved. """ result = ImportParser(line).parse() if not result: return None try: - return self.complete(*result) + return self.complete(*result, include_values=include_values) except Exception: # Some unexpected error occurred, make it look like # no completions are available - return [], None - - def complete(self, from_name: str | None, name: str | None) -> tuple[list[str], CompletionAction | None]: + return [], [], None + + def complete( + self, + from_name: str | None, + name: str | None, + *, + include_values: bool = True, + ) -> tuple[list[str], list[Any], CompletionAction | None]: if from_name is None: # import x.y.z assert name is not None path, prefix = self.get_path_and_prefix(name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules], None + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None if name is None: # from x.y.z path, prefix = self.get_path_and_prefix(from_name) modules = self.find_modules(path, prefix) - return [self.format_completion(path, module) for module in modules], None + names = [self.format_completion(path, module) for module in modules] + # These are always modules, use dummy values to get the right color + values = [sys] * len(names) if include_values else [] + return names, values, None # from x.y import z submodules = self.find_modules(from_name, name) - attributes, action = self.find_attributes(from_name, name) - return sorted({*submodules, *attributes}), action + attr_names, attr_module, action = self._find_attributes(from_name, name) + all_names = sorted({*submodules, *attr_names}) + if not include_values: + return all_names, [], action + + # Build values list matching the sorted order: + # submodules use `sys` as a dummy value so they get the 'module' color, + # attributes use their actual value. + attr_map = {} + if attr_module is not None: + attr_map = {n: safe_getattr(attr_module, n) for n in attr_names} + all_values = [attr_map[n] if n in attr_map else sys for n in all_names] + return all_names, all_values, action def find_modules(self, path: str, prefix: str) -> list[str]: """Find all modules under 'path' that start with 'prefix'.""" @@ -166,31 +190,43 @@ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: return (isinstance(module_info.module_finder, FileFinder) and module_info.module_finder.path == self._stdlib_path) - def find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + def find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], list[Any], CompletionAction | None]: """Find all attributes of module 'path' that start with 'prefix'.""" - attributes, action = self._find_attributes(path, prefix) - # Filter out invalid attribute names - # (for example those containing dashes that cannot be imported with 'import') - return [attr for attr in attributes if attr.isidentifier()], action + attributes, module, action = self._find_attributes(path, prefix) + if module is not None: + values = [safe_getattr(module, attr) for attr in attributes] + else: + values = [] + return attributes, values, action - def _find_attributes(self, path: str, prefix: str) -> tuple[list[str], CompletionAction | None]: + def _find_attributes( + self, path: str, prefix: str + ) -> tuple[list[str], ModuleType | None, CompletionAction | None]: path = self._resolve_relative_path(path) # type: ignore[assignment] if path is None: - return [], None + return [], None, None imported_module = sys.modules.get(path) if not imported_module: if path in self._failed_imports: # Do not propose to import again - return [], None + return [], None, None imported_module = self._maybe_import_module(path) if not imported_module: - return [], self._get_import_completion_action(path) + return [], None, self._get_import_completion_action(path) try: module_attributes = dir(imported_module) except Exception: module_attributes = [] - return [attr_name for attr_name in module_attributes - if self.is_suggestion_match(attr_name, prefix)], None + # Filter out invalid attribute names, such as dashes that cannot be + # imported with 'import'. + names = [ + attr_name for attr_name in module_attributes + if (self.is_suggestion_match(attr_name, prefix) + and attr_name.isidentifier()) + ] + return names, imported_module, None def is_suggestion_match(self, module_name: str, prefix: str) -> bool: if prefix: diff --git a/Lib/_pyrepl/_threading_handler.py b/Lib/_pyrepl/_threading_handler.py index 82f5e8650a20728..040c49724d0b54a 100644 --- a/Lib/_pyrepl/_threading_handler.py +++ b/Lib/_pyrepl/_threading_handler.py @@ -1,30 +1,9 @@ -from __future__ import annotations - from dataclasses import dataclass, field import traceback -TYPE_CHECKING = False -if TYPE_CHECKING: - from threading import Thread - from types import TracebackType - from typing import Protocol - - class ExceptHookArgs(Protocol): - @property - def exc_type(self) -> type[BaseException]: ... - @property - def exc_value(self) -> BaseException | None: ... - @property - def exc_traceback(self) -> TracebackType | None: ... - @property - def thread(self) -> Thread | None: ... - - class ShowExceptions(Protocol): - def __call__(self) -> int: ... - def add(self, s: str) -> None: ... - - from .reader import Reader +lazy from threading import ExceptHookArgs +lazy from .reader import Reader def install_threading_hook(reader: Reader) -> None: diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index 0589a0f437ec7c5..e30ca0d26217fb8 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -54,14 +54,6 @@ def empty(self) -> bool: """ return not self.events - def flush_buf(self) -> bytearray: - """ - Flushes the buffer and returns its contents. - """ - old = self.buf - self.buf = bytearray() - return old - def insert(self, event: Event) -> None: """ Inserts an event into the queue. @@ -87,7 +79,7 @@ def push(self, char: int | bytes) -> None: if isinstance(k, dict): self.keymap = k else: - self.insert(Event('key', k, bytes(self.flush_buf()))) + self.insert(Event('key', k, self.buf.take_bytes())) self.keymap = self.compiled_keymap elif self.buf and self.buf[0] == 27: # escape @@ -97,14 +89,14 @@ def push(self, char: int | bytes) -> None: trace('unrecognized escape sequence, propagating...') self.keymap = self.compiled_keymap self.insert(Event('key', '\033', b'\033')) - for _c in self.flush_buf()[1:]: + for _c in self.buf.take_bytes()[1:]: self.push(_c) else: try: - decoded = bytes(self.buf).decode(self.encoding) + decoded = self.buf.decode(self.encoding) except UnicodeError: return else: - self.insert(Event('key', decoded, bytes(self.flush_buf()))) + self.insert(Event('key', decoded, self.buf.take_bytes())) self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index e79fbfa6bb0b386..cf8b1a6c1e4ec4d 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -19,10 +19,8 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations import os import time -from typing import TYPE_CHECKING # Categories of actions: # killing @@ -37,8 +35,7 @@ from .trace import trace # types -if TYPE_CHECKING: - from .historical_reader import HistoricalReader +lazy from .historical_reader import HistoricalReader class Command: diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py index f783e8db36b0287..43f5159e20ab6a3 100644 --- a/Lib/_pyrepl/completing_reader.py +++ b/Lib/_pyrepl/completing_reader.py @@ -18,10 +18,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations - from dataclasses import dataclass, field -from typing import TYPE_CHECKING import re from . import commands, console, reader @@ -30,9 +27,7 @@ # types -Command = commands.Command -if TYPE_CHECKING: - from .types import CommandName, CompletionAction, Keymap, KeySpec +lazy from .types import CompletionAction, Keymap def prefix(wordlist: list[str], j: int = 0) -> str: @@ -285,7 +280,7 @@ def collect_keymap(self) -> Keymap: return super().collect_keymap() + ( (r'\t', 'complete'),) - def after_command(self, cmd: Command) -> None: + def after_command(self, cmd: commands.Command) -> None: super().after_command(cmd) if not isinstance(cmd, (complete, self_insert)): self.cmpltn_reset() diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py index 2a53d5ff581fa2c..dcf8ff9b083caa3 100644 --- a/Lib/_pyrepl/console.py +++ b/Lib/_pyrepl/console.py @@ -17,8 +17,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations - import os import _colorize @@ -29,15 +27,13 @@ from dataclasses import dataclass import re import sys -from typing import TYPE_CHECKING from .render import RenderedScreen from .trace import trace -if TYPE_CHECKING: - from typing import Callable, IO - - from .types import CursorXY +lazy from collections.abc import Callable +lazy from typing import IO +lazy from .types import CursorXY @dataclass diff --git a/Lib/_pyrepl/content.py b/Lib/_pyrepl/content.py index 3cb22fee84b7406..2e759072662ba4d 100644 --- a/Lib/_pyrepl/content.py +++ b/Lib/_pyrepl/content.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from dataclasses import dataclass from .utils import ColorSpan, StyleRef, THEME, iter_display_chars, unbracket, wlen diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 8d5bd183f213398..076a38ad06da178 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -19,13 +19,7 @@ import termios - -TYPE_CHECKING = False - -if TYPE_CHECKING: - from typing import cast -else: - cast = lambda typ, val: val +from typing import cast class TermState: diff --git a/Lib/_pyrepl/fancycompleter.py b/Lib/_pyrepl/fancycompleter.py index 7a639afd74ef3cc..9b779d79a7ffcb4 100644 --- a/Lib/_pyrepl/fancycompleter.py +++ b/Lib/_pyrepl/fancycompleter.py @@ -8,6 +8,43 @@ import keyword import types +lazy from typing import Any +lazy from _colorize import Theme + + +def safe_getattr(obj: object, name: str) -> object: + # Mirror rlcompleter's safeguards so completion does not + # call properties or reify lazy module attributes. + if isinstance(getattr(type(obj), name, None), property): + return None + if (isinstance(obj, types.ModuleType) + and isinstance(obj.__dict__.get(name), types.LazyImportType) + ): + return obj.__dict__.get(name) + return getattr(obj, name, None) + + +def colorize_matches(names: list[str], values: list[Any], theme: Theme) -> list[str]: + return [ + _color_for_obj(name, obj, theme) + for name, obj in zip(names, values) + ] + +def _color_for_obj(name: str, value: object, theme: Theme) -> str: + t = type(value) + color = _color_by_type(t, theme) + return f"{color}{name}{ANSIColors.RESET}" + + +def _color_by_type(t, theme): + typename = t.__name__ + # this is needed e.g. to turn method-wrapper into method_wrapper, + # because if we want _colorize.FancyCompleter to be "dataclassable" + # our keys need to be valid identifiers. + typename = typename.replace('-', '_').replace('.', '_') + return getattr(theme.fancycompleter, typename, ANSIColors.RESET) + + class Completer(rlcompleter.Completer): """ When doing something like a.b., keep the full a.b.attr completion @@ -143,21 +180,7 @@ def _attr_matches(self, text): word[:n] == attr and not (noprefix and word[:n+1] == noprefix) ): - # Mirror rlcompleter's safeguards so completion does not - # call properties or reify lazy module attributes. - if isinstance(getattr(type(thisobject), word, None), property): - value = None - elif ( - isinstance(thisobject, types.ModuleType) - and isinstance( - thisobject.__dict__.get(word), - types.LazyImportType, - ) - ): - value = thisobject.__dict__.get(word) - else: - value = getattr(thisobject, word, None) - + value = safe_getattr(thisobject, word) names.append(word) values.append(value) if names or not noprefix: @@ -170,23 +193,7 @@ def _attr_matches(self, text): return expr, attr, names, values def colorize_matches(self, names, values): - return [ - self._color_for_obj(name, obj) - for name, obj in zip(names, values) - ] - - def _color_for_obj(self, name, value): - t = type(value) - color = self._color_by_type(t) - return f"{color}{name}{ANSIColors.RESET}" - - def _color_by_type(self, t): - typename = t.__name__ - # this is needed e.g. to turn method-wrapper into method_wrapper, - # because if we want _colorize.FancyCompleter to be "dataclassable" - # our keys need to be valid identifiers. - typename = typename.replace('-', '_').replace('.', '_') - return getattr(self.theme.fancycompleter, typename, ANSIColors.RESET) + return colorize_matches(names, values, self.theme) def commonprefix(names): diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py index 09b969d80bc231f..e0d315915987651 100644 --- a/Lib/_pyrepl/historical_reader.py +++ b/Lib/_pyrepl/historical_reader.py @@ -17,17 +17,13 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations - from contextlib import contextmanager from dataclasses import dataclass, field from . import commands, input from .reader import Reader - -if False: - from .types import SimpleContextManager, KeySpec, CommandName +lazy from .types import SimpleContextManager, KeySpec, CommandName isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( @@ -265,6 +261,7 @@ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: (r"\C-s", "forward-history-isearch"), (r"\M-r", "restore-history"), (r"\M-.", "yank-arg"), + (r"\M-_", "yank-arg"), (r"\", "history-search-forward"), (r"\x1b[6~", "history-search-forward"), (r"\", "history-search-backward"), diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py index 2d65246c700f274..56781662b4fda27 100644 --- a/Lib/_pyrepl/input.py +++ b/Lib/_pyrepl/input.py @@ -33,17 +33,12 @@ # [meta-key] is identified with [esc key]. We demand that any console # class does quite a lot towards emulating a unix terminal. -from __future__ import annotations - from abc import ABC, abstractmethod import unicodedata from collections import deque -from typing import TYPE_CHECKING - # types -if TYPE_CHECKING: - from .types import EventTuple +lazy from .types import EventTuple class InputTranslator(ABC): diff --git a/Lib/_pyrepl/layout.py b/Lib/_pyrepl/layout.py index 6d854d1142dd9f1..cff0821cb95f72b 100644 --- a/Lib/_pyrepl/layout.py +++ b/Lib/_pyrepl/layout.py @@ -1,12 +1,11 @@ """Wrap content lines to the terminal width before rendering.""" -from __future__ import annotations - from dataclasses import dataclass from typing import Self from .content import ContentFragment, ContentLine -from .types import CursorXY, ScreenInfoRow + +lazy from .types import CursorXY, ScreenInfoRow @dataclass(frozen=True, slots=True) diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini index 9375a55b53ce8bc..2510ee8c2d3f5a1 100644 --- a/Lib/_pyrepl/mypy.ini +++ b/Lib/_pyrepl/mypy.ini @@ -6,7 +6,7 @@ files = Lib/_pyrepl mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy explicit_package_bases = True -python_version = 3.13 +python_version = 3.15 platform = linux pretty = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index 92afaa5933a2250..d6f20fe7557e6b3 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import io import os import re diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index b8e1e425b0bb359..b5686409c37cdb8 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -19,8 +19,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations - import sys import _colorize @@ -38,12 +36,13 @@ ) from .layout import LayoutMap, LayoutResult, LayoutRow, WrappedRow, layout_content_lines from .render import RenderCell, RenderLine, RenderedScreen, ScreenOverlay -from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, wlen, gen_colors +from .utils import ANSI_ESCAPE_SEQUENCE, ColorSpan, THEME, StyleRef, gen_colors from .trace import trace # types Command = commands.Command +from collections.abc import Callable, Iterator from .types import ( Callback, CommandName, @@ -304,6 +303,7 @@ class Reader: lxy: CursorXY = field(init=False) scheduled_commands: list[CommandName] = field(default_factory=list) can_colorize: bool = False + gen_colors: Callable[[str], Iterator[ColorSpan]] = gen_colors threading_hook: Callback | None = None invalidation: RefreshInvalidation = field(init=False) @@ -534,7 +534,7 @@ def _build_content_lines( prompt_from_cache: bool, ) -> tuple[ContentLine, ...]: if self.can_colorize: - colors = list(gen_colors(self.get_unicode())) + colors = list(self.gen_colors(self.get_unicode())) else: colors = None trace("colors = {colors}", colors=colors) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index f8f1727d2a1d1f4..1d00de3683db231 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -26,8 +26,6 @@ extensions for multiline input. """ -from __future__ import annotations - import warnings from dataclasses import dataclass, field @@ -40,7 +38,7 @@ from .completing_reader import CompletingReader, stripcolor from .console import Console as ConsoleType from ._module_completer import ModuleCompleter, make_default_module_completer -from .fancycompleter import Completer as FancyCompleter +from .fancycompleter import Completer as FancyCompleter, colorize_matches Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] @@ -55,13 +53,10 @@ # types Command = commands.Command -from collections.abc import Callable, Collection +from collections.abc import Callable, Collection, Mapping from .types import Callback, Completer, KeySpec, CommandName, CompletionAction -TYPE_CHECKING = False - -if TYPE_CHECKING: - from typing import Any, Mapping +lazy from typing import Any MoreLinesCallable = Callable[[str], bool] @@ -104,6 +99,7 @@ class ReadlineConfig: readline_completer: Completer | None = None completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") module_completer: ModuleCompleter = field(default_factory=make_default_module_completer) + colorize_completions: Callable[[list[str], list[Any]], list[str]] | None = None @dataclass(kw_only=True) class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): @@ -169,8 +165,17 @@ def get_completions(self, stem: str) -> tuple[list[str], CompletionAction | None return result, None def get_module_completions(self) -> tuple[list[str], CompletionAction | None] | None: - line = self.get_line() - return self.config.module_completer.get_completions(line) + line = stripcolor(self.get_line()) + colorize_completions = self.config.colorize_completions + result = self.config.module_completer.get_completions( + line, include_values=bool(colorize_completions) + ) + if result is None: + return None + names, values, action = result + if colorize_completions: + names = colorize_completions(names, values) + return names, action def get_trimmed_history(self, maxlength: int) -> list[str]: if maxlength >= 0: @@ -616,13 +621,19 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) - _wrapper.config.module_completer = ModuleCompleter(namespace) use_basic_completer = ( not sys.flags.ignore_environment and os.getenv("PYTHON_BASIC_COMPLETER") ) completer_cls = RLCompleter if use_basic_completer else FancyCompleter - _wrapper.config.readline_completer = completer_cls(namespace).complete + completer = completer_cls(namespace) + _wrapper.config.readline_completer = completer.complete + if isinstance(completer, FancyCompleter) and completer.use_colors: + theme = completer.theme + def _colorize(names: list[str], values: list[object]) -> list[str]: + return colorize_matches(names, values, theme) + _wrapper.config.colorize_completions = _colorize + _wrapper.config.module_completer = ModuleCompleter(namespace) # this is not really what readline.c does. Better than nothing I guess import builtins diff --git a/Lib/_pyrepl/render.py b/Lib/_pyrepl/render.py index b821f35d850825e..06703885735e816 100644 --- a/Lib/_pyrepl/render.py +++ b/Lib/_pyrepl/render.py @@ -1,11 +1,11 @@ -from __future__ import annotations - from collections.abc import Iterable, Sequence from dataclasses import dataclass, field from typing import Literal, Protocol, Self -from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, str_width +from _colorize import ANSIColors + from .types import CursorXY +from .utils import ANSI_ESCAPE_SEQUENCE, THEME, StyleRef, str_width type RenderStyle = StyleRef | str | None type LineUpdateKind = Literal[ @@ -55,7 +55,7 @@ def _style_escape(style: StyleRef) -> str: def _update_terminal_state(state: str, escape: str) -> str: - if escape in {"\x1b[0m", "\x1b[m"}: + if escape in {ANSIColors.RESET, "\x1b[m"}: return "" return state + escape @@ -344,14 +344,14 @@ def render_cells( target_escape += visual_style if target_escape != active_escape: if active_escape: - rendered.append("\x1b[0m") + rendered.append(ANSIColors.RESET) if target_escape: rendered.append(target_escape) active_escape = target_escape rendered.append(cell.text) if active_escape: - rendered.append("\x1b[0m") + rendered.append(ANSIColors.RESET) return "".join(rendered) diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index c169d0191bd8333..e6c355388a7c074 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -23,8 +23,6 @@ allowing multiline input and multiline history entries. """ -from __future__ import annotations - import _sitebuiltins import functools import os @@ -68,6 +66,7 @@ def _clear_screen(): reader.scheduled_commands.append("clear_screen") +# Keep this in sync with _pyrepl.utils.COMMANDS REPL_COMMANDS = { "exit": _sitebuiltins.Quitter('exit', ''), "quit": _sitebuiltins.Quitter('quit' ,''), diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index 395867805196a5c..1c24ab479fcb635 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,11 +1,7 @@ -from __future__ import annotations - import os import sys -# types -if False: - from typing import IO +lazy from typing import IO trace_file: IO[str] | None = None diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index 9c4644db53e3436..0749a56d86fa4a6 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -19,8 +19,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations - import errno import os import re @@ -34,7 +32,9 @@ from collections.abc import Callable from dataclasses import dataclass from fcntl import ioctl -from typing import TYPE_CHECKING, cast, overload +from typing import AbstractSet, IO, Literal, overload + +from _colorize import ANSIColors from . import terminfo from .console import Console, Event @@ -59,9 +59,6 @@ posix = None # types -if TYPE_CHECKING: - from typing import AbstractSet, IO, Literal - type _MoveFunc = Callable[[int, int], None] type _PendingWrite = tuple[str | bytes, bool] @@ -517,6 +514,7 @@ def restore(self) -> None: Restore the console to the default state """ trace("unix.restore") + self.__write(ANSIColors.RESET) self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() @@ -654,6 +652,7 @@ def finish(self): while y >= 0 and not rendered_lines[y].text: y -= 1 self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write(ANSIColors.RESET) self.__write("\n\r") self.flushoutput() diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index b50426c31ead53a..e77001f6a1d10bb 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -1,4 +1,3 @@ -from __future__ import annotations import builtins import functools import keyword @@ -9,10 +8,12 @@ import _colorize from collections import deque +from collections.abc import Iterable, Iterator from dataclasses import dataclass from io import StringIO +from re import Match from tokenize import TokenInfo as TI -from typing import Iterable, Iterator, Match, NamedTuple, Self +from typing import NamedTuple, Self from .types import CharBuffer, CharWidths from .trace import trace @@ -24,6 +25,8 @@ IDENTIFIERS_AFTER = frozenset({"def", "class"}) KEYWORD_CONSTANTS = frozenset({"True", "False", "None"}) BUILTINS = frozenset({str(name) for name in dir(builtins) if not name.startswith('_')}) +# Keep this in sync with _pyrepl.simple_interact.REPL_COMMANDS +COMMANDS = frozenset({"exit", "quit", "copyright", "help", "clear"}) def THEME(**kwargs): @@ -235,6 +238,13 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") + elif ( + token.string in COMMANDS + and (not prev_token or prev_token.type == T.INDENT) + and (not next_token or next_token.type == T.NEWLINE) + ): + span = Span.from_token(token, line_lengths) + yield ColorSpan(span, "command") elif ( token.string in BUILTINS and not (prev_token and prev_token.exact_type == T.DOT) @@ -259,7 +269,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), TI(string="match"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{" | "~" | "...") ): return True case ( @@ -274,7 +284,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "-" | "[" | "{") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{") ): return True case ( diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index c1f9a19545d35fd..65eacf90fde685d 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -17,8 +17,6 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import annotations - import io import os import sys @@ -38,7 +36,9 @@ SHORT, ) from ctypes import Structure, POINTER, Union -from typing import TYPE_CHECKING + +from _colorize import ANSIColors + from .console import Event, Console from .render import ( EMPTY_RENDER_LINE, @@ -73,8 +73,7 @@ def __init__(self, err: int | None, descr: str | None = None) -> None: except ImportError: nt = None -if TYPE_CHECKING: - from typing import IO +lazy from typing import IO # Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes VK_MAP: dict[int, str] = { @@ -480,6 +479,7 @@ def prepare(self) -> None: def restore(self) -> None: trace("windows.restore") if self.__vt_support: + self.__write(ANSIColors.RESET) # Recover to original mode before running REPL self._disable_bracketed_paste() if not SetConsoleMode(InHandle, self.__original_input_mode): @@ -647,6 +647,7 @@ def finish(self) -> None: while y >= 0 and not rendered_lines[y].text: y -= 1 self._move_relative(0, min(y, self.height + self.__offset - 1)) + self.__write(ANSIColors.RESET) self.__write("\r\n") def flushoutput(self) -> None: diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index 84551e3546eb6ee..5259a36123ea276 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -55,7 +55,6 @@ def __setup(self): if not data: data = self.__data self.__lines = data.split('\n') - self.__linecnt = len(self.__lines) def __repr__(self): self.__setup() diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 3367ac485a590c0..59ac96745aa15e2 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -238,7 +238,7 @@ def __calc_date_time(self): current_format = current_format.replace(tz, "%Z") # Transform all non-ASCII digits to digits in range U+0660 to U+0669. if not current_format.isascii() and self.LC_alt_digits is None: - current_format = re_sub(r'\d(?', mode='exec', *, type_comments=False, feature_version=None, optimize=-1, module=None): @@ -117,21 +117,34 @@ def _convert_literal(node): def dump( node, annotate_fields=True, include_attributes=False, *, - indent=None, show_empty=False, + color=False, indent=None, show_empty=False, ): """ Return a formatted dump of the tree in node. This is mainly useful for - debugging purposes. If annotate_fields is true (by default), - the returned string will show the names and the values for fields. - If annotate_fields is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - include_attributes can be set to true. If indent is a non-negative - integer or string, then the tree will be pretty-printed with that indent - level. None (the default) selects the single line representation. + debugging purposes. + + If annotate_fields is true (by default), the returned string will show the + names and the values for fields. If annotate_fields is false, the result + string will be more compact by omitting unambiguous field names. + + Attributes such as line numbers and column offsets are not dumped by default. + If this is wanted, include_attributes can be set to true. + + If color is true, the returned string is syntax highlighted using ANSI + escape sequences. If color is false (the default), colored output is always + disabled. + + If indent is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. If indent is None (the default), + the tree is dumped on a single line. + If show_empty is False, then empty lists and fields that are None will be omitted from the output for better readability. """ + from _colorize import get_theme + + t = get_theme(force_color=color, force_no_color=not color).ast + def _format(node, level=0): if indent is not None: level += 1 @@ -166,7 +179,9 @@ def _format(node, level=0): field_type = cls._field_types.get(name, object) if field_type is expr_context: if not keywords: - args_buffer.append(repr(value)) + args_buffer.append( + f'{t.node}{type(value).__name__}' + f'{t.reset}()') continue if not keywords: args.extend(args_buffer) @@ -174,7 +189,7 @@ def _format(node, level=0): value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append('%s=%s' % (name, value)) + args.append(f'{t.field}{name}{t.reset}={value}') else: args.append(value) if include_attributes and node._attributes: @@ -187,14 +202,21 @@ def _format(node, level=0): continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append('%s=%s' % (name, value)) + args.append(f'{t.attribute}{name}{t.reset}={value}') + cls_name = f'{t.node}{cls.__name__}{t.reset}' if allsimple and len(args) <= 3: - return '%s(%s)' % (node.__class__.__name__, ', '.join(args)), not args - return '%s(%s%s)' % (node.__class__.__name__, prefix, sep.join(args)), False + return f'{cls_name}({", ".join(args)})', not args + return f'{cls_name}({prefix}{sep.join(args)})', False elif isinstance(node, list): if not node: return '[]', True return '[%s%s]' % (prefix, sep.join(_format(x, level)[0] for x in node)), False + if isinstance(node, bool) or node is None or node is Ellipsis: + return f'{t.keyword}{node!r}{t.reset}', True + if isinstance(node, (int, float, complex)): + return f'{t.number}{node!r}{t.reset}', True + if isinstance(node, (str, bytes)): + return f'{t.string}{node!r}{t.reset}', True return repr(node), True if not isinstance(node, AST): @@ -608,9 +630,11 @@ def __new__(cls, dims=(), **kwargs): def _dims_getter(self): """Deprecated. Use elts instead.""" + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) return self.elts def _dims_setter(self, value): + warnings._deprecated(f"ast.Tuple.dims", remove=(3, 21)) self.elts = value Tuple.dims = property(_dims_getter, _dims_setter) @@ -642,9 +666,9 @@ def main(args=None): import argparse import sys - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('infile', nargs='?', default='-', - help='the file to parse; defaults to stdin') + help='the file to parse; defaults to `stdin`') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), help='specify what kind of code must be parsed') @@ -657,11 +681,11 @@ def main(args=None): help='indentation of nodes (number of spaces)') parser.add_argument('--feature-version', type=str, default=None, metavar='VERSION', - help='Python version in the format 3.x ' - '(for example, 3.10)') + help='Python version in the format `3.x` ' + '(for example, `3.10`)') parser.add_argument('-O', '--optimize', type=int, default=-1, metavar='LEVEL', - help='optimization level for parser (default -1)') + help='optimization level for parser') parser.add_argument('--show-empty', default=False, action='store_true', help='show empty lists and fields in dump output') args = parser.parse_args(args) @@ -687,8 +711,28 @@ def main(args=None): tree = parse(source, name, args.mode, type_comments=args.no_type_comments, feature_version=feature_version, optimize=args.optimize) + from _colorize import can_colorize print(dump(tree, include_attributes=args.include_attributes, + color=can_colorize(file=sys.stdout), indent=args.indent, show_empty=args.show_empty)) +_deprecated = { + 'slice': globals().pop("slice"), + 'Index': globals().pop("Index"), + 'ExtSlice': globals().pop("ExtSlice"), + 'Suite': globals().pop("Suite"), + 'AugLoad': globals().pop("AugLoad"), + 'AugStore': globals().pop("AugStore"), + 'Param': globals().pop("Param") +} + +def __getattr__(attr): + try: + val = _deprecated[attr] + except KeyError: + raise AttributeError(f"module 'ast' has no attribute {attr!r}") from None + warnings._deprecated(f"ast.{attr}", remove=(3, 21)) + return val + if __name__ == '__main__': main() diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 32a5dbae03af216..2432e2dad74c23d 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -47,28 +47,3 @@ else: from .unix_events import * # pragma: no cover __all__ += unix_events.__all__ - -def __getattr__(name: str): - import warnings - - match name: - case "AbstractEventLoopPolicy": - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - return events._AbstractEventLoopPolicy - case "DefaultEventLoopPolicy": - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - if sys.platform == 'win32': - return windows_events._DefaultEventLoopPolicy - return unix_events._DefaultEventLoopPolicy - case "WindowsSelectorEventLoopPolicy": - if sys.platform == 'win32': - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - return windows_events._WindowsSelectorEventLoopPolicy - # Else fall through to the AttributeError below. - case "WindowsProactorEventLoopPolicy": - if sys.platform == 'win32': - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - return windows_events._WindowsProactorEventLoopPolicy - # Else fall through to the AttributeError below. - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 0bf3bdded40200a..cbb052630d71ce9 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -99,13 +99,17 @@ def run(self): console.write(banner) - if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")): + if not sys.flags.ignore_environment and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") if CAN_USE_PYREPL: @@ -155,24 +159,35 @@ def interrupt(self) -> None: parser = argparse.ArgumentParser( prog="python3 -m asyncio", description="Interactive asyncio shell and CLI tools", - color=True, ) subparsers = parser.add_subparsers(help="sub-commands", dest="command") ps = subparsers.add_parser( "ps", help="Display a table of all pending tasks in a process" ) ps.add_argument("pid", type=int, help="Process ID to inspect") + ps.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) pstree = subparsers.add_parser( "pstree", help="Display a tree of all pending tasks in a process" ) pstree.add_argument("pid", type=int, help="Process ID to inspect") + pstree.add_argument( + "--retries", + type=int, + default=3, + help="Number of retries on transient attach errors", + ) args = parser.parse_args() match args.command: case "ps": - asyncio.tools.display_awaited_by_tasks_table(args.pid) + asyncio.tools.display_awaited_by_tasks_table(args.pid, retries=args.retries) sys.exit(0) case "pstree": - asyncio.tools.display_awaited_by_tasks_tree(args.pid) + asyncio.tools.display_awaited_by_tasks_tree(args.pid, retries=args.retries) sys.exit(0) case None: pass # continue to the interactive shell @@ -185,7 +200,7 @@ def interrupt(self) -> None: sys.audit("cpython.run_stdin") - if os.getenv('PYTHON_BASIC_REPL'): + if not sys.flags.ignore_environment and os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: try: @@ -197,11 +212,14 @@ def interrupt(self) -> None: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - repl_locals = {'asyncio': asyncio} - for key in {'__name__', '__package__', - '__loader__', '__spec__', - '__builtins__', '__file__'}: - repl_locals[key] = locals()[key] + repl_locals = { + 'asyncio': asyncio, + '__name__': __name__, + '__package__': None, + '__loader__': __loader__, + '__spec__': None, + '__builtins__': __builtins__, + } console = AsyncIOInteractiveConsole(repl_locals, loop) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 7a6837546d930f3..f26fba175b63cd7 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -31,6 +31,7 @@ import traceback import warnings import weakref +import inspect try: import ssl @@ -487,10 +488,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro, **kwargs)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object, and **kwargs will be - arbitrary keyword arguments that should be passed on to Task. - The callable must return a Task. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the + active event loop, 'coro' will be a coroutine object, and **kwargs + will be arbitrary keyword arguments that should be passed on to + Task. The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -589,7 +590,7 @@ async def shutdown_asyncgens(self): return_exceptions=True) for result, agen in zip(results, closing_agens): - if isinstance(result, Exception): + if isinstance(result, BaseException): self.call_exception_handler({ 'message': f'an error occurred during closing of ' f'asynchronous generator {agen!r}', @@ -726,8 +727,8 @@ def run_until_complete(self, future): def stop(self): """Stop running the event loop. - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. + Every callback already scheduled will still run. This simply + informs run_forever to stop looping after a complete iteration. """ self._stopping = True @@ -840,7 +841,7 @@ def call_soon(self, callback, *args, context=None): def _check_callback(self, callback, method): if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError( f"coroutines cannot be used with {method}()") if not callable(callback): @@ -968,7 +969,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1075,12 +1076,12 @@ async def create_connection( Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. + family if specified), socket type SOCK_STREAM. protocol_factory must + be a callable returning a protocol instance. - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. + This method is a coroutine which will try to establish the + connection in the background. When successful, the coroutine + returns a (transport, protocol) pair. """ if server_hostname is not None and not ssl: raise ValueError('server_hostname is only meaningful with ssl') @@ -1218,21 +1219,26 @@ async def _create_connection_transport( ssl_handshake_timeout=None, ssl_shutdown_timeout=None, context=None): - sock.setblocking(False) - context = context if context is not None else contextvars.copy_context() - - protocol = protocol_factory() - waiter = self.create_future() - if ssl: - sslcontext = None if isinstance(ssl, bool) else ssl - transport = self._make_ssl_transport( - sock, protocol, sslcontext, waiter, - server_side=server_side, server_hostname=server_hostname, - ssl_handshake_timeout=ssl_handshake_timeout, - ssl_shutdown_timeout=ssl_shutdown_timeout, - context=context) - else: - transport = self._make_socket_transport(sock, protocol, waiter, context=context) + try: + sock.setblocking(False) + context = context if context is not None else contextvars.copy_context() + + protocol = protocol_factory() + waiter = self.create_future() + if ssl: + sslcontext = None if isinstance(ssl, bool) else ssl + transport = self._make_ssl_transport( + sock, protocol, sslcontext, waiter, + server_side=server_side, server_hostname=server_hostname, + ssl_handshake_timeout=ssl_handshake_timeout, + ssl_shutdown_timeout=ssl_shutdown_timeout, + context=context) + else: + transport = self._make_socket_transport(sock, protocol, waiter, context=context) + except: + # gh-153133: close the socket if the transport is never created. + sock.close() + raise try: await waiter @@ -1282,10 +1288,9 @@ async def sendfile(self, transport, file, offset=0, count=None, raise if not fallback: - raise RuntimeError( + raise exceptions.SendfileNotAvailableError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1294,9 +1299,12 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) - blocksize = min(count, 16384) if count else 16384 + blocksize = ( + min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) + if count else constants.SENDFILE_FALLBACK_READBUFFER_SIZE + ) buf = bytearray(blocksize) total_sent = 0 proto = _SendfileFallbackProtocol(transp) @@ -1550,11 +1558,11 @@ async def create_server( The host parameter can be a string, in that case the TCP server is bound to host and port. - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. If a host - appears multiple times (possibly indirectly e.g. when hostnames - resolve to the same IP address), the server is only bound once to that - host. + The host parameter can also be a sequence of strings and in that + case the TCP server is bound to all hosts of the sequence. If + a host appears multiple times (possibly indirectly e.g. when + hostnames resolve to the same IP address), the server is only bound + once to that host. Return a Server object which can be used to stop the service. diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 224b1883808a412..0423bc100b2d932 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -22,11 +22,10 @@ def __init__(self, loop, protocol, args, shell, self._proc = None self._pid = None self._returncode = None - self._exit_waiters = [] + self._exit_waiters = set() self._pending_calls = collections.deque() self._pipes = {} self._finished = False - self._pipes_connected = False if stdin == subprocess.PIPE: self._pipes[0] = None @@ -166,6 +165,9 @@ def kill(self): else: def send_signal(self, signal): self._check_proc() + if self._returncode is not None: + # The process already exited + return try: os.kill(self._proc.pid, signal) except ProcessLookupError: @@ -209,12 +211,19 @@ async def _connect_pipes(self, waiter): except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: + # Close any pipes that were already connected before the + # error/cancellation to avoid leaking file descriptors. + for proto in self._pipes.values(): + if proto is not None: + proto.pipe.close() + for raw_pipe in (proc.stdin, proc.stdout, proc.stderr): + if raw_pipe is not None: + raw_pipe.close() if waiter is not None and not waiter.cancelled(): waiter.set_exception(exc) else: if waiter is not None and not waiter.cancelled(): waiter.set_result(None) - self._pipes_connected = True def _call(self, cb, *data): if self._pending_calls is not None: @@ -235,6 +244,7 @@ def _process_exited(self, returncode): if self._loop.get_debug(): logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode + if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen # object. On Python 3.6, it is required to avoid a ResourceWarning. @@ -243,6 +253,13 @@ def _process_exited(self, returncode): self._try_finish() + # gh-119710: Wake up futures waiting for wait() as soon as the process + # exits. + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(returncode) + self._exit_waiters = None + async def _wait(self): """Wait until the process exit and return the process return code. @@ -251,22 +268,18 @@ async def _wait(self): return self._returncode waiter = self._loop.create_future() - self._exit_waiters.append(waiter) - return await waiter + self._exit_waiters.add(waiter) + try: + return await waiter + finally: + if self._exit_waiters is not None: + self._exit_waiters.discard(waiter) def _try_finish(self): assert not self._finished if self._returncode is None: return - if not self._pipes_connected: - # self._pipes_connected can be False if not all pipes were connected - # because either the process failed to start or the self._connect_pipes task - # got cancelled. In this broken state we consider all pipes disconnected and - # to avoid hanging forever in self._wait as otherwise _exit_waiters - # would never be woken up, we wake them up here. - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(self._returncode) + if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -276,11 +289,6 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.done(): - waiter.set_result(self._returncode) - self._exit_waiters = None self._loop = None self._proc = None self._protocol = None diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9b..3a4246f6ccd4c24 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -1,7 +1,6 @@ -__all__ = 'iscoroutinefunction', 'iscoroutine' +__all__ = ('iscoroutine',) import collections.abc -import inspect import os import sys import types @@ -13,25 +12,6 @@ def _is_debug_mode(): bool(os.environ.get('PYTHONASYNCIODEBUG'))) -# A marker for iscoroutinefunction. -_is_coroutine = object() - - -def iscoroutinefunction(func): - import warnings - """Return True if func is a decorated coroutine function.""" - warnings._deprecated("asyncio.iscoroutinefunction", - f"{warnings._DEPRECATED_MSG}; " - "use inspect.iscoroutinefunction() instead", - remove=(3,16)) - return _iscoroutinefunction(func) - - -def _iscoroutinefunction(func): - return (inspect.iscoroutinefunction(func) or - getattr(func, '_is_coroutine', None) is _is_coroutine) - - # Prioritize native coroutine check to speed-up # asyncio.iscoroutine. _COROUTINE_TYPES = (types.CoroutineType, collections.abc.Coroutine) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a7fb55982abe9ca..6b2d34e733a6b1a 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -9,8 +9,6 @@ "AbstractServer", "Handle", "TimerHandle", - "get_event_loop_policy", - "set_event_loop_policy", "get_event_loop", "set_event_loop", "new_event_loop", @@ -26,7 +24,6 @@ import subprocess import sys import threading -import warnings from . import format_helpers @@ -374,8 +371,8 @@ async def create_server( If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be - a sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also + be a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -415,8 +412,9 @@ async def create_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -453,7 +451,7 @@ async def create_unix_server( sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - start_serving=True): + start_serving=True, mode=None): """A coroutine which creates a UNIX Domain Socket server. The return value is a Server object, which can be used to stop @@ -479,8 +477,13 @@ async def create_unix_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. + + mode, if not None, is applied to the socket file created for + path with os.chmod() after binding and before the server + starts accepting connections. """ raise NotImplementedError @@ -511,8 +514,8 @@ async def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on - host (or family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending + on host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -552,7 +555,8 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. - protocol_factory should instantiate object with BaseProtocol interface. + protocol_factory should instantiate object with BaseProtocol + interface. Pipe is file-like object already switched to nonblocking. Return pair (transport, protocol), where transport support WriteTransport interface.""" @@ -662,84 +666,11 @@ def set_debug(self, enabled): raise NotImplementedError -class _AbstractEventLoopPolicy: - """Abstract policy for accessing the event loop.""" - - def get_event_loop(self): - """Get the event loop for the current context. - - Returns an event loop object implementing the AbstractEventLoop interface, - or raises an exception in case no event loop has been set for the - current context and the current policy does not specify to create one. - - It should never return None.""" - raise NotImplementedError - - def set_event_loop(self, loop): - """Set the event loop for the current context to loop.""" - raise NotImplementedError - - def new_event_loop(self): - """Create and return a new event loop object according to this - policy's rules. If there's need to set this loop as the event loop for - the current context, set_event_loop must be called explicitly.""" - raise NotImplementedError - -class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy): - """Default policy implementation for accessing the event loop. - - In this policy, each thread has its own event loop. However, we - only automatically create an event loop by default for the main - thread; other threads by default have no event loop. - - Other policies may have different rules (e.g. a single global - event loop, or automatically creating an event loop per thread, or - using some other notion of context to which an event loop is - associated). - """ +class _Local(threading.local): + _loop = None - _loop_factory = None - class _Local(threading.local): - _loop = None - - def __init__(self): - self._local = self._Local() - - def get_event_loop(self): - """Get the event loop for the current context. - - Returns an instance of EventLoop or raises an exception. - """ - if self._local._loop is None: - raise RuntimeError('There is no current event loop in thread %r.' - % threading.current_thread().name) - - return self._local._loop - - def set_event_loop(self, loop): - """Set the event loop.""" - if loop is not None and not isinstance(loop, AbstractEventLoop): - raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") - self._local._loop = loop - - def new_event_loop(self): - """Create a new event loop. - - You must call set_event_loop() to make this the current event - loop. - """ - return self._loop_factory() - - -# Event loop policy. The policy itself is always global, even if the -# policy's rules say that there is an event loop per thread (or other -# notion of context). The default policy is installed by the first -# call to get_event_loop_policy(). -_event_loop_policy = None - -# Lock for protecting the on-the-fly creation of the event loop policy. -_lock = threading.Lock() +_local = _Local() # A TLS for the running event loop, used by _get_running_loop. @@ -784,39 +715,19 @@ def _set_running_loop(loop): _running_loop.loop_pid = (loop, os.getpid()) -def _init_event_loop_policy(): - global _event_loop_policy - with _lock: - if _event_loop_policy is None: # pragma: no branch - if sys.platform == 'win32': - from .windows_events import _DefaultEventLoopPolicy - else: - from .unix_events import _DefaultEventLoopPolicy - _event_loop_policy = _DefaultEventLoopPolicy() - +def _get_event_loop(): + """Return the event loop set for the current thread. -def _get_event_loop_policy(): - """Get the current event loop policy.""" - if _event_loop_policy is None: - _init_event_loop_policy() - return _event_loop_policy - -def get_event_loop_policy(): - warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16)) - return _get_event_loop_policy() - -def _set_event_loop_policy(policy): - """Set the current event loop policy. - - If policy is None, the default policy is restored.""" - global _event_loop_policy - if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy): - raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'") - _event_loop_policy = policy + Raise a RuntimeError if no event loop has been set for the current + thread. This is the slow path of get_event_loop(); the running loop + is checked by the caller. + """ + loop = _local._loop + if loop is None: + raise RuntimeError('There is no current event loop in thread %r.' + % threading.current_thread().name) + return loop -def set_event_loop_policy(policy): - warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16)) - _set_event_loop_policy(policy) def get_event_loop(): """Return an asyncio event loop. @@ -825,23 +736,33 @@ def get_event_loop(): or similar API), this function will always return the running event loop. If there is no running event loop set, the function will return - the result of `get_event_loop_policy().get_event_loop()` call. + the loop set by ``set_event_loop()``, or raise a RuntimeError if + no loop has been set. """ # NOTE: this function is implemented in C (see _asynciomodule.c) current_loop = _get_running_loop() if current_loop is not None: return current_loop - return _get_event_loop_policy().get_event_loop() + return _get_event_loop() def set_event_loop(loop): - """Equivalent to calling get_event_loop_policy().set_event_loop(loop).""" - _get_event_loop_policy().set_event_loop(loop) + """Set the event loop for the current thread to loop. + + If loop is None, the current event loop is unset. + """ + if loop is not None and not isinstance(loop, AbstractEventLoop): + raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'") + _local._loop = loop def new_event_loop(): - """Equivalent to calling get_event_loop_policy().new_event_loop().""" - return _get_event_loop_policy().new_event_loop() + """Create and return a new event loop object.""" + if sys.platform == 'win32': + from .windows_events import EventLoop + else: + from .unix_events import EventLoop + return EventLoop() # Alias pure-Python implementations for testing purposes. @@ -870,8 +791,8 @@ def new_event_loop(): if hasattr(os, 'fork'): def on_fork(): # Reset the loop and wakeupfd in the forked child process. - if _event_loop_policy is not None: - _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local() + global _local + _local = _Local() _set_running_loop(None) signal.set_wakeup_fd(-1) diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index b5bfeb1630a1590..94fcd33d7088a68 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -58,11 +58,12 @@ def _build_graph_for_future( while coro is not None: if hasattr(coro, 'cr_await'): # A native coroutine or duck-type compatible iterator - st.append(FrameCallGraphEntry(coro.cr_frame)) + if coro.cr_frame is not None: + st.append(FrameCallGraphEntry(coro.cr_frame)) coro = coro.cr_await elif hasattr(coro, 'ag_await'): # A native async generator or duck-type compatible iterator - st.append(FrameCallGraphEntry(coro.cr_frame)) + st.append(FrameCallGraphEntry(coro.ag_frame)) coro = coro.ag_await else: break @@ -112,13 +113,13 @@ def capture_call_graph( optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. - If the optional keyword-only 'limit' argument is provided, each call stack - in the resulting graph is truncated to include at most ``abs(limit)`` - entries. If 'limit' is positive, the entries left are the closest to - the invocation point. If 'limit' is negative, the topmost entries are - left. If 'limit' is omitted or None, all entries are present. - If 'limit' is 0, the call stack is not captured at all, only - "awaited by" information is present. + If the optional keyword-only 'limit' argument is provided, each call + stack in the resulting graph is truncated to include at most + ``abs(limit)`` entries. If 'limit' is positive, the entries left are + the closest to the invocation point. If 'limit' is negative, the + topmost entries are left. If 'limit' is omitted or None, all entries + are present. If 'limit' is 0, the call stack is not captured at all, + only "awaited by" information is present. """ loop = events._get_running_loop() @@ -273,4 +274,5 @@ def print_call_graph( limit: int | None = None, ) -> None: """Print the async call graph for the current task or the provided Future.""" - print(format_call_graph(future, depth=depth, limit=limit), file=file) + # gh-156327: print_call_graph() must not report its own frame + print(format_call_graph(future, depth=depth + 1, limit=limit), file=file) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index fa3a94764b507ae..c59ea15111bef2b 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -145,10 +145,7 @@ def _wake_up_first(self): """Ensure that the first waiter will wake up.""" if not self._waiters: return - try: - fut = next(iter(self._waiters)) - except StopIteration: - return + fut = next(iter(self._waiters)) # .done() means that the waiter is already set to wake up. if not fut.done(): @@ -158,10 +155,10 @@ def _wake_up_first(self): class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. + Class implementing event objects. An event manages a flag that can be + set to true with the set() method and reset to false with the clear() + method. The wait() method blocks until the flag is true. The flag is + initially false. """ def __init__(self): @@ -353,9 +350,9 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it + blocks, waiting until some other thread calls release(). Semaphores also support the context management protocol. @@ -511,8 +508,8 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. - When the specified number of tasks have started waiting, they are all - simultaneously awoken. + When the specified number of tasks have started waiting, they are + all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. """ async with self._cond: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 2dc1569d7807911..f18a7fe58558155 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -62,6 +62,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._closing = False # Set when close() called. self._called_connection_lost = False self._eof_written = False + self._empty_waiter = None if self._server is not None: self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) @@ -331,10 +332,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, _start_tls_compatible = True - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self._empty_waiter = None - def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError( @@ -465,7 +462,6 @@ class _ProactorDatagramTransport(_ProactorBasePipeTransport, def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): self._address = address - self._empty_waiter = None self._buffer_size = 0 # We don't need to call _protocol.connection_made() since our base # constructor does it for us. @@ -756,8 +752,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): offset += blocksize total_sent += blocksize finally: - if total_sent > 0: - file.seek(offset) + file.seek(offset) async def _sendfile_native(self, transp, file, offset, count): resume_reading = transp.is_reading() diff --git a/Lib/asyncio/protocols.py b/Lib/asyncio/protocols.py index 09987b164c66e57..c67a26c9e2986cc 100644 --- a/Lib/asyncio/protocols.py +++ b/Lib/asyncio/protocols.py @@ -199,6 +199,7 @@ def process_exited(self): def _feed_data_to_buffered_proto(proto, data): data_len = len(data) + start = 0 while data_len: buf = proto.get_buffer(data_len) buf_len = len(buf) @@ -206,11 +207,11 @@ def _feed_data_to_buffered_proto(proto, data): raise RuntimeError('get_buffer() returned an empty buffer') if buf_len >= data_len: - buf[:data_len] = data + buf[:data_len] = data[start:] if start else data proto.buffer_updated(data_len) return else: - buf[:buf_len] = data[:buf_len] + buf[:buf_len] = data[start:start + buf_len] proto.buffer_updated(buf_len) - data = data[buf_len:] - data_len = len(data) + start += buf_len + data_len -= buf_len diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 756216fac809329..8871cdc16c68b33 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -33,9 +33,9 @@ class QueueShutDown(Exception): class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "await put()" will block when the - queue reaches maxsize, until an item is removed by get(). + If maxsize is less than or equal to zero, the queue size is infinite. + If it is an integer greater than 0, then "await put()" will block when + the queue reaches maxsize, until an item is removed by get(). Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be @@ -174,11 +174,11 @@ async def get(self): If queue is empty, wait until an item is available. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ while self.empty(): - if self._is_shutdown and self.empty(): + if self._is_shutdown: raise QueueShutDown getter = self._get_loop().create_future() self._getters.append(getter) @@ -203,10 +203,11 @@ async def get(self): def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise + QueueEmpty. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ if self.empty(): if self._is_shutdown: @@ -223,12 +224,12 @@ def task_done(self): a subsequent call to task_done() tells the queue that the processing on the task is complete. - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). - Raises ValueError if called more times than there were items placed in - the queue. + Raises ValueError if called more times than there were items placed + in the queue. """ if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -239,10 +240,11 @@ def task_done(self): async def join(self): """Block until all items in the queue have been gotten and processed. - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. + The count of unfinished tasks goes up whenever an item is added to + the queue. The count goes down whenever a consumer calls + task_done() to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + join() unblocks. """ if self._unfinished_tasks > 0: await self._finished.wait() diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index ba37e003a655c0a..774a317564df22c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -35,7 +35,8 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - The run() method can be called multiple times within the runner's context. + The run() method can be called multiple times within the runner's + context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 961dbfb4b96303e..16d5c1b6f0a3e19 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -53,7 +53,7 @@ def _test_selector_event(selector, fd, event): class BaseSelectorEventLoop(base_events.BaseEventLoop): """Selector event loop. - See events.EventLoop for API specification. + See events.AbstractEventLoop for API specification. """ def __init__(self, selector=None): @@ -206,6 +206,7 @@ def _accept_connection( protocol_factory, sock, sslcontext, server, backlog, ssl_handshake_timeout, ssl_shutdown_timeout, context) + return else: raise # The event loop will catch, log and ignore it. else: @@ -252,7 +253,9 @@ async def _accept_connection2( except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - if self._debug: + if transport is None: + conn.close() + if transport is None or self._debug: context = { 'message': 'Error on transport creation for incoming connection', @@ -533,11 +536,12 @@ def _sock_recvfrom_into(self, fut, sock, buf, bufsize): async def sock_sendall(self, sock, data): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -584,13 +588,10 @@ def _sock_sendall(self, fut, sock, view, pos): pos[0] = start async def sock_sendto(self, sock, data, address): - """Send data to the socket. + """Send a datagram from sock to address. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket does not have to be connected. This method sends the + whole datagram in a single call. Return the number of bytes sent. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -701,10 +702,11 @@ def _sock_connect_cb(self, fut, sock, address): async def sock_accept(self, sock): """Accept a connection. - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. + The socket must be bound to an address and listening for + connections. The return value is a pair (conn, address) where + conn is a new socket object usable to send and receive data on the + connection, and address is the address bound to the socket on the + other end of the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -714,6 +716,9 @@ async def sock_accept(self, sock): return await fut def _sock_accept(self, fut, sock): + # gh-153761: _sock_accept must not scheduled with already cancelled future + if fut.done(): + return fd = sock.fileno() try: conn, address = sock.accept() @@ -1195,8 +1200,13 @@ def writelines(self, list_of_data): return for data in list_of_data: + # gh-155888: an empty chunk can never be drained, so never buffer it + if not data: + continue self._buffer.append(memoryview(data)) self._buffer_size += len(data) + if not self._buffer: + return self._write_ready() # If the entire buffer couldn't be written, register a write handler if self._buffer: @@ -1233,7 +1243,6 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): - _buffer_factory = collections.deque _header_size = 8 def __init__(self, loop, sock, protocol, address=None, diff --git a/Lib/asyncio/sslproto.py b/Lib/asyncio/sslproto.py index 74c5f0d5ca0609c..84e82fc69cc5fdc 100644 --- a/Lib/asyncio/sslproto.py +++ b/Lib/asyncio/sslproto.py @@ -588,6 +588,12 @@ def _on_handshake_complete(self, handshake_exc): msg = 'SSL handshake failed on verifying the certificate' else: msg = 'SSL handshake failed' + # gh-98078: When the handshake fails, OpenSSL leaves the fatal + # TLS alert (for example "bad certificate" or "protocol + # version") in the outgoing BIO. Send it to the peer before + # closing the transport so that it knows why the handshake + # failed. + self._process_outgoing() self._fatal_error(exc, msg) self._wakeup_waiter(exc) return @@ -652,6 +658,10 @@ def _do_shutdown(self): except SSLAgainErrors: self._process_outgoing() except ssl.SSLError as exc: + # gh-98078: send what OpenSSL left in the outgoing BIO, e.g. + # the close_notify alert, to the peer before closing (see + # _on_handshake_complete()). + self._process_outgoing() self._on_shutdown_complete(exc) else: self._process_outgoing() @@ -743,6 +753,11 @@ def _do_read(self): else: self._process_outgoing() self._control_ssl_reading() + except ssl.SSLError as ex: + # gh-98078: send the fatal TLS alert left in the outgoing + # BIO to the peer (see _on_handshake_complete()). + self._process_outgoing() + self._fatal_error(ex, 'Fatal error on SSL protocol') except Exception as ex: self._fatal_error(ex, 'Fatal error on SSL protocol') diff --git a/Lib/asyncio/staggered.py b/Lib/asyncio/staggered.py index 845aed4c6a3b352..b9e7b9eb29594fb 100644 --- a/Lib/asyncio/staggered.py +++ b/Lib/asyncio/staggered.py @@ -90,11 +90,7 @@ def task_done(task): return unhandled_exceptions.append(exc) - async def run_one_coro(ok_to_start, previous_failed) -> None: - # in eager tasks this waits for the calling task to append this task - # to running_tasks, in regular tasks this wait is a no-op that does - # not yield a future. See gh-124309. - await ok_to_start.wait() + async def run_one_coro(previous_failed) -> None: # Wait for the previous task to finish, or for delay seconds if previous_failed is not None: with contextlib.suppress(exceptions_mod.TimeoutError): @@ -110,14 +106,13 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: return # Start task that will run the next coroutine this_failed = locks.Event() - next_ok_to_start = locks.Event() - next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed)) + next_task = loop.create_task( + run_one_coro(this_failed), + eager_start=False, + ) futures.future_add_to_awaited_by(next_task, parent_task) running_tasks.add(next_task) next_task.add_done_callback(task_done) - # next_task has been appended to running_tasks so next_task is ok to - # start. - next_ok_to_start.set() # Prepare place to put this coroutine's exceptions if not won exceptions.append(None) assert len(exceptions) == this_index + 1 @@ -149,13 +144,11 @@ async def run_one_coro(ok_to_start, previous_failed) -> None: propagate_cancellation_error = None try: - ok_to_start = locks.Event() - first_task = loop.create_task(run_one_coro(ok_to_start, None)) + first_task = loop.create_task(run_one_coro(None), eager_start=False) futures.future_add_to_awaited_by(first_task, parent_task) running_tasks.add(first_task) first_task.add_done_callback(task_done) - # first_task has been appended to running_tasks so first_task is ok to start. - ok_to_start.set() + # first_task has been appended to running_tasks before the event loop starts running it. propagate_cancellation_error = None # Make sure no tasks are left running if we leave this function while running_tasks: diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index d2db1a930c2ad2b..954e132617ce42b 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -216,6 +216,9 @@ def _stream_reader(self): def _replace_transport(self, transport): self._transport = transport self._over_ssl = transport.get_extra_info('sslcontext') is not None + reader = self._stream_reader + if reader is not None: + reader._replace_transport(transport) def connection_made(self, transport): if self._reject_connection: @@ -236,7 +239,17 @@ def connection_made(self, transport): self._over_ssl = transport.get_extra_info('sslcontext') is not None if self._client_connected_cb is not None: writer = StreamWriter(transport, self, reader, self._loop) - res = self._client_connected_cb(reader, writer) + try: + res = self._client_connected_cb(reader, writer) + except Exception as exc: + self._loop.call_exception_handler({ + 'message': 'Unhandled exception in client_connected_cb', + 'exception': exc, + 'transport': transport, + }) + transport.close() + self._strong_reader = None + return if coroutines.iscoroutine(res): def callback(task): if task.cancelled(): @@ -321,8 +334,6 @@ def __init__(self, transport, protocol, reader, loop): assert reader is None or isinstance(reader, StreamReader) self._reader = reader self._loop = loop - self._complete_fut = self._loop.create_future() - self._complete_fut.set_result(None) def __repr__(self): info = [self.__class__.__name__, f'transport={self._transport!r}'] @@ -475,6 +486,10 @@ def set_transport(self, transport): assert self._transport is None, 'Transport already set' self._transport = transport + def _replace_transport(self, transport): + assert self._transport is not None, 'Transport not set' + self._transport = transport + def _maybe_resume_transport(self): if self._paused and len(self._buffer) <= self._limit: self._paused = False @@ -539,17 +554,17 @@ async def _wait_for_data(self, func_name): self._waiter = None async def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. + r"""Read chunk of data from the stream until newline (b'\n') is found. - On success, return chunk that ends with newline. If only partial + On success, return chunk that ends with newline. If only partial line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. + terminating newline. When EOF was reached while no bytes read, + empty bytes object is returned. - If limit is reached, ValueError will be raised. In that case, if + If limit is reached, ValueError will be raised. In that case, if newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. + from internal buffer. Else, internal buffer will be cleared. + Limit is compared against part of the line without newline. If stream was paused, this function will automatically resume it if needed. diff --git a/Lib/asyncio/subprocess.py b/Lib/asyncio/subprocess.py index 043359bbd03f8aa..26d7b755105c9d5 100644 --- a/Lib/asyncio/subprocess.py +++ b/Lib/asyncio/subprocess.py @@ -124,6 +124,11 @@ def __init__(self, transport, protocol, loop): self.stdout = protocol.stdout self.stderr = protocol.stderr self.pid = transport.get_pid() + self._communication_started = False + self._input = None + self._input_written = False + self._stdout_buf = bytearray() + self._stderr_buf = bytearray() def __repr__(self): return f'<{self.__class__.__name__} {self.pid}>' @@ -148,8 +153,9 @@ def kill(self): async def _feed_stdin(self, input): debug = self._loop.get_debug() try: - if input is not None: + if input is not None and not self._input_written: self.stdin.write(input) + self._input_written = True if debug: logger.debug( '%r communicate: feed stdin (%s bytes)', self, len(input)) @@ -172,22 +178,33 @@ async def _read_stream(self, fd): transport = self._transport.get_pipe_transport(fd) if fd == 2: stream = self.stderr + buf = self._stderr_buf else: assert fd == 1 stream = self.stdout + buf = self._stdout_buf if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: read %s', self, name) - output = await stream.read() + # Append each block to the persistent buffer as soon as it is + # read so that no output is lost if this coroutine is cancelled. + while block := await stream.read(stream._limit): + buf += block if self._loop.get_debug(): name = 'stdout' if fd == 1 else 'stderr' logger.debug('%r communicate: close %s', self, name) transport.close() - return output async def communicate(self, input=None): + if self._communication_started: + if input: + raise ValueError( + "Cannot send input after starting communication") + else: + self._input = input + self._communication_started = True if self.stdin is not None: - stdin = self._feed_stdin(input) + stdin = self._feed_stdin(self._input) else: stdin = self._noop() if self.stdout is not None: @@ -198,8 +215,16 @@ async def communicate(self, input=None): stderr = self._read_stream(2) else: stderr = self._noop() - stdin, stdout, stderr = await tasks.gather(stdin, stdout, stderr) + await tasks.gather(stdin, stdout, stderr) await self.wait() + if self.stdout is not None: + stdout = self._stdout_buf.take_bytes() + else: + stdout = None + if self.stderr is not None: + stderr = self._stderr_buf.take_bytes() + else: + stderr = None return (stdout, stderr) diff --git a/Lib/asyncio/taskgroups.py b/Lib/asyncio/taskgroups.py index 00e8f6d5d1a68b2..2debb4d5849e682 100644 --- a/Lib/asyncio/taskgroups.py +++ b/Lib/asyncio/taskgroups.py @@ -37,6 +37,7 @@ def __init__(self): self._errors = [] self._base_error = None self._on_completed_fut = None + self._cancel_on_enter = False def __repr__(self): info = [''] @@ -63,6 +64,8 @@ async def __aenter__(self): raise RuntimeError( f'TaskGroup {self!r} cannot determine the parent task') self._entered = True + if self._cancel_on_enter: + self.cancel() return self @@ -113,6 +116,7 @@ async def _aexit(self, et, exc): # can be cancelled multiple times if our parent task # is being cancelled repeatedly (or even once, when # our own cancellation is already in progress) + pending_cancellation_error = None while self._tasks: if self._on_completed_fut is None: self._on_completed_fut = self._loop.create_future() @@ -120,6 +124,7 @@ async def _aexit(self, et, exc): try: await self._on_completed_fut except exceptions.CancelledError as ex: + pending_cancellation_error = ex if not self._aborting: # Our parent task is being cancelled: # @@ -137,6 +142,18 @@ async def _aexit(self, et, exc): assert not self._tasks if self._base_error is not None: + # self._base_error (SystemExit or KeyboardInterrupt) is about + # to propagate out of this method, which discards any other + # collected task errors silently. Report them instead of + # losing them. See gh-135736. + for suppressed_exc in self._errors: + self._loop.call_exception_handler({ + 'message': 'TaskGroup task exception was not ' + 'propagated because the TaskGroup body ' + 'is being closed with a BaseException', + 'exception': suppressed_exc, + 'task_group': self, + }) try: raise self._base_error finally: @@ -148,6 +165,9 @@ async def _aexit(self, et, exc): # If there are no pending cancellations left, # don't propagate CancelledError. propagate_cancellation_error = None + elif propagate_cancellation_error is None: + # gh-155433: the remaining cancellation is not ours, don't drop it + propagate_cancellation_error = pending_cancellation_error # Propagate CancelledError if there is one, except if there # are other errors -- those have priority. @@ -171,13 +191,29 @@ async def _aexit(self, et, exc): self._parent_task.uncancel() self._parent_task.cancel() try: - raise BaseExceptionGroup( - 'unhandled errors in a TaskGroup', - self._errors, - ) from None + # If the *only* error is a GeneratorExit from the body + # of the group, then instead of raising an + # ExceptionGroup we raise GeneratorExit. This ensures + # that async generators that use TaskGroup properly + # swallow the exception on `aclose()` while ensuring + # that no exceptions from subtasks are swallowed. + if ( + et is not None + and issubclass(et, GeneratorExit) + and len(self._errors) == 1 + ): + raise exc + else: + raise BaseExceptionGroup( + 'unhandled errors in a TaskGroup', + self._errors, + ) from None finally: exc = None + # Suppress any remaining exception (exceptions deserving to be raised + # were raised above). + return True def create_task(self, coro, **kwargs): """Create a new task in this group and return it. @@ -203,6 +239,9 @@ def create_task(self, coro, **kwargs): # the current task too early. gh-128550, gh-128588 self._tasks.add(task) task.add_done_callback(self._on_task_done) + # gh-155418: an eager task can cancel the group before joining _tasks + if self._aborting and not task.done(): + task.cancel() try: return task finally: @@ -278,3 +317,30 @@ def _on_task_done(self, task): self._abort() self._parent_cancel_requested = True self._parent_task.cancel() + + def cancel(self): + """Cancel the task group + + `cancel()` will be called on any tasks in the group that aren't yet + done, as well as the parent (body) of the group. This will cause + the task group context manager to exit *without* + `asyncio.CancelledError` being raised. + + If `cancel()` is called before entering the task group, the group + will be cancelled upon entry. This is useful for patterns where + one piece of code passes an unused TaskGroup instance to another in + order to have the ability to cancel anything run within the group. + + `cancel()` is idempotent and may be called after the task group has + already exited. + """ + if not self._entered: + self._cancel_on_enter = True + return + if self._exiting and not self._tasks: + return + if not self._aborting: + self._abort() + if self._parent_task and not self._parent_cancel_requested: + self._parent_cancel_requested = True + self._parent_task.cancel() diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index fbd5c39a7c56ac4..f432cf0afa895a2 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -440,15 +440,13 @@ def _release_waiter(waiter, *args): async def wait_for(fut, timeout): """Wait for the single Future or coroutine to complete, with timeout. - Coroutine will be wrapped in Task. - Returns result of the Future or coroutine. When a timeout occurs, - it cancels the task and raises TimeoutError. To avoid the task - cancellation, wrap it in shield(). + it cancels fut and raises TimeoutError. To prevent fut from being + cancelled, wrap it in shield(). - If the wait is cancelled, the task is also cancelled. + If the wait is cancelled, fut is also cancelled. - If the task suppresses the cancellation and returns a value instead, + If fut suppresses the cancellation and returns a value instead, that value is returned. This function is a coroutine. @@ -524,6 +522,7 @@ def _on_completion(f): timeout_handle.cancel() for f in fs: f.remove_done_callback(_on_completion) + futures.future_discard_from_awaited_by(f, cur_task) done, pending = set(), set() for f in fs: @@ -563,9 +562,11 @@ def __init__(self, aws, timeout): self._timeout_handle = None loop = events.get_event_loop() + self._cur_task = current_task() todo = {ensure_future(aw, loop=loop) for aw in set(aws)} for f in todo: f.add_done_callback(self._handle_completion) + futures.future_add_to_awaited_by(f, self._cur_task) if todo and timeout is not None: self._timeout_handle = ( loop.call_later(timeout, self._handle_timeout) @@ -596,6 +597,7 @@ def __next__(self): def _handle_timeout(self): for f in self._todo: f.remove_done_callback(self._handle_completion) + futures.future_discard_from_awaited_by(f, self._cur_task) self._done.put_nowait(None) # Sentinel for _wait_for_one(). self._todo.clear() # Can't do todo.remove(f) in the loop. @@ -603,6 +605,7 @@ def _handle_completion(self, f): if not self._todo: return # _handle_timeout() was here first. self._todo.remove(f) + futures.future_discard_from_awaited_by(f, self._cur_task) self._done.put_nowait(f) if not self._todo and self._timeout_handle is not None: self._timeout_handle.cancel() @@ -624,10 +627,11 @@ def as_completed(fs, *, timeout=None): Run the supplied awaitables concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. - The object returned can be iterated as an asynchronous iterator or a plain - iterator. When asynchronous iteration is used, the originally-supplied - awaitables are yielded if they are tasks or futures. This makes it easy to - correlate previously-scheduled tasks with their results: + The object returned can be iterated as an asynchronous iterator or + a plain iterator. When asynchronous iteration is used, the + originally-supplied awaitables are yielded if they are tasks or + futures. This makes it easy to correlate previously-scheduled tasks + with their results: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) @@ -643,26 +647,27 @@ def as_completed(fs, *, timeout=None): else: print("IPv4 connection established.") - During asynchronous iteration, implicitly-created tasks will be yielded for - supplied awaitables that aren't tasks or futures. + During asynchronous iteration, implicitly-created tasks will be + yielded for supplied awaitables that aren't tasks or futures. - When used as a plain iterator, each iteration yields a new coroutine that - returns the result or raises the exception of the next completed awaitable. - This pattern is compatible with Python versions older than 3.13: + When used as a plain iterator, each iteration yields a new coroutine + that returns the result or raises the exception of the next completed + awaitable. This pattern is compatible with Python versions older than + 3.13: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) tasks = [ipv4_connect, ipv6_connect] for next_connect in as_completed(tasks): - # next_connect is not one of the original task objects. It must be - # awaited to obtain the result value or raise the exception of the - # awaitable that finishes next. + # next_connect is not one of the original task objects. It must + # be awaited to obtain the result value or raise the exception + # of the awaitable that finishes next. reader, writer = await next_connect - A TimeoutError is raised if the timeout occurs before all awaitables are - done. This is raised by the async for loop during asynchronous iteration or - by the coroutines yielded during plain iteration. + A TimeoutError is raised if the timeout occurs before all awaitables + are done. This is raised by the async for loop during asynchronous + iteration or by the coroutines yielded during plain iteration. """ if inspect.isawaitable(fs): raise TypeError( @@ -994,6 +999,9 @@ def _outer_done_callback(outer): # Keep only one callback to log on cancel inner.remove_done_callback(_log_on_exception) inner.add_done_callback(_log_on_exception) + if cur_task is not None: + inner.remove_done_callback(_clear_awaited_by_callback) + futures.future_discard_from_awaited_by(inner, cur_task) if cur_task is not None: inner.add_done_callback(_clear_awaited_by_callback) @@ -1030,21 +1038,22 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any + callable that returns a Task-compatible object and has a signature + compatible with `Task.__init__`; it must have the `eager_start` + keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in - this case there's no need to call `create_eager_task_factory()` - directly. Instead the global `eager_task_factory` instance can be - used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231de166..5001351b0976ecd 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -17,7 +17,8 @@ async def to_thread(func, /, *args, **kwargs): allowing context variables from the main thread to be accessed in the separate thread. - Return a coroutine that can be awaited to get the eventual result of *func*. + Return a coroutine that can be awaited to get the eventual result of + *func*. """ loop = events.get_running_loop() ctx = contextvars.copy_context() diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 09342dc7c1310b0..65ddc285abd971d 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -25,7 +25,8 @@ class _State(enum.Enum): class Timeout: """Asynchronous context manager for cancelling overdue coroutines. - Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + Use `timeout()` or `timeout_at()` rather than instantiating this class + directly. """ def __init__(self, when: float | None) -> None: diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index 62d6a71557fa37b..2ac1738d15c6c72 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -231,27 +231,38 @@ def exit_with_permission_help_text(): print( "Error: The specified process cannot be attached to due to insufficient permissions.\n" "See the Python documentation for details on required privileges and troubleshooting:\n" - "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + "https://docs.python.org/3/howto/remote_debugging.html#permission-requirements\n", + file=sys.stderr, ) sys.exit(1) -def _get_awaited_by_tasks(pid: int) -> list: - try: - return get_all_awaited_by(pid) - except RuntimeError as e: - while e.__context__ is not None: - e = e.__context__ - print(f"Error retrieving tasks: {e}") - sys.exit(1) - except PermissionError: - exit_with_permission_help_text() +_TRANSIENT_ERRORS = (RuntimeError, OSError, UnicodeDecodeError, MemoryError) + + +def _get_awaited_by_tasks(pid: int, retries: int = 3) -> list: + for attempt in range(retries + 1): + try: + return get_all_awaited_by(pid) + except PermissionError: + exit_with_permission_help_text() + except ProcessLookupError: + print(f"Error: process {pid} not found.", file=sys.stderr) + sys.exit(1) + except _TRANSIENT_ERRORS as e: + if attempt < retries: + continue + if isinstance(e, RuntimeError): + while e.__context__ is not None: + e = e.__context__ + print(f"Error retrieving tasks: {e}", file=sys.stderr) + sys.exit(1) -def display_awaited_by_tasks_table(pid: int) -> None: +def display_awaited_by_tasks_table(pid: int, retries: int = 3) -> None: """Build and print a table of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) table = build_task_table(tasks) # Print the table in a simple tabular format print( @@ -262,10 +273,10 @@ def display_awaited_by_tasks_table(pid: int) -> None: print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") -def display_awaited_by_tasks_tree(pid: int) -> None: +def display_awaited_by_tasks_tree(pid: int, retries: int = 3) -> None: """Build and print a tree of all pending tasks under `pid`.""" - tasks = _get_awaited_by_tasks(pid) + tasks = _get_awaited_by_tasks(pid, retries=retries) try: result = build_async_tree(tasks) except CycleFoundException as e: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 49e8067ee7b4e59..3a66cee93da4f50 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -12,6 +12,7 @@ import sys import threading import warnings +import inspect from . import base_events from . import base_subprocess @@ -54,7 +55,8 @@ def waitstatus_to_exitcode(status): class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + Adds signal handling and UNIX Domain Socket support to + SelectorEventLoop. """ def __init__(self, selector=None): @@ -94,7 +96,7 @@ def add_signal_handler(self, sig, callback, *args): Raise RuntimeError if there is a problem setting up the handler. """ if (coroutines.iscoroutine(callback) or - coroutines._iscoroutinefunction(callback)): + inspect.iscoroutinefunction(callback)): raise TypeError("coroutines cannot be used " "with add_signal_handler()") self._check_signal(sig) @@ -211,13 +213,13 @@ async def _make_subprocess_transport(self, protocol, args, shell, raise except BaseException: transp.close() - await transp._wait() + await tasks.shield(transp._wait()) raise return transp def _child_watcher_callback(self, pid, returncode, transp): - self.call_soon_threadsafe(transp._process_exited, returncode) + transp._process_exited(returncode) async def create_unix_connection( self, protocol_factory, path=None, *, @@ -274,7 +276,7 @@ async def create_unix_server( sock=None, backlog=100, ssl=None, ssl_handshake_timeout=None, ssl_shutdown_timeout=None, - start_serving=True, cleanup_socket=True): + start_serving=True, cleanup_socket=True, mode=None): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') @@ -292,6 +294,9 @@ async def create_unix_server( 'path and sock can not be specified at the same time') path = os.fspath(path) + if mode is not None and path and path[0] in (0, '\x00'): + raise ValueError( + 'mode is not supported for abstract sockets') sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # Check for abstract socket. `str` and `bytes` paths are supported. @@ -320,11 +325,26 @@ async def create_unix_server( except: sock.close() raise + + if mode is not None: + # The socket cannot accept connections until listen() is + # called, which happens later in Server._start_serving(), + # so no connection can be accepted while the socket still + # has the default permissions. + try: + os.chmod(path, mode) + except: + sock.close() + raise else: if sock is None: raise ValueError( 'path was not specified, and no sock specified') + if mode is not None: + raise ValueError( + 'mode is only meaningful with path') + if (sock.family != socket.AF_UNIX or sock.type != socket.SOCK_STREAM): raise ValueError( @@ -384,12 +404,12 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # order to simplify the common case. self.remove_writer(registered_fd) if fut.cancelled(): - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) return if count: blocksize = count - total_sent if blocksize <= 0: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) return @@ -423,20 +443,20 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # plain send(). err = exceptions.SendfileNotAvailableError( "os.sendfile call failed") - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(err) else: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) else: if sent == 0: # EOF - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) else: offset += sent @@ -447,9 +467,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fd, sock, fileno, offset, count, blocksize, total_sent) - def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): - if total_sent > 0: - os.lseek(fileno, offset, os.SEEK_SET) + def _sock_sendfile_update_filepos(self, fileno, offset): + # After this helper runs, the source fd's lseek pointer is at offset." + os.lseek(fileno, offset, os.SEEK_SET) def _sock_add_cancellation_callback(self, fut, sock): def cb(fut): @@ -638,7 +658,8 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): self._conn_lost = 0 self._closing = False # Set when close() or write_eof() called. - mode = os.fstat(self._fileno).st_mode + pipe_stat = os.fstat(self._fileno) + mode = pipe_stat.st_mode is_char = stat.S_ISCHR(mode) is_fifo = stat.S_ISFIFO(mode) is_socket = stat.S_ISSOCK(mode) @@ -655,7 +676,19 @@ def __init__(self, loop, pipe, protocol, waiter=None, extra=None): # On AIX, the reader trick (to be notified when the read end of the # socket is closed) only works for sockets. On other platforms it # works for pipes and sockets. (Exception: OS X 10.4? Issue #19294.) - if is_socket or (is_fifo and not sys.platform.startswith("aix")): + # On macOS and Solaris, the trick misfires for named FIFOs (but not for + # pipes created with os.pipe(), which have st_nlink == 0): the write + # end polls as readable whenever unread data sits in the FIFO, and no + # event is delivered when the read end is closed, so it can only + # ever report a false disconnection (gh-145030). The same XNU + # behaviour applies on iOS/tvOS/watchOS (sys.platform is not + # "darwin" there). + is_named_fifo_without_close_event = ( + sys.platform in {"darwin", "ios", "tvos", "watchos", "sunos5"} + and is_fifo and pipe_stat.st_nlink > 0) + if is_socket or (is_fifo + and not sys.platform.startswith("aix") + and not is_named_fifo_without_close_event): # only start reading when connection_made() has been called self._loop.call_soon(self._loop._add_reader, self._fileno, self._read_ready) @@ -834,7 +867,8 @@ class _UnixSubprocessTransport(base_subprocess.BaseSubprocessTransport): def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs): stdin_w = None - if stdin == subprocess.PIPE and sys.platform.startswith('aix'): + if (stdin == subprocess.PIPE + and (sys.platform.startswith('aix') or sys.platform == 'cygwin')): # Use a socket pair for stdin on AIX, since it does not # support selecting read events on the write end of a # socket (which we use in order to detect closing of the @@ -886,8 +920,8 @@ def _do_wait(self, pid, pidfd, callback, args): pid) else: returncode = waitstatus_to_exitcode(status) - - os.close(pidfd) + finally: + os.close(pidfd) callback(pid, returncode, *args) class _ThreadedChildWatcher: @@ -927,6 +961,49 @@ def add_child_handler(self, pid, callback, *args): def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 + if hasattr(os, 'waitid'): + # Wait for the child process using waitid() on platforms which support it. + # WNOWAIT is used to avoid reaping the child process, allowing the event loop to + # reap the child process with waitpid() later in event loop thread. + # This makes the reaping of the child and notification of the return code + # atomic with respect to the event loop thread. + try: + os.waitid(os.P_PID, expected_pid, os.WEXITED | os.WNOWAIT) + except ChildProcessError: + # The child process is already reaped + pass + if loop.is_closed(): + # loop is already closed, reap the zombie here so that it is not leaked. + pid, _ = self._reap(loop, expected_pid) + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + try: + loop.call_soon_threadsafe( + self._reap_and_notify, loop, expected_pid, + callback, args) + except RuntimeError: + # The event loop was closed concurrently. + pid, _ = self._reap(loop, expected_pid) + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + # Fallback for platforms that don't support waitid(): we have to + # reap the child here, which is racy with respect to send_signal() + pid, returncode = self._reap(loop, expected_pid) + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + self._threads.pop(expected_pid) + + def _reap_and_notify(self, loop, expected_pid, callback, args): + pid, returncode = self._reap(loop, expected_pid) + callback(pid, returncode, *args) + + def _reap(self, loop, expected_pid): try: pid, status = os.waitpid(expected_pid, 0) except ChildProcessError: @@ -942,13 +1019,7 @@ def _do_waitpid(self, loop, expected_pid, callback, args): if loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) - - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - self._threads.pop(expected_pid) + return pid, returncode def can_use_pidfd(): if not hasattr(os, 'pidfd_open'): @@ -962,11 +1033,5 @@ def can_use_pidfd(): return True -class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy): - """UNIX event loop policy""" - _loop_factory = _UnixSelectorEventLoop - - SelectorEventLoop = _UnixSelectorEventLoop -_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy EventLoop = SelectorEventLoop diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..2a7c18cda8a76a5 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -16,7 +16,6 @@ import time import weakref -from . import events from . import base_subprocess from . import futures from . import exceptions @@ -29,8 +28,7 @@ __all__ = ( 'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor', - '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy', - '_WindowsProactorEventLoopPolicy', 'EventLoop', + 'EventLoop', ) @@ -408,7 +406,7 @@ async def _make_subprocess_transport(self, protocol, args, shell, raise except BaseException: transp.close() - await transp._wait() + await tasks.shield(transp._wait()) raise return transp @@ -610,6 +608,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, @@ -759,6 +760,46 @@ def _get_accept_socket(self, family): s.settimeout(0) return s + def _process_completion_status(self, status): + """Process a single status from the completion port. + + A caller that waits on the completion port itself can pass each + status it receives here. + """ + err, transferred, key, address = status + try: + f, ov, obj, callback = self._cache.pop(address) + except KeyError: + if self._loop.get_debug(): + self._loop.call_exception_handler({ + 'message': ('GetQueuedCompletionStatus() returned an ' + 'unexpected event'), + 'status': ('err=%s transferred=%s key=%#x address=%#x' + % (err, transferred, key, address)), + }) + + # key is either zero, or it is used to return a pipe + # handle which should be closed to avoid a leak. + if key not in (0, _overlapped.INVALID_HANDLE_VALUE): + _winapi.CloseHandle(key) + return + + if obj in self._stopped_serving: + f.cancel() + # Don't call the callback if _register() already read the result or + # if the overlapped has been cancelled + elif not f.done(): + try: + value = callback(transferred, key, ov) + except OSError as e: + f.set_exception(e) + self._results.append(f) + else: + f.set_result(value) + self._results.append(f) + finally: + f = None + def _poll(self, timeout=None): if timeout is None: ms = INFINITE @@ -777,39 +818,8 @@ def _poll(self, timeout=None): break ms = 0 - err, transferred, key, address = status - try: - f, ov, obj, callback = self._cache.pop(address) - except KeyError: - if self._loop.get_debug(): - self._loop.call_exception_handler({ - 'message': ('GetQueuedCompletionStatus() returned an ' - 'unexpected event'), - 'status': ('err=%s transferred=%s key=%#x address=%#x' - % (err, transferred, key, address)), - }) - - # key is either zero, or it is used to return a pipe - # handle which should be closed to avoid a leak. - if key not in (0, _overlapped.INVALID_HANDLE_VALUE): - _winapi.CloseHandle(key) - continue - - if obj in self._stopped_serving: - f.cancel() - # Don't call the callback if _register() already read the result or - # if the overlapped has been cancelled - elif not f.done(): - try: - value = callback(transferred, key, ov) - except OSError as e: - f.set_exception(e) - self._results.append(f) - else: - f.set_result(value) - self._results.append(f) - finally: - f = None + # gh-154971: split out so custom event loops can call it directly + self._process_completion_status(status) # Remove unregistered futures for ov in self._unregistered: @@ -890,14 +900,4 @@ def callback(f): SelectorEventLoop = _WindowsSelectorEventLoop - -class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): - _loop_factory = SelectorEventLoop - - -class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy): - _loop_factory = ProactorEventLoop - - -_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy EventLoop = ProactorEventLoop diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index acd49441131b042..d6393f0b1ffee5d 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -111,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index 7f39c68070b5da8..fa562f74a810345 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,4 +1,4 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support @@ -68,7 +68,7 @@ def b64encode(s, altchars=None, *, padded=True, wrapcol=0): def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, - *, padded=True, ignorechars=_NOT_SPECIFIED): + *, padded=True, ignorechars=_NOT_SPECIFIED, canonical=False): """Decode the Base64 encoded bytes-like object or ASCII string s. Optional altchars must be a bytes-like object or ASCII string of length 2 @@ -110,11 +110,13 @@ def b64decode(s, altchars=None, validate=_NOT_SPECIFIED, alphabet = binascii.BASE64_ALPHABET[:-2] + altchars return binascii.a2b_base64(s, strict_mode=validate, alphabet=alphabet, - padded=padded, ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars, + canonical=canonical) if ignorechars is _NOT_SPECIFIED: ignorechars = b'' result = binascii.a2b_base64(s, strict_mode=validate, - padded=padded, ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars, + canonical=canonical) if badchar is not None: import warnings if validate: @@ -217,7 +219,7 @@ def urlsafe_b64decode(s, *, padded=False): characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -230,7 +232,8 @@ def b32encode(s, *, padded=True, wrapcol=0): return binascii.b2a_base32(s, padded=padded, wrapcol=wrapcol) b32encode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32') -def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''): +def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) # Handle section 2.4 zero and one mapping. The flag map01 will be either # False, or the character to map the digit 1 (one) to. It should be @@ -240,7 +243,8 @@ def b32decode(s, casefold=False, map01=None, *, padded=True, ignorechars=b''): s = s.translate(bytes.maketrans(b'01', b'O' + map01)) if casefold: s = s.upper() - return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars) + return binascii.a2b_base32(s, padded=padded, ignorechars=ignorechars, + canonical=canonical) b32decode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32', extra_args=_B32_DECODE_MAP01_DOCSTRING) @@ -249,18 +253,20 @@ def b32hexencode(s, *, padded=True, wrapcol=0): alphabet=binascii.BASE32HEX_ALPHABET) b32hexencode.__doc__ = _B32_ENCODE_DOCSTRING.format(encoding='base32hex') -def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b''): +def b32hexdecode(s, casefold=False, *, padded=True, ignorechars=b'', + canonical=False): s = _bytes_from_decode_data(s) # base32hex does not have the 01 mapping if casefold: s = s.upper() return binascii.a2b_base32(s, alphabet=binascii.BASE32HEX_ALPHABET, - padded=padded, ignorechars=ignorechars) + padded=padded, ignorechars=ignorechars, + canonical=canonical) b32hexdecode.__doc__ = _B32_DECODE_DOCSTRING.format(encoding='base32hex', extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s, *, wrapcol=0): @@ -309,38 +315,48 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. + + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. """ return binascii.b2a_ascii85(b, foldspaces=foldspaces, adobe=adobe, wrapcol=wrapcol, pad=pad) -def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v', + canonical=False): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ return binascii.a2b_ascii85(b, foldspaces=foldspaces, - adobe=adobe, ignorechars=ignorechars) + adobe=adobe, ignorechars=ignorechars, + canonical=canonical) def b85encode(b, pad=False, *, wrapcol=0): """Encode bytes-like object b in base85 format and return a bytes object. @@ -348,17 +364,22 @@ def b85encode(b, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ return binascii.b2a_base85(b, wrapcol=wrapcol, pad=pad) -def b85decode(b, *, ignorechars=b''): +def b85decode(b, *, ignorechars=b'', canonical=False): """Decode the base85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - return binascii.a2b_base85(b, ignorechars=ignorechars) + return binascii.a2b_base85(b, ignorechars=ignorechars, + canonical=canonical) def z85encode(s, pad=False, *, wrapcol=0): """Encode bytes-like object b in z85 format and return a bytes object. @@ -366,18 +387,23 @@ def z85encode(s, pad=False, *, wrapcol=0): If wrapcol is non-zero, insert a newline (b'\\n') character after at most every wrapcol characters. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, as required by the ZeroMQ standard. """ return binascii.b2a_base85(s, wrapcol=wrapcol, pad=pad, alphabet=binascii.Z85_ALPHABET) -def z85decode(s, *, ignorechars=b''): +def z85decode(s, *, ignorechars=b'', canonical=False): """Decode the z85-encoded bytes-like object or ASCII string b + If canonical is true, non-canonical encodings are rejected. + The result is returned as a bytes object. """ - return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, ignorechars=ignorechars) + return binascii.a2b_base85(s, alphabet=binascii.Z85_ALPHABET, + ignorechars=ignorechars, canonical=canonical) # Legacy interface. This code could be cleaned up since I don't believe # binascii has any line length limitations. It just doesn't seem worth it diff --git a/Lib/calendar.py b/Lib/calendar.py index d80c3fd9524776f..7aaaf73bfe0044b 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -609,11 +609,11 @@ def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None): content = self.formatyear(theyear, width) return self._format_html_page(theyear, content, css, encoding) - def formatmonthpage(self, theyear, themonth, width=3, css='calendar.css', encoding=None): + def formatmonthpage(self, theyear, themonth, *, css='calendar.css', encoding=None): """ Return a formatted month as a complete HTML page. """ - content = self.formatmonth(theyear, themonth, width) + content = self.formatmonth(theyear, themonth) return self._format_html_page(theyear, content, css, encoding) @@ -686,28 +686,61 @@ def __init__(self, highlight_day=None, *args, **kwargs): super().__init__(*args, **kwargs) self.highlight_day = highlight_day - def formatweek(self, theweek, width, *, highlight_day=None): + def _get_theme(self): + from _colorize import get_theme + + return get_theme(tty_file=sys.stdout) + + def formatday(self, day, weekday, width, *, highlight_day=None): """ - Returns a single week in a string (no newline). + Returns a formatted day. """ - if highlight_day: - from _colorize import get_colors - - ansi = get_colors() - highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}" - reset = ansi.RESET + if day == 0: + s = '' else: - highlight = reset = "" + s = f'{day:2}' + s = s.center(width) + if day == highlight_day: + theme = self._get_theme().calendar + s = f"{theme.highlight}{s}{theme.reset}" + return s + def formatweek(self, theweek, width, *, highlight_day=None): + """ + Returns a single week in a string (no newline). + """ return ' '.join( - ( - f"{highlight}{self.formatday(d, wd, width)}{reset}" - if d == highlight_day - else self.formatday(d, wd, width) - ) + self.formatday(d, wd, width, highlight_day=highlight_day) for (d, wd) in theweek ) + def formatweekheader(self, width): + """ + Return a header for a week. + """ + header = super().formatweekheader(width) + theme = self._get_theme().calendar + return f"{theme.weekday}{header}{theme.reset}" + + def formatmonthname(self, theyear, themonth, width, withyear=True): + """ + Return a formatted month name. + """ + name = super().formatmonthname(theyear, themonth, width, withyear) + theme = self._get_theme().calendar + if ( + self.highlight_day + and self.highlight_day.year == theyear + and self.highlight_day.month == themonth + ): + color = theme.highlight + name_only = name.strip() + colored_name = f"{color}{name_only}{theme.reset}" + return name.replace(name_only, colored_name, 1) + else: + color = theme.header + return f"{color}{name}{theme.reset}" + def formatmonth(self, theyear, themonth, w=0, l=0): """ Return a month's calendar string (multi-line). @@ -742,7 +775,9 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3): colwidth = (w + 1) * 7 - 1 v = [] a = v.append - a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip()) + theme = self._get_theme().calendar + year = repr(theyear).center(colwidth*m+c*(m-1)).rstrip() + a(f"{theme.header}{year}{theme.reset}") a('\n'*l) header = self.formatweekheader(w) for (i, row) in enumerate(self.yeardays2calendar(theyear, m)): @@ -843,28 +878,30 @@ def timegm(tuple): def main(args=None): import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) textgroup = parser.add_argument_group('text only arguments') htmlgroup = parser.add_argument_group('html only arguments') textgroup.add_argument( "-w", "--width", type=int, default=2, - help="width of date column (default 2)" + help="width of date column" ) textgroup.add_argument( "-l", "--lines", type=int, default=1, - help="number of lines for each week (default 1)" + help="number of lines for each week" ) textgroup.add_argument( "-s", "--spacing", type=int, default=6, - help="spacing between months (default 6)" + help="spacing between months" ) textgroup.add_argument( "-m", "--months", type=int, default=3, - help="months per row (default 3)" + help="months per row" ) htmlgroup.add_argument( "-c", "--css", @@ -879,18 +916,18 @@ def main(args=None): parser.add_argument( "-e", "--encoding", default=None, - help="encoding to use for output (default utf-8)" + help="encoding to use for output" ) parser.add_argument( "-t", "--type", default="text", choices=("text", "html"), - help="output type (text or html)" + help="output type (`text` or `html`)" ) parser.add_argument( "-f", "--first-weekday", type=int, default=0, - help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)" + help="weekday (0 is Monday, 6 is Sunday) to start each week" ) parser.add_argument( "year", diff --git a/Lib/code.py b/Lib/code.py index f7e275d8801b7c3..df1d7199e339341 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -385,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=Fa if __name__ == "__main__": import argparse - parser = argparse.ArgumentParser(color=True) + parser = argparse.ArgumentParser() parser.add_argument('-q', action='store_true', help="don't print version and copyright messages") args = parser.parse_args() diff --git a/Lib/codecs.py b/Lib/codecs.py index e4a8010aba90a50..af6ab031157e790 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -93,7 +93,7 @@ class CodecInfo(tuple): def __new__(cls, encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None, - *, _is_text_encoding=None): + *, _is_text_encoding=None, _expat_decoding_table=None): self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter)) self.name = name self.encode = encode @@ -104,6 +104,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None, self.streamreader = streamreader if _is_text_encoding is not None: self._is_text_encoding = _is_text_encoding + if _expat_decoding_table is not None: + self._expat_decoding_table = _expat_decoding_table return self def __repr__(self): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index febab521629228c..5dbcac19e7a9272 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -553,6 +553,9 @@ class Counter(dict): or multiset. Elements are stored as dictionary keys and their counts are stored as dictionary values. + When constructed from a Mapping or Counter, the original object's + values will be used as the initial counts. + >>> c = Counter('abcdeabcdabcaba') # count elements from a string >>> c.most_common(3) # three most common elements @@ -1253,6 +1256,20 @@ def copy(self): c.update(self) return c + + # This method has a default implementation in MutableMapping, but dict's + # equivalent is last-in, first-out instead of first-in, first-out. + def popitem(self): + """Remove and return a (key, value) pair as a 2-tuple. + + Removes pairs in the same order as the wrapped mapping's popitem() + method. For dict objects (the default), that order is last-in, + first-out (LIFO). + Raises KeyError if the UserDict is empty. + """ + return self.data.popitem() + + @classmethod def fromkeys(cls, iterable, value=None): d = cls() diff --git a/Lib/compileall.py b/Lib/compileall.py index c452aed135838f9..812a496611e043f 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -326,7 +326,6 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.', - color=True, ) parser.add_argument('-l', action='store_const', const=0, default=None, dest='maxlevels', @@ -338,10 +337,10 @@ def main(): parser.add_argument('-f', action='store_true', dest='force', help='force rebuild even if timestamps are up to date') parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' + help='output only error messages; `-qq` will suppress ' 'the error messages as well.') parser.add_argument('-b', action='store_true', dest='legacy', - help='use legacy (pre-PEP3147) compiled file locations') + help='use legacy (pre-PEP 3147) compiled file locations') parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, help=('directory to prepend to file paths for use in ' 'compile-time tracebacks and in runtime ' @@ -367,28 +366,28 @@ def main(): 'of each file considered for compilation')) parser.add_argument('-i', metavar='FILE', dest='flist', help=('add all the files and directories listed in ' - 'FILE to the list considered for compilation; ' - 'if "-", names are read from stdin')) + '`FILE` to the list considered for compilation; ' + 'if `"-"`, names are read from `stdin`')) parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', help=('zero or more file and directory names ' 'to compile; if no arguments given, defaults ' - 'to the equivalent of -l sys.path')) + 'to the equivalent of `-l` `sys.path`')) parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') invalidation_modes = [mode.name.lower().replace('_', '-') for mode in py_compile.PycInvalidationMode] parser.add_argument('--invalidation-mode', choices=sorted(invalidation_modes), - help=('set .pyc invalidation mode; defaults to ' - '"checked-hash" if the SOURCE_DATE_EPOCH ' + help=('set `.pyc` invalidation mode; defaults to ' + '`"checked-hash"` if the `SOURCE_DATE_EPOCH` ' 'environment variable is set, and ' - '"timestamp" otherwise.')) + '`"timestamp"` otherwise.')) parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses the optimization level ' - 'of the Python interpreter itself (see -O).')) + 'Default is `-1` which uses the optimization level ' + 'of the Python interpreter itself (see `-O`).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', - help='Ignore symlinks pointing outsite of the DIR') + help='Ignore symlinks pointing outsite of the `DIR`') parser.add_argument('--hardlink-dupes', action='store_true', dest='hardlink_dupes', help='Hardlink duplicated pyc files') diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 84b25914b0aa93a..5326cf9b19cf883 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -61,8 +61,9 @@ def __setattr__(self, name, _): def get_frame_info(frame_buffer): """Get Zstandard frame information from a frame header. - *frame_buffer* is a bytes-like object. It should start from the beginning - of a frame, and needs to include at least the frame header (6 to 18 bytes). + *frame_buffer* is a bytes-like object. It should start from the + beginning of a frame, and needs to include at least the frame header + (6 to 18 bytes). The returned FrameInfo object has two attributes. 'decompressed_size' is the size in bytes of the data in the frame when @@ -103,16 +104,17 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): finalize *zstd_dict* by adding headers and statistics according to the Zstandard dictionary format. - You may compose an effective dictionary content by hand, which is used as - basis dictionary, and use some samples to finalize a dictionary. The basis - dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + You may compose an effective dictionary content by hand, which is used + as basis dictionary, and use some samples to finalize a dictionary. The + basis dictionary may be a "raw content" dictionary. See *is_raw* in + ZstdDict. - *samples* is an iterable of samples, where a sample is a bytes-like object - representing a file. + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. *dict_size* is the dictionary's maximum size, in bytes. *level* is the expected compression level. The statistics for each - compression level differ, so tuning the dictionary to the compression level - can provide improvements. + compression level differ, so tuning the dictionary to the compression + level can provide improvements. """ if not isinstance(zstd_dict, ZstdDict): @@ -140,8 +142,8 @@ def compress(data, level=None, options=None, zstd_dict=None): COMPRESSION_LEVEL_DEFAULT ('3'). *options* is a dict object that contains advanced compression parameters. See CompressionParameter for more on options. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. For incremental compression, use a ZstdCompressor instead. """ @@ -152,8 +154,8 @@ def compress(data, level=None, options=None, zstd_dict=None): def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. *options* is a dict object that contains advanced compression parameters. See DecompressionParameter for more on options. diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index d709f5efc658fa4..26d7c9006375fb6 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -34,11 +34,11 @@ def __init__(self, file, /, mode='r', *, level=None, options=None, zstd_dict=None): """Open a Zstandard compressed file in binary mode. - *file* can be either an file-like object, or a file name to open. + *file* can be either a file-like object, or a file name to open. - *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for - creating exclusively, or 'a' for appending. These can equivalently be - given as 'rb', 'wb', 'xb' and 'ab' respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' + for creating exclusively, or 'a' for appending. These can + equivalently be given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -246,6 +246,7 @@ def peek(self, size=-1): return self._buffer.peek(size) def __next__(self): + self._check_can_read() if ret := self._buffer.readline(): return ret raise StopIteration @@ -296,26 +297,27 @@ def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. - file can be either a file name (given as a str, bytes, or PathLike object), - in which case the named file is opened, or it can be an existing file object - to read from or write to. + file can be either a file name (given as a str, bytes, or PathLike + object), in which case the named file is opened, or it can be + an existing file object to read from or write to. - The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', - 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', + 'a', 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. - The level, options, and zstd_dict parameters specify the settings the same - as ZstdFile. + The level, options, and zstd_dict parameters specify the settings the + same as ZstdFile. When using read mode (decompression), the options parameter is a dict - representing advanced decompression options. The level parameter is not - supported in this case. When using write mode (compression), only one of - level, an int representing the compression level, or options, a dict - representing advanced compression options, may be passed. In both modes, - zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. - - For binary mode, this function is equivalent to the ZstdFile constructor: - ZstdFile(filename, mode, ...). In this case, the encoding, errors and - newline parameters must not be provided. + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one + of level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both + modes, zstd_dict is a ZstdDict instance containing a trained Zstandard + dictionary. + + For binary mode, this function is equivalent to the ZstdFile + constructor: ZstdFile(filename, mode, ...). In this case, the encoding, + errors and newline parameters must not be provided. For text mode, an ZstdFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index d6ac4b3e0b675f6..5d53a5ac50931d3 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -17,6 +17,9 @@ wait, as_completed) +lazy from .process import ProcessPoolExecutor +lazy from .thread import ThreadPoolExecutor + __all__ = [ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', @@ -40,26 +43,9 @@ _interpreters = None if _interpreters: + lazy from .interpreter import InterpreterPoolExecutor # noqa: F401 __all__.append('InterpreterPoolExecutor') def __dir__(): return __all__ + ['__author__', '__doc__'] - - -def __getattr__(name): - global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor - - if name == 'ProcessPoolExecutor': - from .process import ProcessPoolExecutor - return ProcessPoolExecutor - - if name == 'ThreadPoolExecutor': - from .thread import ThreadPoolExecutor - return ThreadPoolExecutor - - if _interpreters and name == 'InterpreterPoolExecutor': - from .interpreter import InterpreterPoolExecutor - return InterpreterPoolExecutor - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index f506ce68aea5b29..e728b8e0a91f744 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -194,15 +194,15 @@ def as_completed(fs, timeout=None): """An iterator over the given futures that yields each as it completes. Args: - fs: The sequence of Futures (possibly created by different Executors) to - iterate over. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. + fs: The sequence of Futures (possibly created by different + Executors) to iterate over. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. Returns: - An iterator that yields the given Futures as they complete (finished or - cancelled). If any given Futures are duplicated, they will be returned - once. + An iterator that yields the given Futures as they complete + (finished or cancelled). If any given Futures are duplicated, + they will be returned once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -258,19 +258,20 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): """Wait for the futures in the given sequence to complete. Args: - fs: The sequence of Futures (possibly created by different Executors) to - wait upon. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - return_when: Indicates when this function should return. The options - are: + fs: The sequence of Futures (possibly created by different + Executors) to wait upon. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + return_when: Indicates when this function should return. + The options are: FIRST_COMPLETED - Return when any future finishes or is cancelled. FIRST_EXCEPTION - Return when any future finishes by raising an - exception. If no future raises an exception + exception. If no future raises an exception then it is equivalent to ALL_COMPLETED. - ALL_COMPLETED - Return when all futures finish or are cancelled. + ALL_COMPLETED - Return when all futures finish or are + cancelled. Returns: A named 2-tuple of sets. The first set, named 'done', contains the @@ -308,7 +309,14 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): def _result_or_cancel(fut, timeout=None): try: try: - return fut.result(timeout) + # fut.exception() returns the call's own error but raises + # TimeoutError only for a map() timeout. + exc = fut.exception(timeout) + if exc is not None: + return (None, exc) + return (fut.result(), None) + except CancelledError as exc: + return (None, exc) finally: fut.cancel() finally: @@ -404,11 +412,12 @@ def add_done_callback(self, fn): Args: fn: A callable that will be called with this future as its only - argument when the future completes or is cancelled. The callable - will always be called by a thread in the same process in which - it was added. If the future has already completed or been - cancelled then the callable will be called immediately. These - callables are called in the order that they were added. + argument when the future completes or is cancelled. The + callable will always be called by a thread in the same + process in which it was added. If the future has already + completed or been cancelled then the callable will be + called immediately. These callables are called in the + order that they were added. """ with self._condition: if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: @@ -423,17 +432,19 @@ def result(self, timeout=None): """Return the result of the call that the future represents. Args: - timeout: The number of seconds to wait for the result if the future - isn't done. If None, then there is no limit on the wait time. + timeout: The number of seconds to wait for the result if the + future isn't done. If None, then there is no limit on the + wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. - Exception: If the call raised then that exception will be raised. + TimeoutError: If the future didn't finish executing before the + given timeout. + Exception: If the call raised then that exception will be + raised. """ try: with self._condition: @@ -459,17 +470,17 @@ def exception(self, timeout=None): Args: timeout: The number of seconds to wait for the exception if the - future isn't done. If None, then there is no limit on the wait - time. + future isn't done. If None, then there is no limit on the + wait time. Returns: - The exception raised by the call that the future represents or None - if the call completed without raising. + The exception raised by the call that the future represents or + None if the call completed without raising. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. + TimeoutError: If the future didn't finish executing before the + given timeout. """ with self._condition: @@ -494,22 +505,23 @@ def set_running_or_notify_cancel(self): Should only be used by Executor implementations and unit tests. If the future has been cancelled (cancel() was called and returned - True) then any threads waiting on the future completing (though calls - to as_completed() or wait()) are notified and False is returned. + True) then any threads waiting on the future completing (though + calls to as_completed() or wait()) are notified and False is + returned. If the future was not cancelled then it is put in the running state (future calls to running() will return True) and True is returned. This method should be called by Executor implementations before - executing the work associated with this future. If this method returns - False then the work should not be executed. + executing the work associated with this future. If this method + returns False then the work should not be executed. Returns: False if the Future was cancelled, True otherwise. Raises: - RuntimeError: if this method was already called or if set_result() - or set_exception() was called. + RuntimeError: if this method was already called or if + set_result() or set_exception() was called. """ with self._condition: if self._state == CANCELLED: @@ -587,14 +599,16 @@ def _get_snapshot(self): __class_getitem__ = classmethod(types.GenericAlias) + class Executor(object): """This is an abstract base class for concrete asynchronous executors.""" def submit(self, fn, /, *args, **kwargs): """Submits a callable to be executed with the given arguments. - Schedules the callable to be executed as fn(*args, **kwargs) and returns - a Future instance representing the execution of the callable. + Schedules the callable to be executed as fn(*args, **kwargs) and + returns a Future instance representing the execution of the + callable. Returns: A Future representing the given call. @@ -607,32 +621,35 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: The size of the chunks the iterable will be broken into - before being passed to a child process. This argument is only - used by ProcessPoolExecutor; it is ignored by + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: The size of the chunks the iterable will be broken + into before being passed to a child process. This argument + is only used by ProcessPoolExecutor; it is ignored by ThreadPoolExecutor. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if buffersize is not None and not isinstance(buffersize, int): raise TypeError("buffersize must be an integer or None") if buffersize is not None and buffersize < 1: raise ValueError("buffersize must be None or > 0") + return _MapResultIterator(self._map(fn, *iterables, timeout=timeout, + buffersize=buffersize)) + def _map(self, fn, *iterables, timeout=None, buffersize=None): if timeout is not None: end_time = timeout + time.monotonic() @@ -679,8 +696,8 @@ def shutdown(self, wait=True, *, cancel_futures=False): Args: wait: If True then shutdown will not return until all running - futures have finished executing and the resources used by the - executor have been reclaimed. + futures have finished executing and the resources used by + the executor have been reclaimed. cancel_futures: If True then shutdown will cancel all pending futures. Futures that are completed or running will not be cancelled. @@ -695,7 +712,25 @@ def __exit__(self, exc_type, exc_val, exc_tb): return False +class _MapResultIterator: + """The iterator returned by map().""" + def __init__(self, gen): + self.gen = gen + + def __iter__(self): + return self + + def __next__(self): + value, exc = next(self.gen) + if exc is not None: + raise exc + return value + + def close(self): + self.gen.close() + + class BrokenExecutor(RuntimeError): """ - Raised when a executor has become non-functional after a severe failure. + Raised when an executor has become non-functional after a severe failure. """ diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index 85c1da2c7228943..fd3d45144b49a71 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -110,8 +110,8 @@ def __init__(self, max_workers=None, thread_name_prefix='', """Initializes a new InterpreterPoolExecutor instance. Args: - max_workers: The maximum number of interpreters that can be used to - execute the given calls. + max_workers: The maximum number of interpreters that can be used + to execute the given calls. thread_name_prefix: An optional name prefix to give our threads. initializer: A callable or script used to initialize each worker interpreter. diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index a42afa68efcb148..c130259acb737ab 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -200,7 +200,14 @@ def _process_chunk(fn, chunk): This function is run in a separate process. """ - return [fn(*args) for args in chunk] + results = [] + for args in chunk: + try: + result = (fn(*args), None) + except BaseException as exc: + result = (None, exc) + results.append(result) + return results def _sendback_result(result_queue, work_id, result=None, exception=None, @@ -269,6 +276,20 @@ def _process_worker(call_queue, result_queue, initializer, initargs, max_tasks=N return +def _spawn_worker(mp_context, call_queue, result_queue, initializer, + initargs, max_tasks_per_child, processes): + """Start one worker process and record it in *processes* by pid.""" + p = mp_context.Process( + target=_process_worker, + args=(call_queue, + result_queue, + initializer, + initargs, + max_tasks_per_child)) + p.start() + processes[p.pid] = p + + class _ExecutorManagerThread(threading.Thread): """Manages the communication between this process and the worker processes. @@ -321,6 +342,15 @@ def weakref_cb(_, # exiting safely self.max_tasks_per_child = executor._max_tasks_per_child + # gh-119592: Needed to size worker replacement, and immutable, so + # keep a copy rather than reading it back through the executor + # weakref. The rest of the spawn configuration is deliberately NOT + # copied here: holding user-provided objects (initializer, + # initargs, mp_context) in this always-reachable running thread + # could keep the executor itself reachable through them, breaking + # garbage-collection-triggered shutdown. + self.max_workers = executor._max_workers + # A dict mapping work ids to _WorkItems e.g. # {5: <_WorkItem...>, 6: <_WorkItem...>, ...} self.pending_work_items = executor._pending_work_items @@ -357,12 +387,14 @@ def run(self): # while waiting on new results. del result_item - if executor := self.executor_reference(): - if process_exited: - with self.shutdown_lock: - executor._adjust_process_count() - else: - executor._idle_worker_semaphore.release() + if process_exited: + with self.shutdown_lock: + broken = self._replace_dead_worker() + if broken is not None: + self.terminate_broken(*broken) + return + elif executor := self.executor_reference(): + executor._idle_worker_semaphore.release() del executor if self.is_shutting_down(): @@ -379,6 +411,71 @@ def run(self): self.join_executor_internals() return + def _replace_dead_worker(self): + """Spawn a replacement for a worker that exited at its + max_tasks_per_child limit. Called under self.shutdown_lock. + + Returns None while the pool can still make progress, otherwise a + (cause, message) tuple describing why the remaining work items can + never run, so that run() can fail their futures. + """ + assert self.shutdown_lock.locked() + cause = None + message = None + executor = self.executor_reference() + if executor is None: + # gh-152967: The executor was garbage collected; nothing can + # spawn a replacement worker for it anymore. + message = ("The ProcessPoolExecutor was garbage collected with " + "work pending after its last worker process exited " + "upon reaching max_tasks_per_child; the pending work " + "can never be run.") + elif executor._force_shutting_down: + # terminate_workers()/kill_workers() is tearing the pool down; + # a replacement worker would escape the kill and run work + # items that were enqueued before it. + message = ("A worker process exited while the pool was being " + "forcefully shut down; work that was still enqueued " + "will not be run.") + elif self.pending_work_items or not self.is_shutting_down(): + # gh-115634: Do not consult the executor's + # _idle_worker_semaphore here: it counts task completions, not + # idle workers, so it can hold a stale token released by the + # now-dead worker. Trusting such a token would leave the pool + # a worker short, deadlocking once all workers reach their + # task limit. Spawning from this (manager) thread is safe + # despite gh-90622 because max_tasks_per_child is rejected for + # the "fork" start method. + if len(self.processes) < self.max_workers: + # gh-119592: Spawn using state owned by this thread and + # configuration read through the live weakref (which + # shutdown() never clears), not the executor state that + # shutdown(wait=False) clears concurrently. + try: + _spawn_worker(executor._mp_context, self.call_queue, + self.result_queue, executor._initializer, + executor._initargs, + self.max_tasks_per_child, self.processes) + except Exception as exc: + # While other workers remain the pool has merely lost + # capacity and they keep draining the queue; with none + # left the failure is reported below. + cause = format_exception(exc) + message = ("A replacement worker process could not be " + "started, leaving the pool without workers " + "to run the remaining work.") + del executor + + if not self.processes and (self.pending_work_items + or cause is not None): + # No worker processes remain and no replacement can be + # spawned: any remaining work items can never run. A spawn + # failure breaks the pool even with nothing pending; leaving + # a zero-worker pool alive would hang a later submit() on a + # stale _idle_worker_semaphore token instead of raising. + return (cause, message) + return None + def add_call_item_to_queue(self): # Fills call_queue with _WorkItems from pending_work_items. # This function never blocks. @@ -455,10 +552,11 @@ def is_shutting_down(self): return (_global_shutdown or executor is None or executor._shutdown_thread) - def _terminate_broken(self, cause): + def _terminate_broken(self, cause, bpe_message=None): # Terminate the executor because it is in a broken state. The cause # argument can be used to display more information on the error that - # lead the executor into becoming broken. + # lead the executor into becoming broken. bpe_message overrides the + # default message on the BrokenProcessPool set on pending futures. # Mark the process pool broken so that submits fail right now. executor = self.executor_reference() @@ -469,11 +567,9 @@ def _terminate_broken(self, cause): executor._shutdown_thread = True executor = None - # All pending tasks are to be marked failed with the following - # BrokenProcessPool error - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + # All pending tasks are to be marked failed with a + # BrokenProcessPool error, as separate instances to avoid sharing + # a traceback (gh-101267). cause_str = None if cause is not None: cause_str = ''.join(cause) @@ -489,11 +585,16 @@ def _terminate_broken(self, cause): f"with exit code {p.exitcode}") if errors: cause_str = "\n".join(errors) - if cause_str: - bpe.__cause__ = _RemoteTraceback(f"\n'''\n{cause_str}'''") + cause_tb = f"\n'''\n{cause_str}'''" if cause_str else None + if bpe_message is None: + bpe_message = ("A process in the process pool was terminated " + "abruptly while the future was running or pending.") # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): + bpe = BrokenProcessPool(bpe_message) + if cause_tb is not None: + bpe.__cause__ = _RemoteTraceback(cause_tb) try: work_item.future.set_exception(bpe) except _base.InvalidStateError: @@ -516,9 +617,9 @@ def _terminate_broken(self, cause): # clean up resources self._join_executor_internals(broken=True) - def terminate_broken(self, cause): + def terminate_broken(self, cause, bpe_message=None): with self.shutdown_lock: - self._terminate_broken(cause) + self._terminate_broken(cause, bpe_message) def flag_executor_shutting_down(self): # Flag the executor as shutting down and cancel remaining tasks if @@ -656,19 +757,21 @@ def __init__(self, max_workers=None, mp_context=None, Args: max_workers: The maximum number of processes that can be used to - execute the given calls. If None or not given then as many - worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers created - using the multiprocessing.get_context('start method') API. This - object should provide SimpleQueue, Queue and Process. + execute the given calls. If None or not given then as many + worker processes will be created as the machine has + processors. + mp_context: A multiprocessing context to launch the workers + created using the multiprocessing.get_context('start method') + API. This object should provide SimpleQueue, Queue and + Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. - max_tasks_per_child: The maximum number of tasks a worker process - can complete before it will exit and be replaced with a fresh - worker process. The default of None means worker process will - live as long as the executor. Requires a non-'fork' mp_context - start method. When given, we default to using 'spawn' if no - mp_context is supplied. + max_tasks_per_child: The maximum number of tasks a worker + process can complete before it will exit and be replaced + with a fresh worker process. The default of None means + worker process will live as long as the executor. Requires + a non-'fork' mp_context start method. When given, we + default to using 'spawn' if no mp_context is supplied. """ _check_system_limits() @@ -729,6 +832,7 @@ def __init__(self, max_workers=None, mp_context=None, self._queue_count = 0 self._pending_work_items = {} self._cancel_pending_futures = False + self._force_shutting_down = False # _ThreadWakeup is a communication channel used to interrupt the wait # of the main loop of executor_manager_thread from another thread (e.g. @@ -774,18 +878,23 @@ def _adjust_process_count(self): if self._processes is None: return + # gh-152967: A forceful shutdown is in progress; a worker spawned + # here could escape its process snapshot and keep running work. + if self._force_shutting_down: + return + # if there's an idle process, we don't need to spawn a new one. if self._idle_worker_semaphore.acquire(blocking=False): return process_count = len(self._processes) if process_count < self._max_workers: - # Assertion disabled as this codepath is also used to replace a - # worker that unexpectedly dies, even when using the 'fork' start - # method. That means there is still a potential deadlock bug. If a - # 'fork' mp_context worker dies, we'll be forking a new one when - # we know a thread is running (self._executor_manager_thread). - #assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622' + # gh-90622: spawning a child via fork while another thread is + # running can deadlock in the child. submit() only calls this + # method when using a non-fork start method. + assert (self._safe_to_dynamically_spawn_children + or not self._executor_manager_thread), ( + 'https://github.com/python/cpython/issues/90622') self._spawn_process() def _launch_processes(self): @@ -797,15 +906,10 @@ def _launch_processes(self): self._spawn_process() def _spawn_process(self): - p = self._mp_context.Process( - target=_process_worker, - args=(self._call_queue, - self._result_queue, - self._initializer, - self._initargs, - self._max_tasks_per_child)) - p.start() - self._processes[p.pid] = p + _spawn_worker(self._mp_context, self._call_queue, + self._result_queue, self._initializer, + self._initargs, self._max_tasks_per_child, + self._processes) def submit(self, fn, /, *args, **kwargs): with self._shutdown_lock: @@ -838,24 +942,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: If greater than one, the iterables will be chopped into - chunks of size chunksize and submitted to the process pool. - If set to one, the items in the list will be sent one at a time. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: If greater than one, the iterables will be chopped + into chunks of size chunksize and submitted to the process + pool. If set to one, the items in the list will be sent + one at a time. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if chunksize < 1: @@ -865,7 +970,7 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): itertools.batched(zip(*iterables), chunksize), timeout=timeout, buffersize=buffersize) - return _chain_from_iterable_of_lists(results) + return _base._MapResultIterator(_chain_from_iterable_of_lists(results)) def shutdown(self, wait=True, *, cancel_futures=False): with self._shutdown_lock: @@ -901,6 +1006,14 @@ def _force_shutdown(self, operation): if operation not in _SHUTDOWN_CALLBACK_OPERATION: raise ValueError(f"Unsupported operation: {operation!r}") + # gh-152967: Stop the manager thread from spawning replacement + # workers before we copy the processes to signal: a worker spawned + # after the copy would survive the loop below and run enqueued + # work items. Taking the lock orders this against the manager's + # worker replacement, which runs under the same lock. + with self._shutdown_lock: + self._force_shutting_down = True + processes = {} if self._processes: processes = self._processes.copy() diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index ea4147ee9a25da5..335a744b727d100 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -68,8 +68,14 @@ def create(): def list_all(): """Return all existing interpreters.""" - return [Interpreter(id, _whence=whence) - for id, whence in _interpreters.list_all(require_ready=True)] + interps = [] + for id, whence in _interpreters.list_all(require_ready=True): + try: + interps.append(Interpreter(id, _whence=whence)) + except InterpreterNotFoundError: + # It was destroyed after it was listed. + pass + return interps def get_current(): diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index ee159d7de638272..fc4ee595f3aa995 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -185,7 +185,8 @@ def put(self, obj, block=True, timeout=None, *, underlying data is actually shared. Furthermore, some types can be sent through a queue more efficiently than others. This group includes various immutable types like int, str, bytes, and - tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). + tuple (if the items are likewise efficiently shareable). + See interpreters.is_shareable(). "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later @@ -219,12 +220,12 @@ def put(self, obj, block=True, timeout=None, *, timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout + end = time.monotonic() + timeout while True: try: _queues.put(self._id, obj, unboundop) except QueueFull: - if timeout is not None and time.time() >= end: + if timeout is not None and time.monotonic() >= end: raise # re-raise time.sleep(_delay) else: @@ -254,12 +255,12 @@ def get(self, block=True, timeout=None, *, timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') - end = time.time() + timeout + end = time.monotonic() + timeout while True: try: obj, unboundop = _queues.get(self._id) except QueueEmpty: - if timeout is not None and time.time() >= end: + if timeout is not None and time.monotonic() >= end: raise # re-raise time.sleep(_delay) else: diff --git a/Lib/configparser.py b/Lib/configparser.py index e76647d339e9138..88015ef60865698 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -315,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod @@ -615,7 +618,8 @@ class RawConfigParser(MutableMapping): _OPT_TMPL = r""" (?P