diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 63252a76abb69f..9b3415fd2b5861 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,83 +1,16 @@ -variables: - coverage: false - -trigger: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] +trigger: ['main', '3.*'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-22.04 + vmImage: ubuntu-24.04 steps: - template: ./prebuild-checks.yml -- job: macOS_CI_Tests - displayName: macOS CI Tests - dependsOn: Prebuild - #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - # bpo-39837: macOS tests on Azure Pipelines are disabled - condition: false - - variables: - testRunTitle: '$(build.sourceBranchName)-macos' - testRunPlatform: macos - - pool: - vmImage: macos-10.15 - - steps: - - template: ./macos-steps.yml - - -- job: Ubuntu_CI_Tests - displayName: Ubuntu CI Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(build.sourceBranchName)-linux' - testRunPlatform: linux - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - - -- job: Ubuntu_Coverage_CI_Tests - displayName: Ubuntu CI Tests (coverage) - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['coverage'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-coverage' - testRunPlatform: linux-coverage - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - coverage: true - - - job: Windows_CI_Tests displayName: Windows CI Tests dependsOn: Prebuild diff --git a/.azure-pipelines/macos-steps.yml b/.azure-pipelines/macos-steps.yml deleted file mode 100644 index fa38a0df8c87b8..00000000000000 --- a/.azure-pipelines/macos-steps.yml +++ /dev/null @@ -1,27 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- script: ./configure --with-pydebug --with-openssl=/usr/local/opt/openssl --prefix=/opt/python-azdev - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - continueOnError: true - timeoutInMinutes: 30 - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/.azure-pipelines/posix-deps-apt.sh b/.azure-pipelines/posix-deps-apt.sh deleted file mode 100755 index e0f4ca5d8d8e88..00000000000000 --- a/.azure-pipelines/posix-deps-apt.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -apt-get update - -apt-get -yq install \ - build-essential \ - zlib1g-dev \ - libbz2-dev \ - liblzma-dev \ - libncurses5-dev \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libgdbm-dev \ - tk-dev \ - lzma \ - lzma-dev \ - liblzma-dev \ - libffi-dev \ - uuid-dev \ - xvfb - -if [ ! -z "$1" ] -then - echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 - echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 - python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux -fi diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml deleted file mode 100644 index 9d7c5e1279f46d..00000000000000 --- a/.azure-pipelines/posix-steps.yml +++ /dev/null @@ -1,83 +0,0 @@ -parameters: - coverage: false - sudo_dependencies: sudo - dependencies: apt - patchcheck: true - xvfb: true - -steps: -- checkout: self - clean: true - fetchDepth: 5 - -# Work around a known issue affecting Ubuntu VMs on Pipelines -- script: sudo setfacl -Rb /home/vsts - displayName: 'Workaround ACL issue' - -- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version) - displayName: 'Install dependencies' - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- ${{ if eq(parameters.coverage, 'true') }}: - - script: ./python -m venv venv && ./venv/bin/python -m pip install -U coverage - displayName: 'Set up virtual environment' - - - script: ./venv/bin/python -m test.pythoninfo - displayName: 'Display build info' - - - script: | - $COMMAND -m coverage run --pylib -m test \ - --fail-env-changed \ - -uall,-cpu \ - --junit-xml=$(build.binariesDirectory)/test-results.xml \ - -x test_multiprocessing_fork \ - -x test_multiprocessing_forkserver \ - -x test_multiprocessing_spawn \ - -x test_concurrent_futures - displayName: 'Tests with coverage' - env: - ${{ if eq(parameters.xvfb, 'true') }}: - COMMAND: xvfb-run ./venv/bin/python - ${{ if ne(parameters.xvfb, 'true') }}: - COMMAND: ./venv/bin/python - - - script: ./venv/bin/python -m coverage xml - displayName: 'Generate coverage.xml' - - - script: source ./venv/bin/activate && bash <(curl -s https://codecov.io/bash) -y .github/codecov.yml - displayName: 'Publish code coverage results' - - -- ${{ if ne(parameters.coverage, 'true') }}: - - script: make pythoninfo - displayName: 'Display build info' - - - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" - displayName: 'Tests' - env: - ${{ if eq(parameters.xvfb, 'true') }}: - COMMAND: xvfb-run make - ${{ if ne(parameters.xvfb, 'true') }}: - COMMAND: make - -- ${{ if eq(parameters.patchcheck, 'true') }}: - - script: | - git fetch origin - ./python Tools/patchcheck/patchcheck.py --ci true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml deleted file mode 100644 index 939c9b4249a3c2..00000000000000 --- a/.azure-pipelines/pr.yml +++ /dev/null @@ -1,111 +0,0 @@ -variables: - coverage: false - -pr: ['main', '3.11', '3.10', '3.9', '3.8', '3.7'] - -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: macOS_PR_Tests - displayName: macOS PR Tests - dependsOn: Prebuild - #condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - # bpo-39837: macOS tests on Azure Pipelines are disabled - condition: false - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-macos' - testRunPlatform: macos - - pool: - vmImage: macos-10.15 - - steps: - - template: ./macos-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) - - -- job: Ubuntu_PR_Tests - displayName: Ubuntu PR Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-linux' - testRunPlatform: linux - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - - -- job: Ubuntu_Coverage_PR_Tests - displayName: Ubuntu PR Tests (coverage) - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['coverage'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(Build.SourceBranchName)-linux-coverage' - testRunPlatform: linux-coverage - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: apt - coverage: true - - -- job: Windows_PR_Tests - displayName: Windows PR Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: windows-2022 - - strategy: - matrix: - win32: - arch: win32 - buildOpt: '-p Win32' - testRunTitle: '$(System.PullRequest.TargetBranch)-win32' - testRunPlatform: win32 - win64: - arch: amd64 - buildOpt: '-p x64' - testRunTitle: '$(System.PullRequest.TargetBranch)-win64' - testRunPlatform: win64 - winarm64: - arch: arm64 - buildOpt: '-p arm64' - maxParallel: 4 - - steps: - - template: ./windows-steps.yml - parameters: - targetBranch: $(System.PullRequest.TargetBranch) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 590d7834b2b8be..00000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM docker.io/library/fedora:37 - -ENV CC=clang - -ENV WASI_SDK_VERSION=20 -ENV WASI_SDK_PATH=/opt/wasi-sdk - -ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=9.0.1 -ENV WASMTIME_CPU_ARCH=x86_64 - -RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ - dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ - dnf -y clean all - -RUN mkdir ${WASI_SDK_PATH} && \ - curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-linux.tar.gz | \ - tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip - -RUN mkdir --parents ${WASMTIME_HOME} && \ - curl --location "https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-${WASMTIME_CPU_ARCH}-linux.tar.xz" | \ - xz --decompress | \ - tar --strip-components 1 --directory ${WASMTIME_HOME} -x && \ - ln -s ${WASMTIME_HOME}/wasmtime /usr/local/bin diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0dc303015df5c7..fd1d7151518f40 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,5 @@ { - "build": { - "dockerfile": "Dockerfile" - }, + "image": "ghcr.io/python/devcontainer:2025.05.25.15232270922", "onCreateCommand": [ // Install common tooling. "dnf", diff --git a/.editorconfig b/.editorconfig index 81445d2d79c739..5b04b32a89e3d2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,6 +1,6 @@ root = true -[*.{py,c,cpp,h,rst,md,yml}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space @@ -8,5 +8,8 @@ indent_style = space [*.{py,c,cpp,h}] indent_size = 4 -[*.yml] +[*.rst] +indent_size = 3 + +[*.{js,yml,yaml}] indent_size = 2 diff --git a/.gitattributes b/.gitattributes index 1f2f8a4cd8735a..5682b9150a36e2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.ico binary *.jpg binary *.pck binary +*.pdf binary *.png binary *.psd binary *.tar binary @@ -24,11 +25,9 @@ PC/classicAppCompat.* binary [attr]noeol -text Lib/test/cjkencodings/* noeol -Lib/test/coding20731.py noeol +Lib/test/tokenizedata/coding20731.py noeol Lib/test/decimaltestdata/*.decTest noeol Lib/test/test_email/data/*.txt noeol -Lib/test/test_importlib/resources/data01/* noeol -Lib/test/test_importlib/resources/namespacedata01/* noeol Lib/test/xmltestdata/* noeol # Shell scripts should have LF even on Windows because of Cygwin @@ -66,7 +65,10 @@ PCbuild/readme.txt dos [attr]generated linguist-generated=true diff=generated **/clinic/*.c.h generated +**/clinic/*.cpp.h generated +**/clinic/*.h.h generated *_db.h generated +Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated @@ -74,13 +76,19 @@ Include/internal/pycore_ast_state.h generated Include/internal/pycore_opcode.h generated Include/internal/pycore_opcode_metadata.h generated Include/internal/pycore_*_generated.h generated +Include/internal/pycore_uop_ids.h generated +Include/internal/pycore_uop_metadata.h generated Include/opcode.h generated +Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated Lib/keyword.py generated +Lib/test/certdata/*.pem generated +Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated Lib/test/test_stable_abi_ctypes.py generated Lib/token.py generated +Misc/sbom.spdx.json generated Objects/typeslots.inc generated PC/python3dll.c generated Parser/parser.c generated @@ -89,6 +97,7 @@ Programs/test_frozenmain.h generated Python/Python-ast.c generated Python/executor_cases.c.h generated Python/generated_cases.c.h generated +Python/optimizer_cases.c.h generated Python/opcode_targets.h generated Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 578cd71a7bd211..775d9c63260c83 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,58 +5,124 @@ # https://git-scm.com/docs/gitignore#_pattern_format # GitHub -.github/** @ezio-melotti @hugovk +.github/** @ezio-melotti @hugovk @AA-Turner # pre-commit .pre-commit-config.yaml @hugovk @AlexWaygood +.ruff.toml @hugovk @AlexWaygood @AA-Turner # Build system configure* @erlend-aasland @corona10 +Makefile.pre.in @erlend-aasland +Modules/Setup* @erlend-aasland + +# argparse +**/*argparse* @savannahostrowski # asyncio -**/*asyncio* @1st1 @asvetlov @gvanrossum @kumaraditya303 @willingc +**/*asyncio* @1st1 @asvetlov @kumaraditya303 @willingc # Core **/*context* @1st1 **/*genobject* @markshannon **/*hamt* @1st1 +**/*jit* @brandtbucher @savannahostrowski Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra +Objects/unionobject.c @JelleZijlstra Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval.c @markshannon +Python/ceval*.c @markshannon +Python/ceval*.h @markshannon +Python/codegen.c @markshannon @iritkatriel Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel -Python/ast_opt.c @isidentical +Python/instruction_sequence.c @iritkatriel +Python/bytecodes.c @markshannon +Python/optimizer*.c @markshannon +Python/optimizer_analysis.c @Fidget-Spinner +Python/optimizer_bytecodes.c @Fidget-Spinner +Python/symtable.c @JelleZijlstra @carljm +Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher -Lib/test/test_peepholer.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra +Lib/test/test_capi/test_misc.py @markshannon +Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv +Tools/c-analyzer/ @ericsnowcurrently + +# dbm +**/*dbm* @corona10 @erlend-aasland @serhiy-storchaka + +# Doc/ tools +Doc/conf.py @AA-Turner @hugovk +Doc/Makefile @AA-Turner @hugovk +Doc/make.bat @AA-Turner @hugovk +Doc/requirements.txt @AA-Turner @hugovk +Doc/_static/** @AA-Turner @hugovk +Doc/tools/** @AA-Turner @hugovk + +# runtime state/lifecycle +**/*pylifecycle* @ericsnowcurrently +**/*pystate* @ericsnowcurrently +**/*preconfig* @ericsnowcurrently +**/*initconfig* @ericsnowcurrently +**/*pathconfig* @ericsnowcurrently +**/*sysmodule* @ericsnowcurrently +**/*bltinmodule* @ericsnowcurrently +**/*gil* @ericsnowcurrently +Include/internal/pycore_runtime.h @ericsnowcurrently +Include/internal/pycore_interp.h @ericsnowcurrently +Include/internal/pycore_tstate.h @ericsnowcurrently +Include/internal/pycore_*_state.h @ericsnowcurrently +Include/internal/pycore_*_init.h @ericsnowcurrently +Include/internal/pycore_atexit.h @ericsnowcurrently +Include/internal/pycore_freelist.h @ericsnowcurrently +Include/internal/pycore_global_objects.h @ericsnowcurrently +Include/internal/pycore_obmalloc.h @ericsnowcurrently +Include/internal/pycore_pymem.h @ericsnowcurrently +Include/internal/pycore_stackref.h @Fidget-Spinner +Modules/main.c @ericsnowcurrently +Programs/_bootstrap_python.c @ericsnowcurrently +Programs/python.c @ericsnowcurrently +Tools/build/generate_global_objects.py @ericsnowcurrently + +# Initialization +Doc/library/sys_path_init.rst @FFY00 +Doc/c-api/init_config.rst @FFY00 + +# getpath +**/*getpath* @FFY00 + +# site +**/*site.py @FFY00 +Doc/library/site.rst @FFY00 # Exceptions -Lib/traceback.py @iritkatriel Lib/test/test_except*.py @iritkatriel -Lib/test/test_traceback.py @iritkatriel Objects/exceptions.c @iritkatriel -Python/traceback.c @iritkatriel -# Hashing -**/*hashlib* @tiran -**/*pyhash* @tiran -**/*sha* @tiran -**/*md5* @tiran -**/*blake* @tiran -/Modules/_blake2/** @tiran -/Modules/_sha3/** @tiran +# Hashing & cryptographic primitives +**/*hashlib* @gpshead @tiran @picnixz +**/*hashopenssl* @gpshead @tiran @picnixz +**/*pyhash* @gpshead @tiran @picnixz +Modules/*blake* @gpshead @tiran @picnixz +Modules/*md5* @gpshead @tiran @picnixz +Modules/*sha* @gpshead @tiran @picnixz +Modules/_hacl/** @gpshead @picnixz +**/*hmac* @gpshead @picnixz + +# libssl +**/*ssl* @gpshead @picnixz # logging **/*logging* @vsajip # venv -**/*venv* @vsajip +**/*venv* @vsajip @FFY00 # Launcher /PC/launcher.c @vsajip @@ -70,8 +136,21 @@ Python/traceback.c @iritkatriel # Import (including importlib). **/*import* @brettcannon @ericsnowcurrently @ncoghlan @warsaw /Python/import.c @kumaraditya303 -**/*importlib/resources/* @jaraco @warsaw @FFY00 -**/importlib/metadata/* @jaraco @warsaw +Python/dynload_*.c @ericsnowcurrently +**/*freeze* @ericsnowcurrently +**/*frozen* @ericsnowcurrently +**/*modsupport* @ericsnowcurrently +**/*modulefinder* @ericsnowcurrently +**/*moduleobject* @ericsnowcurrently +**/*multiphase* @ericsnowcurrently +**/*pkgutil* @ericsnowcurrently +**/*pythonrun* @ericsnowcurrently +**/*runpy* @ericsnowcurrently +**/*singlephase* @ericsnowcurrently +Lib/test/test_module/ @ericsnowcurrently +Doc/c-api/module.rst @ericsnowcurrently +**/*importlib/resources/* @jaraco @warsaw @FFY00 +**/*importlib/metadata/* @jaraco @warsaw # Dates and times **/*datetime* @pganssle @abalkin @@ -89,6 +168,9 @@ Include/internal/pycore_time.h @pganssle @abalkin **/*imap* @python/email-team **/*poplib* @python/email-team +# Exclude .mailmap from being owned by @python/email-team +/.mailmap + # Garbage collector /Modules/gcmodule.c @pablogsal /Doc/library/gc.rst @pablogsal @@ -101,16 +183,25 @@ Include/internal/pycore_time.h @pganssle @abalkin /Lib/tokenize.py @pablogsal @lysnikolaou /Lib/test/test_tokenize.py @pablogsal @lysnikolaou +# Code generator +/Tools/cases_generator/ @markshannon + # AST -Python/ast.c @isidentical -Parser/asdl.py @isidentical -Parser/asdl_c.py @isidentical -Lib/ast.py @isidentical +Python/ast.c @isidentical @JelleZijlstra @eclips4 +Python/ast_preprocess.c @isidentical @eclips4 +Parser/asdl.py @isidentical @JelleZijlstra @eclips4 +Parser/asdl_c.py @isidentical @JelleZijlstra @eclips4 +Lib/ast.py @isidentical @JelleZijlstra @eclips4 +Lib/_ast_unparse.py @isidentical @JelleZijlstra @eclips4 +Lib/test/test_ast/ @eclips4 # Mock /Lib/unittest/mock.py @cjw296 /Lib/test/test_unittest/testmock/* @cjw296 +# multiprocessing +**/*multiprocessing* @gpshead + # SQLite 3 **/*sqlite* @berkerpeksag @erlend-aasland @@ -119,6 +210,10 @@ Lib/ast.py @isidentical /Lib/test/test_subprocess.py @gpshead /Modules/*subprocess* @gpshead +# debugger +**/*pdb* @gaogaotiantian +**/*bdb* @gaogaotiantian + # Limited C API & stable ABI Tools/build/stable_abi.py @encukou Misc/stable_abi.toml @encukou @@ -140,17 +235,20 @@ Doc/c-api/stable.rst @encukou **/*itertools* @rhettinger **/*collections* @rhettinger **/*random* @rhettinger -**/*queue* @rhettinger **/*bisect* @rhettinger **/*heapq* @rhettinger **/*functools* @rhettinger -**/*decimal* @rhettinger **/*dataclasses* @ericvsmith +**/*ensurepip* @pfmoore @pradyunsg + +/Doc/library/idle.rst @terryjreedy **/*idlelib* @terryjreedy +**/*turtledemo* @terryjreedy -**/*typing* @gvanrossum @Fidget-Spinner @JelleZijlstra @AlexWaygood +**/*annotationlib* @JelleZijlstra +**/*typing* @JelleZijlstra @AlexWaygood **/*ftplib @giampaolo **/*shutil @giampaolo @@ -176,6 +274,65 @@ Doc/c-api/stable.rst @encukou **/*zipfile/_path/* @jaraco # Argument Clinic -/Tools/clinic/** @erlend-aasland @AlexWaygood -/Lib/test/test_clinic.py @erlend-aasland @AlexWaygood +/Tools/clinic/** @erlend-aasland +/Lib/test/test_clinic.py @erlend-aasland Doc/howto/clinic.rst @erlend-aasland + +# Subinterpreters +**/*interpreteridobject.* @ericsnowcurrently +**/*crossinterp* @ericsnowcurrently +Lib/test/support/interpreters/ @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently +Lib/test/test_interpreters/ @ericsnowcurrently + +# Android +**/*Android* @mhsmith @freakboy3742 +**/*android* @mhsmith @freakboy3742 + +# iOS (but not termios) +**/iOS* @freakboy3742 +**/ios* @freakboy3742 +**/*_iOS* @freakboy3742 +**/*_ios* @freakboy3742 +**/*-iOS* @freakboy3742 +**/*-ios* @freakboy3742 + +# WebAssembly +Tools/wasm/config.site-wasm32-emscripten @freakboy3742 +/Tools/wasm/README.md @brettcannon @freakboy3742 +/Tools/wasm/wasi-env @brettcannon +/Tools/wasm/wasi.py @brettcannon +/Tools/wasm/emscripten @freakboy3742 +/Tools/wasm/wasi @brettcannon + +# SBOM +/Misc/externals.spdx.json @sethmlarson +/Misc/sbom.spdx.json @sethmlarson +/Tools/build/generate_sbom.py @sethmlarson + +# Config Parser +Lib/configparser.py @jaraco +Lib/test/test_configparser.py @jaraco + +# Doc sections +Doc/reference/ @willingc @AA-Turner + +**/*weakref* @kumaraditya303 + +# Colorize +Lib/_colorize.py @hugovk +Lib/test/test__colorize.py @hugovk + +# Fuzzing +Modules/_xxtestfuzz/ @ammaraskar + +# t-strings +**/*interpolationobject* @lysnikolaou +**/*templateobject* @lysnikolaou +**/*templatelib* @lysnikolaou +**/*tstring* @lysnikolaou + +# Remote debugging +Python/remote_debug.h @pablogsal +Python/remote_debugging.c @pablogsal +Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 diff --git a/.github/ISSUE_TEMPLATE/bug.md b/.github/ISSUE_TEMPLATE/bug.md deleted file mode 100644 index 47037cd319e7e2..00000000000000 --- a/.github/ISSUE_TEMPLATE/bug.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Bug report -about: Submit a bug report -labels: "type-bug" ---- - -<!-- - New to Python? The issue tracker isn't the right place to get help. - Consider instead: - - - reading the Python tutorial: https://docs.python.org/3/tutorial/ - - posting at https://discuss.python.org/c/users/7 - - emailing https://mail.python.org/mailman/listinfo/python-list ---> - -# Bug report - -## Checklist - -<!-- Bugs in third-party projects (e.g. `requests`) do not belong in the CPython issue tracker --> - -- [ ] I am confident this is a bug in CPython, not a bug in a third-party project -- [ ] I have searched the CPython issue tracker, and am confident this bug has not been reported before - -## A clear and concise description of the bug - -<!-- - Include a minimal, reproducible example if possible. - (https://stackoverflow.com/help/minimal-reproducible-example) - - Put any code blocks inside triple backticks: - - ```py - your code here - ``` - ---> - - - -# Your environment - -<!-- Include all relevant details about the environment you experienced the bug in --> - -- CPython versions tested on: -- Operating system and architecture: - -<!-- -You can freely edit this form. Remove any lines you believe are unnecessary. ---> diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 00000000000000..da70710b7ecfa3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,57 @@ +name: Bug report +description: Submit a bug report +labels: ["type-bug"] +body: + - type: markdown + attributes: + value: | + **New to Python?** + + For help or advice on using Python, try one of the following options instead of opening a GitHub issue: + + - Asking on [Discourse](https://discuss.python.org/c/users/7) or [Stack Overflow](https://stackoverflow.com) + - Reading the [Python tutorial](https://docs.python.org/3/tutorial/) + - Emailing [python-list](https://mail.python.org/mailman/listinfo/python-list) + + Make sure to also search the [CPython issue tracker](https://github.com/python/cpython/issues?q=is%3Aissue+sort%3Acreated-desc) to check that the bug has not already been reported. + - type: textarea + attributes: + label: "Bug description:" + description: > + Give a clear and concise description of what happened. + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), + and put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + - "3.15" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/crash.md b/.github/ISSUE_TEMPLATE/crash.md deleted file mode 100644 index a268249d1c1e65..00000000000000 --- a/.github/ISSUE_TEMPLATE/crash.md +++ /dev/null @@ -1,48 +0,0 @@ ---- -name: Crash report -about: A hard crash of the interpreter, possibly with a core dump -labels: "type-crash" ---- - -<!-- - This form is for hard crashes of the Python interpreter, segmentation faults, - failed C-level assertions, and similar. - Exceptions unexpectedly raised from stdlib Python functions - count as bugs rather than crashes. - - The CPython interpreter is written in a different programming language, C. - A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. ---> - -# Crash report - -<!-- - Tell us what happened. Ideally, include a minimal, reproducible example. - (https://stackoverflow.com/help/minimal-reproducible-example) - - Put any code blocks inside triple backticks: - - ```py - your code here - ``` - ---> - - - -# Error messages - -<!-- Enter any error messages caused by the crash, including a core dump if there is one --> - - - -# Your environment - -<!-- Include all relevant details about the environment you experienced the crash in --> - -- CPython versions tested on: -- Operating system and architecture: - -<!-- -You can freely edit this form. Remove any lines you believe are unnecessary. ---> diff --git a/.github/ISSUE_TEMPLATE/crash.yml b/.github/ISSUE_TEMPLATE/crash.yml new file mode 100644 index 00000000000000..470ad581367b10 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/crash.yml @@ -0,0 +1,56 @@ +name: Crash report +description: A hard crash of the interpreter, possibly with a core dump +labels: ["type-crash"] +body: + - type: markdown + attributes: + value: | + This form is for hard crashes of the Python interpreter, segmentation faults, failed C-level assertions, and similar. Unexpected exceptions raised from Python functions in the standard library count as bugs rather than crashes. + + The CPython interpreter is written in a different programming language, C. A "CPython crash" is when Python itself fails, leading to a traceback in the C stack. + - type: textarea + attributes: + label: What happened? + description: > + Include a [minimal, reproducible example](https://stackoverflow.com/help/minimal-reproducible-example) if possible. + [Copy and paste code where possible rather than using screenshots](https://meta.stackoverflow.com/a/285557/13990016), + and put any code blocks inside triple backticks. + + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: dropdown + attributes: + label: "CPython versions tested on:" + multiple: true + options: + - "3.9" + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + - "3.15" + - "CPython main branch" + validations: + required: true + - type: dropdown + attributes: + label: "Operating systems tested on:" + multiple: true + options: + - Linux + - macOS + - Windows + - Other + validations: + required: false + - type: input + attributes: + label: "Output from running 'python -VV' on the command line:" + description: If you tested with multiple operating systems or architectures, feel free to provide details in the main bug description. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature.md b/.github/ISSUE_TEMPLATE/feature.md deleted file mode 100644 index 7e96bc9df665c2..00000000000000 --- a/.github/ISSUE_TEMPLATE/feature.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -name: Feature or enhancement -about: Submit a proposal for a new CPython feature or enhancement -labels: "type-feature" ---- - -<!-- - Proposing a new feature for Python? - You'll need to demonstrate widespread support for your idea among the community. - - Major feature proposals should generally be discussed at - https://discuss.python.org/c/ideas/6 before opening a GitHub issue. - Wait until it's clear that most people support your idea - before filling in this form. ---> - -# Feature or enhancement - -<!-- A clear and concise description of your proposal. --> - - - -# Pitch - -<!-- - Explain why this feature or enhancement should be implemented and how it would be used. - Add examples, if applicable. - - Put any code blocks inside triple backticks: - - ```py - your code here - ``` - ---> - - - -# Previous discussion - -<!-- - Use this space to post links to the places - where you have already discussed your feature proposal: ---> - - - -<!-- -You can freely edit this form. Remove any lines you believe are unnecessary. ---> diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 00000000000000..4361ab2bf827fb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,40 @@ +name: Feature or enhancement +description: Submit a proposal for a new CPython feature or enhancement +labels: ["type-feature"] +body: + - type: markdown + attributes: + value: | + # Proposing a feature to CPython? + + You'll need to demonstrate widespread support for your idea among the community. + + Major feature proposals should generally be discussed on [Discourse](https://discuss.python.org/c/ideas/6) before opening a GitHub issue. Wait until it's clear that most people support your idea before filling in this form. + - type: textarea + attributes: + label: "Proposal:" + description: > + Explain your proposal, why it should be implemented, and how it would be used. + Add examples, if applicable. + Put any code blocks inside triple backticks. + value: | + ```python + # Add a code block here, if required + ``` + validations: + required: true + - type: dropdown + attributes: + label: Has this already been discussed elsewhere? + options: + - No response given + - I have already discussed this feature proposal on Discourse + - This is a minor feature, which does not need previous discussion elsewhere + multiple: false + validations: + required: true + - type: textarea + attributes: + label: "Links to previous discussion of this feature:" + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4cc2f461dbe2f2..03ea959ca14e28 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,10 +7,10 @@ Please read this comment in its entirety. It's quite important. It should be in the following format: ``` -gh-NNNNN: Summary of the changes made +gh-NNNNNN: Summary of the changes made ``` -Where: gh-NNNNN refers to the GitHub issue number. +Where: gh-NNNNNN refers to the GitHub issue number. Most PRs will require an issue number. Trivial changes, like fixing a typo, do not need an issue. @@ -20,11 +20,11 @@ If this is a backport PR (PR made against branches other than `main`), please ensure that the PR title is in the following format: ``` -[X.Y] <title from the original PR> (GH-NNNN) +[X.Y] <title from the original PR> (GH-NNNNNN) ``` -Where: [X.Y] is the branch name, e.g. [3.6]. +Where: [X.Y] is the branch name, for example: [3.13]. -GH-NNNN refers to the PR number from `main`. +GH-NNNNNN refers to the PR number from `main`. --> diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000000000..68aae196357414 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,11 @@ +self-hosted-runner: + # Pending https://github.com/rhysd/actionlint/issues/533 + labels: ["windows-11-arm"] + +config-variables: null + +paths: + .github/workflows/**/*.yml: + ignore: + - 1st argument of function call is not assignable + - SC2(015|038|086|091|097|098|129|155) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f026b0f5f9454a..c8a3165d690364 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,7 +13,7 @@ updates: - "version-update:semver-minor" - "version-update:semver-patch" - package-ecosystem: "pip" - directory: "/Tools/clinic/" + directory: "/Tools/" schedule: interval: "monthly" labels: diff --git a/.github/problem-matchers/sphinx.json b/.github/problem-matchers/sphinx.json deleted file mode 100644 index 09848608a7b034..00000000000000 --- a/.github/problem-matchers/sphinx.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "problemMatcher": [ - { - "owner": "sphinx-problem-matcher", - "pattern": [ - { - "regexp": "^(.*):(\\d+):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "line": 2, - "severity": 3, - "message": 4 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose", - "pattern": [ - { - "_comment": "A bit of a looser pattern, doesn't look for line numbers, just looks for file names relying on them to start with / and end with .rst", - "regexp": "(\/.*\\.rst):\\s+(\\w*):\\s+(.*)$", - "file": 1, - "severity": 2, - "message": 3 - } - ] - }, - { - "owner": "sphinx-problem-matcher-loose-no-severity", - "pattern": [ - { - "_comment": "Looks for file names ending with .rst and line numbers but without severity", - "regexp": "^(.*\\.rst):(\\d+):(.*)$", - "file": 1, - "line": 2, - "message": 3 - } - ] - } - ] -} \ No newline at end of file diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml new file mode 100644 index 00000000000000..3cbc23af578d10 --- /dev/null +++ b/.github/workflows/add-issue-header.yml @@ -0,0 +1,54 @@ +name: Add issue header +# Automatically edits an issue's descriptions with a header, +# one of: +# +# - Bug report +# - Crash report +# - Feature or enhancement + +on: + issues: + types: + # Only ever run once + - opened + + +jobs: + add-header: + runs-on: ubuntu-latest + permissions: + issues: write + timeout-minutes: 5 + steps: + - uses: actions/github-script@v7 + with: + # language=JavaScript + script: | + // https://devguide.python.org/triage/labels/#type-labels + const HEADERS = new Map([ + ['type-bug', 'Bug report'], + ['type-crash', 'Crash report'], + ['type-feature', 'Feature or enhancement'], + ]); + let issue_data = await github.rest.issues.get({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo + }).then(issue => issue.data); + let header = ''; + for (const label_data of issue_data.labels) { + const label_name = (typeof label_data === 'string') ? label_data : label_data.name; + if (HEADERS.has(label_name)) { + header = HEADERS.get(label_name); + break; + } + } + if (header !== '') { + console.log(`Setting new header: ${header}`); + await github.rest.issues.update({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `# ${header}\n\n${issue_data.body.replaceAll('\r', '')}` + }); + } diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 06551b13219c2a..54ebc914b46821 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,148 +1,136 @@ name: Tests -# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because -# it prevents to mark a job as mandatory. A PR cannot be merged if a job is -# mandatory but not scheduled because of "paths-ignore". on: workflow_dispatch: push: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' + - '3.*' pull_request: branches: - 'main' - - '3.12' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' + - '3.*' permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency + # 'group' must be a key uniquely representing a PR or push event. + # github.workflow is the workflow name + # github.actor is the user invoking the workflow + # github.head_ref is the source branch of the PR or otherwise blank + # github.run_id is a unique number for the current run + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -jobs: - check_source: - name: 'Check for source changes' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} - run_tests: ${{ steps.check.outputs.run_tests }} - run_hypothesis: ${{ steps.check.outputs.run_hypothesis }} - config_hash: ${{ steps.config_hash.outputs.hash }} - steps: - - uses: actions/checkout@v3 - - name: Check for source changes - id: check - run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run_tests=true" >> $GITHUB_OUTPUT - else - git fetch origin $GITHUB_BASE_REF --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true - fi +env: + FORCE_COLOR: 1 - # Check if we should run hypothesis tests - GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo $GIT_BRANCH - if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then - echo "Branch too old for hypothesis tests" - echo "run_hypothesis=false" >> $GITHUB_OUTPUT - else - echo "Run hypothesis tests" - echo "run_hypothesis=true" >> $GITHUB_OUTPUT - fi - - name: Compute hash for config cache key - id: config_hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - if: >- - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" +jobs: + build-context: + name: Change detection + # To use boolean outputs from this job, parse them as JSON. + # Here's some examples: + # + # if: fromJSON(needs.build-context.outputs.run-docs) + # + # ${{ + # fromJSON(needs.build-context.outputs.run-tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + uses: ./.github/workflows/reusable-context.yml check-docs: name: Docs - needs: check_source - if: fromJSON(needs.check_source.outputs.run-docs) + needs: build-context + if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml - check_generated_files: + check-autoconf-regen: + name: 'Check if Autoconf files are up to date' + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-24.04 + container: + image: ghcr.io/python/autoconf:2025.01.02.12581854023 + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - name: Install Git + run: | + apt update && apt install git -yq + git config --global --add safe.directory "$GITHUB_WORKSPACE" + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + persist-credentials: false + - name: Check Autoconf and aclocal versions + run: | + grep "Generated by GNU Autoconf 2.72" configure + grep "aclocal 1.16.5" aclocal.m4 + grep -q "runstatedir" configure + grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + - name: Regenerate autoconf files + # Same command used by Tools/build/regen-configure.sh ($AUTORECONF) + run: autoreconf -ivf -Werror + - name: Check for changes + run: | + git add -u + changes=$(git status --porcelain) + # Check for changes in regenerated files + if test -n "$changes"; then + echo "Generated files not up to date." + echo "Perhaps you forgot to run make regen-configure ;)" + echo "configure files must be regenerated with a specific version of autoconf." + echo "$changes" + echo "" + git diff --staged || true + exit 1 + fi + + check-generated-files: name: 'Check if generated files are up to date' - runs-on: ubuntu-latest + # Don't use ubuntu-latest but a specific version to make the job + # reproducible: to get the same tools versions (autoconf, aclocal, ...) + runs-on: ubuntu-24.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - uses: actions/setup-python@v3 - - name: Install Dependencies + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - - name: Check Autoconf and aclocal versions - run: | - grep "Generated by GNU Autoconf 2.71" configure - grep "aclocal 1.16.4" aclocal.m4 - grep -q "runstatedir" configure - grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 + with: + save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files with container image - run: make regen-configure - name: Build CPython run: | - # Deepfreeze will usually cause global objects to be added or removed, - # so we run it before regen-global-objects gets rum (in regen-all). - make regen-deepfreeze make -j4 regen-all - make regen-stdlib-module-names + make regen-stdlib-module-names regen-sbom regen-unicodedata - name: Check for changes run: | git add -u @@ -165,192 +153,168 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals - build_win32: - name: 'Windows (x86)' - runs-on: windows-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v3 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + build-windows: + name: >- + Windows + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-tests) + strategy: + fail-fast: false + matrix: + arch: + - x64 + - Win32 + - arm64 + free-threading: + - false + - true + exclude: + # Skip Win32 on free-threaded builds + - { arch: Win32, free-threading: true } + uses: ./.github/workflows/reusable-windows.yml + with: + arch: ${{ matrix.arch }} + free-threading: ${{ matrix.free-threading }} - build_win_amd64: - name: 'Windows (x64)' - runs-on: windows-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v3 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + build-windows-msi: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Windows MSI${{ '' }} + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-msi) + strategy: + fail-fast: false + matrix: + arch: + - x86 + - x64 + - arm64 + uses: ./.github/workflows/reusable-windows-msi.yml + with: + arch: ${{ matrix.arch }} - build_macos: - name: 'macOS' - runs-on: macos-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - HOMEBREW_NO_ANALYTICS: 1 - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v3 - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - - name: Configure CPython - run: | - GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ - GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ - ./configure \ - --config-cache \ - --with-pydebug \ - --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: make buildbottest TESTOPTS="-j4 -uall,-cpu" + build-macos: + name: >- + macOS + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. + # macOS 13 only runs tests against the GIL-enabled CPython. + # Cirrus used for upstream, macos-14 for forks. + os: + - ghcr.io/cirruslabs/macos-runner:sonoma + - macos-14 + - macos-13 + is-fork: # only used for the exclusion trick + - ${{ github.repository_owner != 'python' }} + free-threading: + - false + - true + exclude: + - os: ghcr.io/cirruslabs/macos-runner:sonoma + is-fork: true + - os: macos-14 + is-fork: false + - os: macos-13 + free-threading: true + uses: ./.github/workflows/reusable-macos.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} - build_ubuntu: - name: 'Ubuntu' - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - OPENSSL_VER: 1.1.1u - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v3 - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + build-ubuntu: + name: >- + Ubuntu + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + bolt: + - false + - true + free-threading: + - false + - true + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + exclude: + # Do not test BOLT with free-threading, to conserve resources + - bolt: true + free-threading: true + # BOLT currently crashes during instrumentation on aarch64 + - os: ubuntu-24.04-arm + bolt: true + uses: ./.github/workflows/reusable-ubuntu.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + bolt-optimizations: ${{ matrix.bolt }} + free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} - build_ubuntu_ssltests: + build-ubuntu-ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' strategy: fail-fast: false matrix: - openssl_ver: [1.1.1u, 3.0.9, 3.1.1] + os: [ubuntu-24.04] + openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] + # See Tools/ssl/make_ssl_data.py for notes on adding a new version env: OPENSSL_VER: ${{ matrix.openssl_ver }} MULTISSL_DIR: ${{ github.workspace }}/multissl OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Configure CPython - run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" - name: Build CPython run: make -j4 - name: Display build info @@ -358,60 +322,75 @@ jobs: - name: SSL tests run: ./python Lib/test/ssltests.py - test_hypothesis: + build-wasi: + name: 'WASI' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + uses: ./.github/workflows/reusable-wasi.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + + test-hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 3.0.16 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: false - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | ../cpython-ro-srcdir/configure \ --config-cache \ --with-pydebug \ - --with-openssl=$OPENSSL_DIR + --enable-slower-safety \ + --with-openssl="$OPENSSL_DIR" - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -420,26 +399,26 @@ jobs: run: make pythoninfo - name: Remount sources writable for tests # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw - name: Setup directory envs for out-of-tree builds run: | - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" - name: "Create hypothesis venv" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | VENV_LOC=$(realpath -m .)/hypovenv VENV_PYTHON=$VENV_LOC/bin/python - echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV - echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV - ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis + echo "HYPOVENV=${VENV_LOC}" >> "$GITHUB_ENV" + echo "VENV_PYTHON=${VENV_PYTHON}" >> "$GITHUB_ENV" + ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v3 + uses: actions/cache@v4 with: - path: ./hypothesis + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} restore-keys: | - - hypothesis-database- + hypothesis-database- - name: "Run tests" working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -447,10 +426,11 @@ jobs: # # (GH-104097) test_sysconfig is skipped because it has tests that are # failing when executed from inside a virtual environment. - ${{ env.VENV_PYTHON }} -m test \ + "${VENV_PYTHON}" -m test \ -W \ - -o \ + --slowest \ -j4 \ + --timeout 900 \ -x test_asyncio \ -x test_multiprocessing_fork \ -x test_multiprocessing_forkserver \ @@ -460,33 +440,40 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 if: always() with: name: hypothesis-example-db - path: .hypothesis/examples/ - + path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/examples/ - build_asan: + build-asan: name: 'Address sanitizer' - runs-on: ubuntu-20.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04] env: - OPENSSL_VER: 1.1.1u + OPENSSL_VER: 3.0.16 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies + - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Set up GCC-10 for ASAN uses: egor-tensin/setup-gcc@v1 @@ -494,23 +481,26 @@ jobs: version: 10 - name: Configure OpenSSL env vars run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - name: Add ccache to PATH run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -518,60 +508,171 @@ jobs: - name: Display build info run: make pythoninfo - name: Tests - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + run: xvfb-run make ci + + build-tsan: + name: >- + Thread sanitizer + ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + strategy: + fail-fast: false + matrix: + free-threading: + - false + - true + uses: ./.github/workflows/reusable-tsan.yml + with: + config_hash: ${{ needs.build-context.outputs.config-hash }} + free-threading: ${{ matrix.free-threading }} + + cross-build-linux: + name: Cross build Linux + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Set build dir + run: + # an absolute path outside of the working directoy + echo "BUILD_DIR=$(realpath ${{ github.workspace }}/../build)" >> "$GITHUB_ENV" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure host build + run: ./configure --prefix="$BUILD_DIR/host-python" + - name: Install host Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/host-python/bin/python3" -m test test_sysconfig test_site test_embed + - name: Configure cross build + run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" + - name: Install cross Python + run: make -j8 install + - name: Run test subset with host build + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed + + # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ + cifuzz: + name: CIFuzz + runs-on: ubuntu-latest + timeout-minutes: 60 + needs: build-context + if: needs.build-context.outputs.run-ci-fuzz == 'true' + permissions: + security-events: write + strategy: + fail-fast: false + matrix: + sanitizer: [address, undefined, memory] + steps: + - name: Build fuzzers (${{ matrix.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: cpython3 + sanitizer: ${{ matrix.sanitizer }} + - name: Run fuzzers (${{ matrix.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: cpython3 + output-sarif: true + sanitizer: ${{ matrix.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass - if: always() - + runs-on: ubuntu-latest + timeout-minutes: 5 needs: - - check_source # Transitive dependency, needed to access `run_tests` value + - build-context # Transitive dependency, needed to access `run-tests` value - check-docs - - check_generated_files - - build_win32 - - build_win_amd64 - - build_macos - - build_ubuntu - - build_ubuntu_ssltests - - test_hypothesis - - build_asan - - runs-on: ubuntu-latest + - check-autoconf-regen + - check-generated-files + - build-windows + - build-windows-msi + - build-macos + - build-ubuntu + - build-ubuntu-ssltests + - build-wasi + - test-hypothesis + - build-asan + - build-tsan + - cross-build-linux + - cifuzz + if: always() steps: - name: Check whether the needed jobs succeeded or failed uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- - build_macos, - build_ubuntu_ssltests, - build_win32, - test_hypothesis, + build-windows-msi, + build-ubuntu-ssltests, + test-hypothesis, + cifuzz, allowed-skips: >- ${{ - !fromJSON(needs.check_source.outputs.run-docs) + !fromJSON(needs.build-context.outputs.run-docs) && ' check-docs, ' || '' }} ${{ - needs.check_source.outputs.run_tests != 'true' + needs.build-context.outputs.run-tests != 'true' + && ' + check-autoconf-regen, + check-generated-files, + build-macos, + build-ubuntu, + build-ubuntu-ssltests, + build-wasi, + test-hypothesis, + build-asan, + build-tsan, + cross-build-linux, + ' + || '' + }} + ${{ + !fromJSON(needs.build-context.outputs.run-windows-tests) && ' - check_generated_files, - build_win32, - build_win_amd64, - build_macos, - build_ubuntu, - build_ubuntu_ssltests, - build_asan, + build-windows, ' || '' }} ${{ - !fromJSON(needs.check_source.outputs.run_hypothesis) + !fromJSON(needs.build-context.outputs.run-ci-fuzz) && ' - test_hypothesis, + cifuzz, ' || '' }} diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml deleted file mode 100644 index 22f613a88aa11e..00000000000000 --- a/.github/workflows/build_msi.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: TestsMSI - -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - pull_request: - branches: - - 'main' - - '3.*' - paths: - - 'Tools/msi/**' - - '.github/workflows/build_msi.yml' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - name: Windows Installer - runs-on: windows-latest - timeout-minutes: 60 - strategy: - matrix: - type: [x86, x64, arm64] - steps: - - uses: actions/checkout@v3 - - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ matrix.type }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 43a7afec73884e..a09a30587b35eb 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -10,9 +10,6 @@ on: - 'Doc/**' - '.github/workflows/doc.yml' -permissions: - pull-requests: write - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -20,6 +17,10 @@ concurrency: jobs: documentation-links: runs-on: ubuntu-latest + permissions: + pull-requests: write + timeout-minutes: 5 + steps: - uses: readthedocs/actions/preview@v1 with: diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml new file mode 100644 index 00000000000000..116e0c1e945e38 --- /dev/null +++ b/.github/workflows/jit.yml @@ -0,0 +1,155 @@ +name: JIT +on: + pull_request: + paths: + - '**jit**' + - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' + push: + paths: + - '**jit**' + - 'Python/bytecodes.c' + - 'Python/optimizer*.c' + - '!Python/perf_jit_trampoline.c' + - '!**/*.md' + - '!**/*.ini' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + jit: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + needs: interpreter + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: + - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc + - aarch64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + debug: + - true + - false + llvm: + - 19 + include: + - target: i686-pc-windows-msvc/msvc + architecture: Win32 + runner: windows-latest + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest + - target: aarch64-pc-windows-msvc/msvc + architecture: ARM64 + runner: windows-11-arm + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + # PCbuild downloads LLVM automatically: + - name: Windows + if: runner.os == 'Windows' + run: | + ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. + - name: macOS + if: runner.os == 'macOS' + run: | + brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Linux + if: runner.os == 'Linux' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # XXX: GH-133171 + # jit-with-disabled-gil: + # name: Free-Threaded (Debug) + # needs: interpreter + # runs-on: ubuntu-24.04 + # timeout-minutes: 90 + # strategy: + # fail-fast: false + # matrix: + # llvm: + # - 19 + # steps: + # - uses: actions/checkout@v4 + # with: + # persist-credentials: false + # - uses: actions/setup-python@v5 + # with: + # python-version: '3.11' + # - name: Build with JIT enabled and GIL disabled + # run: | + # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + # ./configure --enable-experimental-jit --with-pydebug --disable-gil + # make all --jobs 4 + # - name: Run tests + # run: | + # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4481ea80bfd936..d74ce8fcc256dc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,6 +5,10 @@ on: [push, pull_request, workflow_dispatch] permissions: contents: read +env: + FORCE_COLOR: 1 + RUFF_OUTPUT_FORMAT: github + concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -15,8 +19,10 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 with: python-version: "3.x" - - uses: pre-commit/action@v3.0.0 + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 1315bb5a966f01..95133c1338b682 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -7,8 +7,24 @@ on: - main pull_request: paths: - - "Tools/clinic/**" - ".github/workflows/mypy.yml" + - "Lib/_colorize.py" + - "Lib/_pyrepl/**" + - "Lib/test/libregrtest/**" + - "Lib/tomllib/**" + - "Misc/mypy/**" + - "Tools/build/compute-changes.py" + - "Tools/build/deepfreeze.py" + - "Tools/build/generate_sbom.py" + - "Tools/build/generate-build-details.py" + - "Tools/build/verify_ensurepip_wheels.py" + - "Tools/build/update_file.py" + - "Tools/build/umarshal.py" + - "Tools/cases_generator/**" + - "Tools/clinic/**" + - "Tools/jit/**" + - "Tools/peg_generator/**" + - "Tools/requirements-dev.txt" workflow_dispatch: permissions: @@ -25,15 +41,31 @@ concurrency: jobs: mypy: - name: Run mypy on Tools/clinic/ + name: Run mypy on ${{ matrix.target }} runs-on: ubuntu-latest timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + target: [ + "Lib/_pyrepl", + "Lib/test/libregrtest", + "Lib/tomllib", + "Tools/build", + "Tools/cases_generator", + "Tools/clinic", + "Tools/jit", + "Tools/peg_generator", + ] steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 with: - python-version: "3.x" + python-version: "3.13" cache: pip - cache-dependency-path: Tools/clinic/requirements-dev.txt - - run: pip install -r Tools/clinic/requirements-dev.txt - - run: mypy --config-file Tools/clinic/mypy.ini + cache-dependency-path: Tools/requirements-dev.txt + - run: pip install -r Tools/requirements-dev.txt + - run: python3 Misc/mypy/make_symlinks.py --symlink + - run: mypy --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 80514b4d2ca572..9f1a8a824e5f19 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -13,12 +13,12 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index a220896f2cd7be..d5538cd9367ec6 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -19,6 +19,7 @@ apt-get -yq install \ libssl-dev \ lzma \ lzma-dev \ + strace \ tk-dev \ uuid-dev \ xvfb \ diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 7574bfc208ff76..1d9d637ec848a6 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -15,6 +15,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 strategy: + fail-fast: false matrix: include: # if an issue has any of these labels, it will be added @@ -23,7 +24,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v0.1.0 + - uses: actions/add-to-project@v1.0.0 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 9327b43ae02710..7e534c58c798d1 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,19 +4,58 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: - issues: read - pull-requests: read - jobs: - label: - name: DO-NOT-MERGE / unresolved review + label-dnm: + name: DO-NOT-MERGE + if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + pull-requests: read timeout-minutes: 10 steps: - - uses: mheap/github-action-required-labels@v5 + - name: Check there's no DO-NOT-MERGE + uses: mheap/github-action-required-labels@v5 with: mode: exactly count: 0 - labels: "DO-NOT-MERGE, awaiting changes, awaiting change review" + labels: | + DO-NOT-MERGE + + label-reviews: + name: Unresolved review + if: github.repository_owner == 'python' + runs-on: ubuntu-latest + permissions: + pull-requests: read + timeout-minutes: 10 + + steps: + # Check that the PR is not awaiting changes from the author due to previous review. + - name: Check there's no required changes + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 0 + labels: | + awaiting changes + awaiting change review + - id: is-feature + name: Check whether this PR is a feature (contains a "type-feature" label) + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + type-feature + exit_type: success # don't fail the check if the PR is not a feature, just record the result + # In case of a feature PR, check for a complete review (contains an "awaiting merge" label). + - id: awaiting-merge + if: steps.is-feature.outputs.status == 'success' + name: Check for complete review + uses: mheap/github-action-required-labels@v5 + with: + mode: exactly + count: 1 + labels: | + awaiting merge diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml new file mode 100644 index 00000000000000..d2668ddcac1a3d --- /dev/null +++ b/.github/workflows/reusable-context.yml @@ -0,0 +1,107 @@ +name: Reusable build context + +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + # Every referenced step MUST always set its output variable, + # either via ``Tools/build/compute-changes.py`` or in this workflow file. + # Boolean outputs (generally prefixed ``run-``) can then later be used + # safely through the following idiom in job conditionals and other + # expressions. Here's some examples: + # + # if: fromJSON(needs.build-context.outputs.run-tests) + # + # ${{ + # fromJSON(needs.build-context.outputs.run-tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + config-hash: + description: Config hash value for use in cache keys + value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-docs: + description: Whether to build the docs + value: ${{ jobs.compute-changes.outputs.run-docs }} # bool + run-tests: + description: Whether to run the regular tests + value: ${{ jobs.compute-changes.outputs.run-tests }} # bool + run-windows-tests: + description: Whether to run the Windows tests + value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + run-windows-msi: + description: Whether to run the MSI installer smoke tests + value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool + run-ci-fuzz: + description: Whether to run the CIFuzz job + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + +jobs: + compute-changes: + name: Create context from changed files + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + config-hash: ${{ steps.config-hash.outputs.hash }} + run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-docs: ${{ steps.changes.outputs.run-docs }} + run-tests: ${{ steps.changes.outputs.run-tests }} + run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} + run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3" + + - run: >- + echo '${{ github.event_name }}' + + - uses: actions/checkout@v4 + with: + persist-credentials: false + ref: >- + ${{ + github.event_name == 'pull_request' + && github.event.pull_request.head.sha + || '' + }} + + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: Fetch commits to get branch diff + if: github.event_name == 'pull_request' + run: | + set -eux + + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: main): + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules + env: + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' + + # We only want to run tests on PRs when related files are changed, + # or when someone triggers a manual workflow run. + - name: Compute changed files + id: changes + run: python Tools/build/compute-changes.py + env: + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_EVENT_NAME: ${{ github.event_name }} + CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} + CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Compute hash for config cache key + id: config-hash + run: | + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 56932c4860573c..657e0a6bf662f7 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -1,4 +1,4 @@ -name: Docs +name: Reusable Docs on: workflow_call: @@ -11,17 +11,48 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + FORCE_COLOR: 1 + jobs: - build_doc: + build-doc: name: 'Docs' runs-on: ubuntu-latest timeout-minutes: 60 + env: + branch_base: 'origin/${{ github.event.pull_request.base.ref }}' + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - - uses: actions/checkout@v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" + - name: 'Check out latest PR branch commit' + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: >- + ${{ + github.event_name == 'pull_request' + && github.event.pull_request.head.sha + || '' + }} + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: 'Fetch commits to get branch diff' + if: github.event_name == 'pull_request' + run: | + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) + + # Get all commits since that commit date from the base branch (eg: master or main): + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3' cache: 'pip' @@ -30,55 +61,31 @@ jobs: run: make -C Doc/ venv # To annotate PRs with Sphinx nitpicks (missing references) - - name: 'Get list of changed files' - if: github.event_name == 'pull_request' - id: changed_files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: "Doc/**" - format: csv # works for paths with spaces - name: 'Build HTML documentation' continue-on-error: true run: | set -Eeuo pipefail - # Build docs with the '-n' (nit-picky) option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="-q -n -W --keep-going -w sphinx-warnings.txt" html + # Build docs with the nit-picky option; write warnings to file + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --check-and-annotate '${{ steps.changed_files.outputs.added_modified }}' \ + --annotate-diff "${branch_base}" "${branch_pr}" \ --fail-if-regression \ - --fail-if-improved - - # This build doesn't use problem matchers or check annotations - build_doc_oldest_supported_sphinx: - name: 'Docs (Oldest Sphinx)' - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v3 - - name: 'Set up Python' - uses: actions/setup-python@v4 - with: - python-version: '3.11' # known to work with Sphinx 3.2 - cache: 'pip' - cache-dependency-path: 'Doc/requirements-oldest-sphinx.txt' - - name: 'Install build dependencies' - run: make -C Doc/ venv REQUIREMENTS="requirements-oldest-sphinx.txt" - - name: 'Build HTML documentation' - run: make -C Doc/ SPHINXOPTS="-q" SPHINXERRORHANDLING="-W --keep-going" html + --fail-if-improved \ + --fail-if-new-news-nit # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v3 - - name: Register Sphinx problem matcher - run: echo "::add-matcher::.github/problem-matchers/sphinx.json" - - uses: actions/cache@v3 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} @@ -94,4 +101,4 @@ jobs: run: make -C Doc/ PYTHON=../python venv # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' - run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="-W --keep-going" doctest + run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml new file mode 100644 index 00000000000000..de0c40221364ad --- /dev/null +++ b/.github/workflows/reusable-macos.yml @@ -0,0 +1,81 @@ +name: Reusable macOS + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + free-threading: + required: false + type: boolean + default: false + os: + description: OS to run the job + required: true + type: string + +env: + FORCE_COLOR: 1 + +jobs: + build-macos: + name: build and test (${{ inputs.os }}) + runs-on: ${{ inputs.os }} + timeout-minutes: 60 + env: + HOMEBREW_NO_ANALYTICS: 1 + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 + PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} + - name: Install Homebrew dependencies + run: | + brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make + # Because alternate versions are not symlinked into place by default: + brew link --overwrite tcl-tk@8 + - name: Configure CPython + run: | + MACOSX_DEPLOYMENT_TARGET=10.15 \ + GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ + GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ + ./configure \ + --config-cache \ + --with-pydebug \ + --enable-slower-safety \ + --enable-safety \ + ${{ inputs.free-threading && '--disable-gil' || '' }} \ + --prefix=/opt/python-dev \ + --with-openssl="$(brew --prefix openssl@3.0)" + - name: Build CPython + if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + run: gmake -j8 + - name: Build CPython for compiler warning check + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt + - name: Display build info + run: make pythoninfo + - name: Check compiler warnings + if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + run: >- + python3 Tools/build/check_warnings.py + --compiler-output-file-path=compiler_output_macos.txt + --warning-ignore-file-path=Tools/build/.warningignore_macos + --compiler-output-type=clang + --fail-on-regression + --fail-on-improvement + --path-prefix="./" + - name: Tests + run: make ci diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml new file mode 100644 index 00000000000000..6a58e5305f8e09 --- /dev/null +++ b/.github/workflows/reusable-tsan.yml @@ -0,0 +1,94 @@ +name: Reusable Thread Sanitizer + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + +jobs: + build-tsan-reusable: + name: 'Thread sanitizer' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang-18 + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSAN crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + - name: TSAN option setup + run: | + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: >- + ./configure + --config-cache + --with-thread-sanitizer + --with-pydebug + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: ./python -m test --tsan -j4 + - name: Parallel tests + if: fromJSON(inputs.free-threading) + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + - name: Display TSAN logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 + - name: Archive TSAN logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: >- + tsan-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: tsan_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml new file mode 100644 index 00000000000000..76b19fd5d1a72e --- /dev/null +++ b/.github/workflows/reusable-ubuntu.yml @@ -0,0 +1,127 @@ +name: Reusable Ubuntu + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + bolt-optimizations: + description: Whether to enable BOLT optimizations + required: false + type: boolean + default: false + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + os: + description: OS to run the job + required: true + type: string + +env: + FORCE_COLOR: 1 + +jobs: + build-ubuntu-reusable: + name: build and test (${{ inputs.os }}) + runs-on: ${{ inputs.os }} + timeout-minutes: 60 + env: + OPENSSL_VER: 3.0.15 + PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" + - name: Install dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Install Clang and BOLT + if: ${{ fromJSON(inputs.bolt-optimizations) }} + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 + sudo apt-get install bolt-19 + echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Setup directory envs for out-of-tree builds + run: | + echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" + echo "CPYTHON_BUILDDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-builddir)" >> "$GITHUB_ENV" + - name: Create directories for read-only out-of-tree builds + run: mkdir -p "$CPYTHON_RO_SRCDIR" "$CPYTHON_BUILDDIR" + - name: Bind mount sources read-only + run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} + - name: Configure CPython out-of-tree + working-directory: ${{ env.CPYTHON_BUILDDIR }} + # `test_unpickle_module_race` writes to the source directory, which is + # read-only during builds — so we exclude it from profiling with BOLT. + run: >- + PROFILE_TASK='-m test --pgo --ignore test_unpickle_module_race' + ../cpython-ro-srcdir/configure + --config-cache + --with-pydebug + --enable-slower-safety + --enable-safety + --with-openssl="$OPENSSL_DIR" + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + ${{ fromJSON(inputs.bolt-optimizations) && '--enable-bolt' || '' }} + - name: Build CPython out-of-tree + if: ${{ inputs.free-threading }} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make -j + - name: Build CPython out-of-tree (for compiler warning check) + if: ${{ !inputs.free-threading }} + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: set -o pipefail; make -j --output-sync 2>&1 | tee compiler_output_ubuntu.txt + - name: Display build info + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: make pythoninfo + - name: Check compiler warnings + if: ${{ !inputs.free-threading }} + run: >- + python Tools/build/check_warnings.py + --compiler-output-file-path="${CPYTHON_BUILDDIR}/compiler_output_ubuntu.txt" + --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" + --compiler-output-type=gcc + --fail-on-regression + --fail-on-improvement + --path-prefix="../cpython-ro-srcdir/" + - name: Remount sources writable for tests + # some tests write to srcdir, lack of pyc files slows down testing + run: sudo mount "$CPYTHON_RO_SRCDIR" -oremount,rw + - name: Tests + working-directory: ${{ env.CPYTHON_BUILDDIR }} + run: xvfb-run make ci diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml new file mode 100644 index 00000000000000..6beb91e66d4027 --- /dev/null +++ b/.github/workflows/reusable-wasi.yml @@ -0,0 +1,84 @@ +name: Reusable WASI + +on: + workflow_call: + inputs: + config_hash: + required: true + type: string + +env: + FORCE_COLOR: 1 + +jobs: + build-wasi-reusable: + name: 'build and test' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + env: + WASMTIME_VERSION: 22.0.0 + WASI_SDK_VERSION: 24 + WASI_SDK_PATH: /opt/wasi-sdk + CROSS_BUILD_PYTHON: cross-build/build + CROSS_BUILD_WASI: cross-build/wasm32-wasip1 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + # No problem resolver registered as one doesn't currently exist for Clang. + - name: "Install wasmtime" + uses: bytecodealliance/actions/wasmtime/setup@v1 + with: + version: ${{ env.WASMTIME_VERSION }} + - name: "Restore WASI SDK" + id: cache-wasi-sdk + uses: actions/cache@v4 + with: + path: ${{ env.WASI_SDK_PATH }} + key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} + - name: "Install WASI SDK" # Hard-coded to x64. + if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' + run: | + mkdir "${WASI_SDK_PATH}" && \ + curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ + tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip + - name: "Configure ccache action" + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: "Add ccache to PATH" + run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: "Install Python" + uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Restore Python build config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache + # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. + # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. + # (Make sure to keep the key in sync with the other config.cache step below.) + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure build Python" + run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Tools/wasm/wasi.py make-build-python + - name: "Restore host config.cache" + uses: actions/cache@v4 + with: + path: ${{ env.CROSS_BUILD_WASI }}/config.cache + # Should be kept in sync with the other config.cache step above. + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} + - name: "Configure host" + # `--with-pydebug` inferred from configure-build-python + run: python3 Tools/wasm/wasi.py configure-host -- --config-cache + - name: "Make host" + run: python3 Tools/wasm/wasi.py make-host + - name: "Display build info" + run: make --directory "${CROSS_BUILD_WASI}" pythoninfo + - name: "Test" + run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml new file mode 100644 index 00000000000000..a50de344bba4da --- /dev/null +++ b/.github/workflows/reusable-windows-msi.yml @@ -0,0 +1,31 @@ +name: Reusable Windows MSI + +on: + workflow_call: + inputs: + arch: + description: CPU architecture + required: true + type: string + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build: + name: installer for ${{ inputs.arch }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + timeout-minutes: 60 + env: + ARCH: ${{ inputs.arch }} + IncludeFreethreaded: true + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build CPython installer + run: ./Tools/msi/build.bat --doc -"${ARCH}" + shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml new file mode 100644 index 00000000000000..37c802095b0f2f --- /dev/null +++ b/.github/workflows/reusable-windows.yml @@ -0,0 +1,50 @@ +name: Reusable Windows + +on: + workflow_call: + inputs: + arch: + description: CPU architecture + required: true + type: string + free-threading: + description: Whether to compile CPython in free-threading mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + IncludeUwp: >- + true + +jobs: + build: + name: Build and test (${{ inputs.arch }}) + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + timeout-minutes: 60 + env: + ARCH: ${{ inputs.arch }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Register MSVC problem matcher + if: inputs.arch != 'Win32' + run: echo "::add-matcher::.github/problem-matchers/msvc.json" + - name: Build CPython + run: >- + .\\PCbuild\\build.bat + -e -d -v + -p "${ARCH}" + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash + - name: Display build info + run: .\\python.bat -m test.pythoninfo + - name: Tests + run: >- + .\\PCbuild\\rt.bat + -p "${ARCH}" + -d -q --fast-ci + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 94676f5ee5fffc..febb2dd823a8fe 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -2,21 +2,19 @@ name: Mark stale pull requests on: schedule: - - cron: "0 0 * * *" - -permissions: - pull-requests: write + - cron: "0 */6 * * *" jobs: stale: if: github.repository_owner == 'python' - runs-on: ubuntu-latest + permissions: + pull-requests: write timeout-minutes: 10 steps: - name: "Check PRs" - uses: actions/stale@v8 + uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml new file mode 100644 index 00000000000000..e32cbf0aaa3c3e --- /dev/null +++ b/.github/workflows/tail-call.yml @@ -0,0 +1,139 @@ +name: Tail calling interpreter +on: + pull_request: + paths: + - '.github/workflows/tail-call.yml' + - 'Python/bytecodes.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' + - 'Python/generated_cases.c.h' + push: + paths: + - '.github/workflows/tail-call.yml' + - 'Python/bytecodes.c' + - 'Python/ceval.c' + - 'Python/ceval_macros.h' + - 'Python/generated_cases.c.h' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + FORCE_COLOR: 1 + +jobs: + tail-call: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + target: +# Un-comment as we add support for more platforms for tail-calling interpreters. +# - i686-pc-windows-msvc/msvc + - x86_64-pc-windows-msvc/msvc +# - aarch64-pc-windows-msvc/msvc + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + - free-threading + llvm: + - 20 + include: +# - target: i686-pc-windows-msvc/msvc +# architecture: Win32 +# runner: windows-latest + - target: x86_64-pc-windows-msvc/msvc + architecture: x64 + runner: windows-latest +# - target: aarch64-pc-windows-msvc/msvc +# architecture: ARM64 +# runner: windows-latest + - target: x86_64-apple-darwin/clang + architecture: x86_64 + runner: macos-13 + - target: aarch64-apple-darwin/clang + architecture: aarch64 + runner: macos-14 + - target: x86_64-unknown-linux-gnu/gcc + architecture: x86_64 + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + architecture: aarch64 + runner: ubuntu-24.04-arm + - target: free-threading + architecture: x86_64 + runner: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Native Windows (debug) + if: runner.os == 'Windows' && matrix.architecture != 'ARM64' + shell: cmd + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + set PlatformToolset=clangcl + set LLVMToolsVersion=${{ matrix.llvm }}.1.0 + set LLVMInstallDir=C:\Program Files\LLVM + call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} + call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + # No tests (yet): + - name: Emulated Windows (release) + if: runner.os == 'Windows' && matrix.architecture == 'ARM64' + shell: cmd + run: | + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 + set PlatformToolset=clangcl + set LLVMToolsVersion=${{ matrix.llvm }}.1.0 + set LLVMInstallDir=C:\Program Files\LLVM + ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} + + # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. + # This is a bug in the macOS runner image where the pre-installed Python is installed in the same + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # the symlink to the pre-installed Python so that the Homebrew Python is used instead. + # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail. + # We either need to upgrade LLVM or change the directory being pointed to. + - name: Native macOS (release) + if: runner.os == 'macOS' + run: | + brew update + find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete + brew install llvm@${{ matrix.llvm }} + export SDKROOT="$(xcrun --show-sdk-path)" + export PATH="/usr/local/opt/llvm/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm/bin:$PATH" + CC=clang-20 ./configure --with-tail-call-interp + make all --jobs 4 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux (debug) + if: runner.os == 'Linux' && matrix.target != 'free-threading' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-20 ./configure --with-tail-call-interp --with-pydebug + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + + - name: Native Linux with free-threading (release) + if: matrix.target == 'free-threading' + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} + export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + CC=clang-20 ./configure --with-tail-call-interp --disable-gil + make all --jobs 4 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 17d841f1f1c54a..463e7bf3355cc3 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,8 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000000..9b42b47cc85545 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,10 @@ +# Configuration for the zizmor static analysis tool, run via pre-commit in CI +# https://woodruffw.github.io/zizmor/configuration/ +rules: + dangerous-triggers: + ignore: + - documentation-links.yml + unpinned-uses: + config: + policies: + "*": ref-pin diff --git a/.gitignore b/.gitignore index dddf28da016192..cdb0352e0a8836 100644 --- a/.gitignore +++ b/.gitignore @@ -38,10 +38,12 @@ tags TAGS .vs/ .vscode/ +.cache/ gmon.out .coverage .mypy_cache/ .pytest_cache/ +.ruff_cache/ .DS_Store *.exe @@ -68,6 +70,17 @@ Lib/test/data/* /_bootstrap_python /Makefile /Makefile.pre +/iOSTestbed.* +iOS/Frameworks/ +iOS/Resources/Info.plist +iOS/testbed/build +iOS/testbed/Python.xcframework/ios-*/bin +iOS/testbed/Python.xcframework/ios-*/include +iOS/testbed/Python.xcframework/ios-*/lib +iOS/testbed/Python.xcframework/ios-*/Python.framework +iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace +iOS/testbed/iOSTestbed.xcodeproj/xcuserdata +iOS/testbed/iOSTestbed.xcodeproj/xcshareddata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -124,11 +137,13 @@ Tools/unicode/data/ /config.status.lineno # hendrikmuhs/ccache-action@v1 /.ccache +/cross-build/ +/jit_stencils*.h /platform /profile-clean-stamp /profile-run-stamp /profile-bolt-stamp -/Python/deepfreeze/*.c +/profile-gen-stamp /pybuilddir.txt /pyconfig.h /python-config @@ -156,5 +171,10 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained +# People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. +/.claude/ +CLAUDE.local.md + +#### main branch only stuff below this line, things to backport go above. #### +# main branch only: ABI files are not checked/maintained. Doc/data/python*.abi diff --git a/.mailmap b/.mailmap index 013c839ed6b7a4..8f11bebb18fc17 100644 --- a/.mailmap +++ b/.mailmap @@ -1,3 +1,4 @@ # This file sets the canonical name for contributors to the repository. # Documentation: https://git-scm.com/docs/gitmailmap +Willow Chargin <wchargin@gmail.com> Amethyst Reese <amethyst@n7.gg> <john@noswap.com> diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85a6de4abe0146..822a8a9f4e5076 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,18 +1,98 @@ repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.11.8 + hooks: + - id: ruff + name: Run Ruff (lint) on Doc/ + args: [--exit-non-zero-on-fix] + files: ^Doc/ + - id: ruff + name: Run Ruff (lint) on Lib/test/ + args: [--exit-non-zero-on-fix] + files: ^Lib/test/ + - id: ruff + name: Run Ruff (lint) on Tools/build/ + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] + files: ^Tools/build/ + - id: ruff + name: Run Ruff (lint) on Argument Clinic + args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] + files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-format + name: Run Ruff (format) on Doc/ + args: [--check] + files: ^Doc/ + - id: ruff-format + name: Run Ruff (format) on Tools/build/check_warnings.py + args: [--check, --config=Tools/build/.ruff.toml] + files: ^Tools/build/check_warnings.py + + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 25.1.0 + hooks: + - id: black + name: Run Black on Tools/jit/ + files: ^Tools/jit/ + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v5.0.0 hooks: + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + exclude: ^Lib/test/test_tomllib/ - id: check-yaml - id: end-of-file-fixer - types: [python] - exclude: Lib/test/coding20731.py + types_or: [python, yaml] + exclude: Lib/test/tokenizedata/coding20731.py + - id: end-of-file-fixer + files: '^\.github/CODEOWNERS$' + - id: trailing-whitespace + types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace - types_or: [c, python, rst] + files: '^\.github/CODEOWNERS|\.(gram)$' + + - repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.33.0 + hooks: + - id: check-dependabot + - id: check-github-workflows + - id: check-readthedocs + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.7 + hooks: + - id: actionlint + + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.6.0 + hooks: + - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v0.6.8 + rev: v1.0.0 hooks: - id: sphinx-lint args: [--enable=default-role] - files: ^Doc/ - types: [rst] + files: ^Doc/|^Misc/NEWS.d/ + + - repo: local + hooks: + - id: blurb-no-space-c-api + name: Check C API news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/C_API/ + files: Misc/NEWS.d/next/C API/20.*.rst + + - repo: local + hooks: + - id: blurb-no-space-core-and-builtins + name: Check Core and Builtins news entries + language: fail + entry: Space found in path, move to Misc/NEWS.d/next/Core_and_Builtins/ + files: Misc/NEWS.d/next/Core and Builtins/20.*.rst + + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes diff --git a/.readthedocs.yml b/.readthedocs.yml index 59830c79a404e0..0a2c3f8345367f 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,7 +8,7 @@ sphinx: configuration: Doc/conf.py build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: python: "3" @@ -26,7 +26,9 @@ build: exit 183; fi + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest - make -C Doc venv html - mkdir _readthedocs - mv Doc/build/html _readthedocs/html - diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000000000..1c015fa88415bc --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,12 @@ +# Default settings for Ruff in CPython + +# PYTHON_FOR_REGEN +target-version = "py310" + +# PEP 8 +line-length = 79 + +# Enable automatic fixes by default. +# To override this, use ``fix = false`` in a subdirectory's config file +# or ``--no-fix`` on the command line. +fix = true diff --git a/Android/README.md b/Android/README.md new file mode 100644 index 00000000000000..6cabd6ba5d6844 --- /dev/null +++ b/Android/README.md @@ -0,0 +1,162 @@ +# Python for Android + +If you obtained this README as part of a release package, then the only +applicable sections are "Prerequisites", "Testing", and "Using in your own app". + +If you obtained this README as part of the CPython source tree, then you can +also follow the other sections to compile Python for Android yourself. + +However, most app developers should not need to do any of these things manually. +Instead, use one of the tools listed +[here](https://docs.python.org/3/using/android.html), which will provide a much +easier experience. + + +## Prerequisites + +If you already have an Android SDK installed, export the `ANDROID_HOME` +environment variable to point at its location. Otherwise, here's how to install +it: + +* Download the "Command line tools" from <https://developer.android.com/studio>. +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + +The `android.py` script will automatically use the SDK's `sdkmanager` to install +any packages it needs. + +The script also requires the following commands to be on the `PATH`: + +* `curl` +* `java` (or set the `JAVA_HOME` environment variable) + + +## Building + +Python can be built for Android on any POSIX platform supported by the Android +development tools, which currently means Linux or macOS. + +First we'll make a "build" Python (for your development machine), then use it to +help produce a "host" Python for Android. So make sure you have all the usual +tools and libraries needed to build Python for your development machine. + +The easiest way to do a build is to use the `android.py` script. You can either +have it perform the entire build process from start to finish in one step, or +you can do it in discrete steps that mirror running `configure` and `make` for +each of the two builds of Python you end up producing. + +The discrete steps for building via `android.py` are: + +```sh +./android.py configure-build +./android.py make-build +./android.py configure-host HOST +./android.py make-host HOST +``` + +`HOST` identifies which architecture to build. To see the possible values, run +`./android.py configure-host --help`. + +To do all steps in a single command, run: + +```sh +./android.py build HOST +``` + +In the end you should have a build Python in `cross-build/build`, and a host +Python in `cross-build/HOST`. + +You can use `--` as a separator for any of the `configure`-related commands – +including `build` itself – to pass arguments to the underlying `configure` +call. For example, if you want a pydebug build that also caches the results from +`configure`, you can do: + +```sh +./android.py build HOST -- -C --with-pydebug +``` + + +## Packaging + +After building an architecture as described in the section above, you can +package it for release with this command: + +```sh +./android.py package HOST +``` + +`HOST` is defined in the section above. + +This will generate a tarball in `cross-build/HOST/dist`, whose structure is +similar to the `Android` directory of the CPython source tree. + + +## Testing + +The Python test suite can be run on Linux, macOS, or Windows: + +* On Linux, the emulator needs access to the KVM virtualization interface, and + a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. + +The test suite can usually be run on a device with 2 GB of RAM, but this is +borderline, so you may need to increase it to 4 GB. As of Android +Studio Koala, 2 GB is the default for all emulators, although the user interface +may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, +and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these +manually to the same value, or use the Android Studio Device Manager, which will +update both files. + +You can run the test suite either: + +* Within the CPython repository, after doing a build as described above. On + Windows, you won't be able to do the build on the same machine, so you'll have + to copy the `cross-build/HOST/prefix` directory from somewhere else. + +* Or by taking a release package built using the `package` command, extracting + it wherever you want, and using its own copy of `android.py`. + +The test script supports the following modes: + +* In `--connected` mode, it runs on a device or emulator you have already + connected to the build machine. List the available devices with + `$ANDROID_HOME/platform-tools/adb devices -l`, then pass a device ID to the + script like this: + + ```sh + ./android.py test --connected emulator-5554 + ``` + +* In `--managed` mode, it uses a temporary headless emulator defined in the + `managedDevices` section of testbed/app/build.gradle.kts. This mode is slower, + but more reproducible. + + We currently define two devices: `minVersion` and `maxVersion`, corresponding + to our minimum and maximum supported Android versions. For example: + + ```sh + ./android.py test --managed maxVersion + ``` + +By default, the only messages the script will show are Python's own stdout and +stderr. Add the `-v` option to also show Gradle output, and non-Python logcat +messages. + +Any other arguments on the `android.py test` command line will be passed through +to `python -m test` – use `--` to separate them from android.py's own options. +See the [Python Developer's +Guide](https://devguide.python.org/testing/run-write-tests/) for common options +– most of them will work on Android, except for those that involve subprocesses, +such as `-j`. + +Every time you run `android.py test`, changes in pure-Python files in the +repository's `Lib` directory will be picked up immediately. Changes in C files, +and architecture-specific files such as sysconfigdata, will not take effect +until you re-run `android.py make-host` or `build`. + + +## Using in your own app + +See https://docs.python.org/3/using/android.html. diff --git a/Android/android-env.sh b/Android/android-env.sh new file mode 100644 index 00000000000000..bab4130c9e92d0 --- /dev/null +++ b/Android/android-env.sh @@ -0,0 +1,99 @@ +# This script must be sourced with the following variables already set: +: "${ANDROID_HOME:?}" # Path to Android SDK +: "${HOST:?}" # GNU target triplet + +# You may also override the following: +: "${api_level:=24}" # Minimum Android API level the build will run on +: "${PREFIX:-}" # Path in which to find required libraries + + +# Print all messages on stderr so they're visible when running within build-wheel. +log() { + echo "$1" >&2 +} + +fail() { + log "$1" + exit 1 +} + +# When moving to a new version of the NDK, carefully review the following: +# +# * https://developer.android.com/ndk/downloads/revision_history +# +# * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md +# where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: +# https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md +ndk_version=27.1.12297006 + +ndk=$ANDROID_HOME/ndk/$ndk_version +if ! [ -e "$ndk" ]; then + log "Installing NDK - this may take several minutes" + yes | "$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" "ndk;$ndk_version" +fi + +if [ "$HOST" = "arm-linux-androideabi" ]; then + clang_triplet=armv7a-linux-androideabi +else + clang_triplet="$HOST" +fi + +# These variables are based on BuildSystemMaintainers.md above, and +# $ndk/build/cmake/android.toolchain.cmake. +toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) +export AR="$toolchain/bin/llvm-ar" +export AS="$toolchain/bin/llvm-as" +export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CXX="${CC}++" +export LD="$toolchain/bin/ld" +export NM="$toolchain/bin/llvm-nm" +export RANLIB="$toolchain/bin/llvm-ranlib" +export READELF="$toolchain/bin/llvm-readelf" +export STRIP="$toolchain/bin/llvm-strip" + +# The quotes make sure the wildcard in the `toolchain` assignment has been expanded. +for path in "$AR" "$AS" "$CC" "$CXX" "$LD" "$NM" "$RANLIB" "$READELF" "$STRIP"; do + if ! [ -e "$path" ]; then + fail "$path does not exist" + fi +done + +export CFLAGS="-D__BIONIC_NO_PAGE_SIZE_MACRO" +export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,-z,max-page-size=16384" + +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + +# Many packages get away with omitting -lm on Linux, but Android is stricter. +LDFLAGS="$LDFLAGS -lm" + +# -mstackrealign is included where necessary in the clang launcher scripts which are +# pointed to by $CC, so we don't need to include it here. +if [ "$HOST" = "arm-linux-androideabi" ]; then + CFLAGS="$CFLAGS -march=armv7-a -mthumb" +fi + +if [ -n "${PREFIX:-}" ]; then + abs_prefix="$(realpath "$PREFIX")" + CFLAGS="$CFLAGS -I$abs_prefix/include" + LDFLAGS="$LDFLAGS -L$abs_prefix/lib" + + export PKG_CONFIG="pkg-config --define-prefix" + export PKG_CONFIG_LIBDIR="$abs_prefix/lib/pkgconfig" +fi + +# When compiling C++, some build systems will combine CFLAGS and CXXFLAGS, and some will +# use CXXFLAGS alone. +export CXXFLAGS="$CFLAGS" + +# Use the same variable name as conda-build +if [ "$(uname)" = "Darwin" ]; then + CPU_COUNT="$(sysctl -n hw.ncpu)" + export CPU_COUNT +else + CPU_COUNT="$(nproc)" + export CPU_COUNT +fi diff --git a/Android/android.py b/Android/android.py new file mode 100755 index 00000000000000..3f48b42aa17571 --- /dev/null +++ b/Android/android.py @@ -0,0 +1,723 @@ +#!/usr/bin/env python3 + +import asyncio +import argparse +import os +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +from asyncio import wait_for +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from glob import glob +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError +from tempfile import TemporaryDirectory + + +SCRIPT_NAME = Path(__file__).name +ANDROID_DIR = Path(__file__).resolve().parent +CHECKOUT = ANDROID_DIR.parent +TESTBED_DIR = ANDROID_DIR / "testbed" +CROSS_BUILD_DIR = CHECKOUT / "cross-build" + +HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] +APP_ID = "org.python.testbed" +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +try: + android_home = Path(os.environ['ANDROID_HOME']) +except KeyError: + sys.exit("The ANDROID_HOME environment variable is required.") + +adb = Path( + f"{android_home}/platform-tools/adb" + + (".exe" if os.name == "nt" else "") +) + +gradlew = Path( + f"{TESTBED_DIR}/gradlew" + + (".bat" if os.name == "nt" else "") +) + +logcat_started = False + + +def delete_glob(pattern): + # Path.glob doesn't accept non-relative patterns. + for path in glob(str(pattern)): + path = Path(path) + print(f"Deleting {path} ...") + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + +def subdir(*parts, create=False): + path = CROSS_BUILD_DIR.joinpath(*parts) + if not path.exists(): + if not create: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}.") + else: + path.mkdir(parents=True) + return path + + +def run(command, *, host=None, env=None, log=True, **kwargs): + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + original_env = env.copy() + + if host: + env_script = ANDROID_DIR / "android-env.sh" + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={subdir(host)}/prefix; " + f". {env_script}; " + f"export", + check=True, shell=True, text=True, stdout=subprocess.PIPE + ).stdout + + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if env.get(key) != value: + print(line) + env[key] = value + + if env == original_env: + raise ValueError(f"Found no variables in {env_script.name} output:\n" + + env_output) + + if log: + print(">", " ".join(map(str, command))) + return subprocess.run(command, env=env, **kwargs) + + +def build_python_path(): + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError("Unable to find `python(.exe)` in " + f"{build_dir}") + + return binary + + +def configure_build_python(context): + if context.clean: + clean("build") + os.chdir(subdir("build", create=True)) + + command = [relpath(CHECKOUT / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context): + os.chdir(subdir("build")) + run(["make", "-j", str(os.cpu_count())]) + + +def unpack_deps(host, prefix_dir): + deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" + for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", + "sqlite-3.49.1-0", "xz-5.4.6-1"]: + filename = f"{name_ver}-{host}.tar.gz" + download(f"{deps_url}/{name_ver}/{filename}") + shutil.unpack_archive(filename, prefix_dir) + os.remove(filename) + + +def download(url, target_dir="."): + out_path = f"{target_dir}/{basename(url)}" + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) + return out_path + + +def configure_host_python(context): + if context.clean: + clean(context.host) + + host_dir = subdir(context.host, create=True) + prefix_dir = host_dir / "prefix" + if not prefix_dir.exists(): + prefix_dir.mkdir() + unpack_deps(context.host, prefix_dir) + + os.chdir(host_dir) + command = [ + # Basic cross-compiling configuration + relpath(CHECKOUT / "configure"), + f"--host={context.host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--without-ensurepip", + + # Android always uses a shared libpython. + "--enable-shared", + "--without-static-libpython", + + # Dependent libraries. The others are found using pkg-config: see + # android-env.sh. + f"--with-openssl={prefix_dir}", + ] + + if context.args: + command.extend(context.args) + run(command, host=context.host) + + +def make_host_python(context): + # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so + # delete any previous Python installation to prevent it being used during + # the build. + host_dir = subdir(context.host) + prefix_dir = host_dir / "prefix" + for pattern in ("include/python*", "lib/libpython*", "lib/python*"): + delete_glob(f"{prefix_dir}/{pattern}") + + os.chdir(host_dir) + run(["make", "-j", str(os.cpu_count())], host=context.host) + run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + + +def build_all(context): + steps = [configure_build_python, make_build_python, configure_host_python, + make_host_python] + for step in steps: + step(context) + + +def clean(host): + delete_glob(CROSS_BUILD_DIR / host) + + +def clean_all(context): + for host in HOSTS + ["build"]: + clean(host) + + +def setup_sdk(): + sdkmanager = android_home / ( + "cmdline-tools/latest/bin/sdkmanager" + + (".bat" if os.name == "nt" else "") + ) + + # Gradle will fail if it needs to install an SDK package whose license + # hasn't been accepted, so pre-accept all licenses. + if not all((android_home / "licenses" / path).exists() for path in [ + "android-sdk-arm-dbt-license", "android-sdk-license" + ]): + run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + + # Gradle may install this automatically, but we can't rely on that because + # we need to run adb within the logcat task. + if not adb.exists(): + run([sdkmanager, "platform-tools"]) + + +# To avoid distributing compiled artifacts without corresponding source code, +# the Gradle wrapper is not included in the CPython repository. Instead, we +# extract it from the Gradle GitHub repository. +def setup_testbed(): + paths = ["gradlew", "gradlew.bat", "gradle/wrapper/gradle-wrapper.jar"] + if all((TESTBED_DIR / path).exists() for path in paths): + return + + # The wrapper version isn't important, as any version of the wrapper can + # download any version of Gradle. The Gradle version actually used for the + # build is specified in testbed/gradle/wrapper/gradle-wrapper.properties. + version = "8.9.0" + + for path in paths: + out_path = TESTBED_DIR / path + out_path.parent.mkdir(exist_ok=True) + download( + f"https://raw.githubusercontent.com/gradle/gradle/v{version}/{path}", + out_path.parent, + ) + os.chmod(out_path, 0o755) + + +# run_testbed will build the app automatically, but it's useful to have this as +# a separate command to allow running the app outside of this script. +def build_testbed(context): + setup_sdk() + setup_testbed() + run( + [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], + cwd=TESTBED_DIR, + ) + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# The `test` subcommand runs all subprocesses through this context manager so +# that no matter what happens, they can always be cancelled from another task, +# and they will always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Gradle to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise CalledProcessError( + process.returncode, args, + stdout.decode(*DECODE_ARGS), stderr.decode(*DECODE_ARGS) + ) + + +# Return a list of the serial numbers of connected devices. Emulators will have +# serials of the form "emulator-5678". +async def list_devices(): + serials = [] + header_found = False + + lines = (await async_check_output(adb, "devices")).splitlines() + for line in lines: + # Ignore blank lines, and all lines before the header. + line = line.strip() + if line == "List of devices attached": + header_found = True + elif header_found and line: + try: + serial, status = line.split() + except ValueError: + raise ValueError(f"failed to parse {line!r}") + if status == "device": + serials.append(serial) + + if not header_found: + raise ValueError(f"failed to parse {lines}") + return serials + + +async def find_device(context, initial_devices): + if context.managed: + print("Waiting for managed device - this may take several minutes") + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + serial = new_devices.pop() + print(f"Serial: {serial}") + return serial + else: + exit(f"Found more than one new device: {new_devices}") + else: + return context.connected + + +# An older version of this script in #121595 filtered the logs by UID instead. +# But logcat can't filter by UID until API level 31. If we ever switch back to +# filtering by UID, we'll also have to filter by time so we only show messages +# produced after the initial call to `stop_app`. +# +# We're more likely to miss the PID because it's shorter-lived, so there's a +# workaround in PythonSuite.kt to stop it being *too* short-lived. +async def find_pid(serial): + print("Waiting for app to start - this may take several minutes") + shown_error = False + while True: + try: + # `pidof` requires API level 24 or higher. The level 23 emulator + # includes it, but it doesn't work (it returns all processes). + pid = (await async_check_output( + adb, "-s", serial, "shell", "pidof", "-s", APP_ID + )).strip() + except CalledProcessError as e: + # If the app isn't running yet, pidof gives no output. So if there + # is output, there must have been some other error. However, this + # sometimes happens transiently, especially when running a managed + # emulator for the first time, so don't make it fatal. + if (e.stdout or e.stderr) and not shown_error: + print_called_process_error(e) + print("This may be transient, so continuing to wait") + shown_error = True + else: + # Some older devices (e.g. Nexus 4) return zero even when no process + # was found, so check whether we actually got any output. + if pid: + print(f"PID: {pid}") + return pid + + # Loop fairly rapidly to avoid missing a short-lived process. + await asyncio.sleep(0.2) + + +async def logcat_task(context, initial_devices): + # Gradle may need to do some large downloads of libraries and emulator + # images. This will happen during find_device in --managed mode, or find_pid + # in --connected mode. + startup_timeout = 600 + serial = await wait_for(find_device(context, initial_devices), startup_timeout) + pid = await wait_for(find_pid(serial), startup_timeout) + + # `--pid` requires API level 24 or higher. + args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] + hidden_output = [] + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + level, message = match.groups() + else: + # If the regex doesn't match, this is probably the second or + # subsequent line of a multi-line message. Python won't produce + # such messages, but other components might. + level, message = None, line + + # Exclude high-volume messages which are rarely useful. + if context.verbose < 2 and "from python test_syslog" in message: + continue + + # Put high-level messages on stderr so they're highlighted in the + # buildbot logs. This will include Python's own stderr. + stream = ( + sys.stderr + if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) + else sys.stdout + ) + + # To simplify automated processing of the output, e.g. a buildbot + # posting a failure notice on a GitHub PR, we strip the level and + # tag indicators from Python's stdout and stderr. + for prefix in ["python.stdout: ", "python.stderr: "]: + if message.startswith(prefix): + global logcat_started + logcat_started = True + stream.write(message.removeprefix(prefix)) + break + else: + if context.verbose: + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + stream.write(line) + else: + hidden_output.append(line) + + # If the device disconnects while logcat is running, which always + # happens in --managed mode, some versions of adb return non-zero. + # Distinguish this from a logcat startup error by checking whether we've + # received a message from Python yet. + status = await wait_for(process.wait(), timeout=1) + if status != 0 and not logcat_started: + raise CalledProcessError(status, args, "".join(hidden_output)) + + +def stop_app(serial): + run([adb, "-s", serial, "shell", "am", "force-stop", APP_ID], log=False) + + +async def gradle_task(context): + env = os.environ.copy() + if context.managed: + task_prefix = context.managed + else: + task_prefix = "connected" + env["ANDROID_SERIAL"] = context.connected + + args = [ + gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", + "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" + + shlex.join(context.args), + ] + hidden_output = [] + try: + async with async_process( + *args, cwd=TESTBED_DIR, env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if context.verbose or line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + hidden_output.append(line) + + status = await wait_for(process.wait(), timeout=1) + if status == 0: + exit(0) + else: + raise CalledProcessError(status, args) + finally: + # If logcat never started, then something has gone badly wrong, so the + # user probably wants to see the Gradle output even in non-verbose mode. + if hidden_output and not logcat_started: + sys.stdout.write("".join(hidden_output)) + + # Gradle does not stop the tests when interrupted. + if context.connected: + stop_app(context.connected) + + +async def run_testbed(context): + setup_sdk() + setup_testbed() + + if context.managed: + # In this mode, Gradle will create a device with an unpredictable name. + # So we save a list of the running devices before starting Gradle, and + # find_device then waits for a new device to appear. + initial_devices = await list_devices() + else: + # In case the previous shutdown was unclean, make sure the app isn't + # running, otherwise we might show logs from a previous run. This is + # unnecessary in --managed mode, because Gradle creates a new emulator + # every time. + stop_app(context.connected) + initial_devices = None + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(logcat_task(context, initial_devices)) + tg.create_task(gradle_task(context)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +def package_version(prefix_dir): + patchlevel_glob = f"{prefix_dir}/include/python*/patchlevel.h" + patchlevel_paths = glob(patchlevel_glob) + if len(patchlevel_paths) != 1: + sys.exit(f"{patchlevel_glob} matched {len(patchlevel_paths)} paths.") + + for line in open(patchlevel_paths[0]): + if match := re.fullmatch(r'\s*#define\s+PY_VERSION\s+"(.+)"\s*', line): + version = match[1] + break + else: + sys.exit(f"Failed to find Python version in {patchlevel_paths[0]}.") + + # If not building against a tagged commit, add a timestamp to the version. + # Follow the PyPA version number rules, as this will make it easier to + # process with other tools. + if version.endswith("+"): + version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") + + return version + + +def package(context): + prefix_dir = subdir(context.host, "prefix") + version = package_version(prefix_dir) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + temp_dir = Path(temp_dir) + + # Include all tracked files from the Android directory. + for line in run( + ["git", "ls-files"], + cwd=ANDROID_DIR, capture_output=True, text=True, log=False, + ).stdout.splitlines(): + src = ANDROID_DIR / line + dst = temp_dir / line + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(src, dst, follow_symlinks=False) + + # Include anything from the prefix directory which could be useful + # either for embedding Python in an app, or building third-party + # packages against it. + for rel_dir, patterns in [ + ("include", ["openssl*", "python*", "sqlite*"]), + ("lib", ["engines-3", "libcrypto*.so", "libpython*", "libsqlite*", + "libssl*.so", "ossl-modules", "python*"]), + ("lib/pkgconfig", ["*crypto*", "*ssl*", "*python*", "*sqlite*"]), + ]: + for pattern in patterns: + for src in glob(f"{prefix_dir}/{rel_dir}/{pattern}"): + dst = temp_dir / relpath(src, prefix_dir.parent) + dst.parent.mkdir(parents=True, exist_ok=True) + if Path(src).is_dir(): + shutil.copytree( + src, dst, symlinks=True, + ignore=lambda *args: ["__pycache__"] + ) + else: + shutil.copy2(src, dst, follow_symlinks=False) + + dist_dir = subdir(context.host, "dist", create=True) + package_path = shutil.make_archive( + f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir + ) + print(f"Wrote {package_path}") + + +# Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated +# by the buildbot worker, we'll make an attempt to clean up our subprocesses. +def install_signal_handler(): + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + +def parse_args(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + + # Subcommands + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser("configure-build", + help="Run `configure` for the " + "build Python") + make_build = subcommands.add_parser("make-build", + help="Run `make` for the build Python") + configure_host = subcommands.add_parser("configure-host", + help="Run `configure` for Android") + make_host = subcommands.add_parser("make-host", + help="Run `make` for Android") + subcommands.add_parser( + "clean", help="Delete all build and prefix directories") + subcommands.add_parser( + "build-testbed", help="Build the testbed app") + test = subcommands.add_parser( + "test", help="Run the test suite") + package = subcommands.add_parser("package", help="Make a release package") + + # Common arguments + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "--clean", action="store_true", default=False, dest="clean", + help="Delete the relevant build and prefix directories first") + for subcommand in [build, configure_host, make_host, package]: + subcommand.add_argument( + "host", metavar="HOST", choices=HOSTS, + help="Host triplet: choices=[%(choices)s]") + for subcommand in build, configure_build, configure_host: + subcommand.add_argument("args", nargs="*", + help="Extra arguments to pass to `configure`") + + # Test arguments + test.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show Gradle output, and non-Python logcat messages. " + "Use twice to include high-volume messages which are rarely useful.") + device_group = test.add_mutually_exclusive_group(required=True) + device_group.add_argument( + "--connected", metavar="SERIAL", help="Run on a connected device. " + "Connect it yourself, then get its serial from `adb devices`.") + device_group.add_argument( + "--managed", metavar="NAME", help="Run on a Gradle-managed device. " + "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + test.add_argument( + "args", nargs="*", help=f"Arguments for `python -m test`. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + + return parser.parse_args() + + +def main(): + install_signal_handler() + + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + + context = parse_args() + dispatch = { + "configure-build": configure_build_python, + "make-build": make_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "build": build_all, + "clean": clean_all, + "build-testbed": build_testbed, + "test": run_testbed, + "package": package, + } + + try: + result = dispatch[context.subcommand](context) + if asyncio.iscoroutine(result): + asyncio.run(result) + except CalledProcessError as e: + print_called_process_error(e) + sys.exit(1) + + +def print_called_process_error(e): + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # Format the command so it can be copied into a shell. shlex uses single + # quotes, so we surround the whole command with double quotes. + args_joined = ( + e.cmd if isinstance(e.cmd, str) + else " ".join(shlex.quote(str(arg)) for arg in e.cmd) + ) + print( + f'Command "{args_joined}" returned exit status {e.returncode}' + ) + + +if __name__ == "__main__": + main() diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore new file mode 100644 index 00000000000000..7c57aee58c160a --- /dev/null +++ b/Android/testbed/.gitignore @@ -0,0 +1,22 @@ +# The Gradle wrapper can be downloaded by running the `test` or `build-testbed` +# commands of android.py. +/gradlew +/gradlew.bat +/gradle/wrapper/gradle-wrapper.jar + +# The repository's top-level .gitignore file ignores all .idea directories, but +# we want to keep any files which can't be regenerated from the Gradle +# configuration. +!.idea/ +/.idea/* +!/.idea/inspectionProfiles + +*.iml +.gradle +/local.properties +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/Android/testbed/.idea/inspectionProfiles/Project_Default.xml b/Android/testbed/.idea/inspectionProfiles/Project_Default.xml new file mode 100644 index 00000000000000..220d9ed4ef20f7 --- /dev/null +++ b/Android/testbed/.idea/inspectionProfiles/Project_Default.xml @@ -0,0 +1,8 @@ +<component name="InspectionProjectProfileManager"> + <profile version="1.0"> + <option name="myName" value="Project Default" /> + <inspection_tool class="AndroidLintGradleDependency" enabled="true" level="WEAK WARNING" enabled_by_default="true" editorAttributes="INFO_ATTRIBUTES" /> + <inspection_tool class="AndroidLintOldTargetApi" enabled="true" level="WEAK WARNING" enabled_by_default="true" editorAttributes="INFO_ATTRIBUTES" /> + <inspection_tool class="UnstableApiUsage" enabled="true" level="WEAK WARNING" enabled_by_default="true" editorAttributes="INFO_ATTRIBUTES" /> + </profile> +</component> \ No newline at end of file diff --git a/Android/testbed/app/.gitignore b/Android/testbed/app/.gitignore new file mode 100644 index 00000000000000..42afabfd2abebf --- /dev/null +++ b/Android/testbed/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts new file mode 100644 index 00000000000000..c627cb1b0e0b22 --- /dev/null +++ b/Android/testbed/app/build.gradle.kts @@ -0,0 +1,250 @@ +import com.android.build.api.variant.* +import kotlin.math.max + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +val ANDROID_DIR = file("../..") +val PYTHON_DIR = ANDROID_DIR.parentFile!! +val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") +val inSourceTree = ( + ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() +) + +val KNOWN_ABIS = mapOf( + "aarch64-linux-android" to "arm64-v8a", + "x86_64-linux-android" to "x86_64", +) + +// Discover prefixes. +val prefixes = ArrayList<File>() +if (inSourceTree) { + for ((triplet, _) in KNOWN_ABIS.entries) { + val prefix = file("$PYTHON_CROSS_DIR/$triplet/prefix") + if (prefix.exists()) { + prefixes.add(prefix) + } + } +} else { + // Testbed is inside a release package. + val prefix = file("$ANDROID_DIR/prefix") + if (prefix.exists()) { + prefixes.add(prefix) + } +} +if (prefixes.isEmpty()) { + throw GradleException( + "No Android prefixes found: see README.md for testing instructions" + ) +} + +// Detect Python versions and ABIs. +lateinit var pythonVersion: String +var abis = HashMap<File, String>() +for ((i, prefix) in prefixes.withIndex()) { + val libDir = file("$prefix/lib") + val version = run { + for (filename in libDir.list()!!) { + """python(\d+\.\d+)""".toRegex().matchEntire(filename)?.let { + return@run it.groupValues[1] + } + } + throw GradleException("Failed to find Python version in $libDir") + } + if (i == 0) { + pythonVersion = version + } else if (pythonVersion != version) { + throw GradleException( + "${prefixes[0]} is Python $pythonVersion, but $prefix is Python $version" + ) + } + + val libPythonDir = file("$libDir/python$pythonVersion") + val triplet = run { + for (filename in libPythonDir.list()!!) { + """_sysconfigdata__android_(.+).py""".toRegex().matchEntire(filename)?.let { + return@run it.groupValues[1] + } + } + throw GradleException("Failed to find Python triplet in $libPythonDir") + } + abis[prefix] = KNOWN_ABIS[triplet]!! +} + + +android { + val androidEnvFile = file("../../android-env.sh").absoluteFile + + namespace = "org.python.testbed" + compileSdk = 34 + + defaultConfig { + applicationId = "org.python.testbed" + + minSdk = androidEnvFile.useLines { + for (line in it) { + """api_level:=(\d+)""".toRegex().find(line)?.let { + return@useLines it.groupValues[1].toInt() + } + } + throw GradleException("Failed to find API level in $androidEnvFile") + } + targetSdk = 34 + + versionCode = 1 + versionName = "1.0" + + ndk.abiFilters.addAll(abis.values) + externalNativeBuild.cmake.arguments( + "-DPYTHON_PREFIX_DIR=" + if (inSourceTree) { + // AGP uses the ${} syntax for its own purposes, so use a Jinja style + // placeholder. + "$PYTHON_CROSS_DIR/{{triplet}}/prefix" + } else { + prefixes[0] + }, + "-DPYTHON_VERSION=$pythonVersion", + "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON", + ) + + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + } + + ndkVersion = androidEnvFile.useLines { + for (line in it) { + """ndk_version=(\S+)""".toRegex().find(line)?.let { + return@useLines it.groupValues[1] + } + } + throw GradleException("Failed to find NDK version in $androidEnvFile") + } + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } + + // Set this property to something non-empty, otherwise it'll use the default + // list, which ignores asset directories beginning with an underscore. + aaptOptions.ignoreAssetsPattern = ".git" + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } + + testOptions { + managedDevices { + localDevices { + create("minVersion") { + device = "Small Phone" + + // Managed devices have a minimum API level of 27. + apiLevel = max(27, defaultConfig.minSdk!!) + + // ATD devices are smaller and faster, but have a minimum + // API level of 30. + systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" + } + + create("maxVersion") { + device = "Small Phone" + apiLevel = defaultConfig.targetSdk!! + systemImageSource = "aosp-atd" + } + } + + // If the previous test run succeeded and nothing has changed, + // Gradle thinks there's no need to run it again. Override that. + afterEvaluate { + (localDevices.names + listOf("connected")).forEach { + tasks.named("${it}DebugAndroidTest") { + outputs.upToDateWhen { false } + } + } + } + } + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") + androidTestImplementation("androidx.test.ext:junit:1.1.5") + androidTestImplementation("androidx.test:rules:1.5.0") +} + + +// Create some custom tasks to copy Python and its standard library from +// elsewhere in the repository. +androidComponents.onVariants { variant -> + val pyPlusVer = "python$pythonVersion" + generateTask(variant, variant.sources.assets!!) { + into("python") { + // Include files such as pyconfig.h are used by some of the tests. + into("include/$pyPlusVer") { + for (prefix in prefixes) { + from("$prefix/include/$pyPlusVer") + } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + + into("lib/$pyPlusVer") { + // To aid debugging, the source directory takes priority when + // running inside a CPython source tree. + if (inSourceTree) { + from("$PYTHON_DIR/Lib") + } + for (prefix in prefixes) { + from("$prefix/lib/$pyPlusVer") + } + + into("site-packages") { + from("$projectDir/src/main/python") + } + + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude("**/__pycache__") + } + } + } + + generateTask(variant, variant.sources.jniLibs!!) { + for ((prefix, abi) in abis.entries) { + into(abi) { + from("$prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } + } + } +} + + +fun generateTask( + variant: ApplicationVariant, directories: SourceDirectories, + configure: GenerateTask.() -> Unit +) { + val taskName = "generate" + + listOf(variant.name, "Python", directories.name) + .map { it.replaceFirstChar(Char::uppercase) } + .joinToString("") + + directories.addGeneratedSourceDirectory( + tasks.register<GenerateTask>(taskName) { + into(outputDir) + configure() + }, + GenerateTask::outputDir) +} + + +// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. +abstract class GenerateTask: Sync() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty +} diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt new file mode 100644 index 00000000000000..0e888ab71d87da --- /dev/null +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -0,0 +1,35 @@ +package org.python.testbed + +import androidx.test.annotation.UiThreadTest +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.ext.junit.runners.AndroidJUnit4 + +import org.junit.Test +import org.junit.runner.RunWith + +import org.junit.Assert.* + + +@RunWith(AndroidJUnit4::class) +class PythonSuite { + @Test + @UiThreadTest + fun testPython() { + val start = System.currentTimeMillis() + try { + val context = + InstrumentationRegistry.getInstrumentation().targetContext + val args = + InstrumentationRegistry.getArguments().getString("pythonArgs", "") + val status = PythonTestRunner(context).run(args) + assertEquals(0, status) + } finally { + // Make sure the process lives long enough for the test script to + // detect it (see `find_pid` in android.py). + val delay = 2000 - (System.currentTimeMillis() - start) + if (delay > 0) { + Thread.sleep(delay) + } + } + } +} diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000000000..2be8a82d426099 --- /dev/null +++ b/Android/testbed/app/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="utf-8"?> +<manifest xmlns:android="http://schemas.android.com/apk/res/android"> + + <uses-permission android:name="android.permission.INTERNET"/> + + <application + android:icon="@drawable/ic_launcher" + android:label="@string/app_name" + android:theme="@style/Theme.Material3.Light.NoActionBar"> + <activity + android:name=".MainActivity" + android:exported="true"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + </application> + +</manifest> \ No newline at end of file diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt new file mode 100644 index 00000000000000..6d5ccd96f8ae29 --- /dev/null +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -0,0 +1,14 @@ +cmake_minimum_required(VERSION 3.4.1) +project(testbed) + +# Resolve variables from the command line. +string( + REPLACE {{triplet}} ${CMAKE_LIBRARY_ARCHITECTURE} + PYTHON_PREFIX_DIR ${PYTHON_PREFIX_DIR} +) + +include_directories(${PYTHON_PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PYTHON_PREFIX_DIR}/lib) +link_libraries(log python${PYTHON_VERSION}) + +add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c new file mode 100644 index 00000000000000..ec7f93a3e5ee13 --- /dev/null +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -0,0 +1,152 @@ +#include <android/log.h> +#include <errno.h> +#include <jni.h> +#include <pthread.h> +#include <Python.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> + + +static void throw_runtime_exception(JNIEnv *env, const char *message) { + (*env)->ThrowNew( + env, + (*env)->FindClass(env, "java/lang/RuntimeException"), + message); +} + + +// --- Stdio redirection ------------------------------------------------------ + +// Most apps won't need this, because the Python-level sys.stdout and sys.stderr +// are redirected to the Android logcat by Python itself. However, in the +// testbed it's useful to redirect the native streams as well, to debug problems +// in the Python startup or redirection process. +// +// Based on +// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp + +typedef struct { + FILE *file; + int fd; + android_LogPriority priority; + char *tag; + int pipe[2]; +} StreamInfo; + +// The FILE member can't be initialized here because stdout and stderr are not +// compile-time constants. Instead, it's initialized immediately before the +// redirection. +static StreamInfo STREAMS[] = { + {NULL, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, + {NULL, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, + {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, +}; + +// The maximum length of a log message in bytes, including the level marker and +// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +// longer than this will be be truncated by logcat. This limit has already been +// reduced at least once in the history of Android (from 4076 to 4068 between API +// level 23 and 26), so leave some headroom. +static const int MAX_BYTES_PER_WRITE = 4000; + +static void *redirection_thread(void *arg) { + StreamInfo *si = (StreamInfo*)arg; + ssize_t read_size; + char buf[MAX_BYTES_PER_WRITE]; + while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { + buf[read_size] = '\0'; /* add null-terminator */ + __android_log_write(si->priority, si->tag, buf); + } + return 0; +} + +static char *redirect_stream(StreamInfo *si) { + /* make the FILE unbuffered, to ensure messages are never lost */ + if (setvbuf(si->file, 0, _IONBF, 0)) { + return "setvbuf"; + } + + /* create the pipe and redirect the file descriptor */ + if (pipe(si->pipe)) { + return "pipe"; + } + if (dup2(si->pipe[1], si->fd) == -1) { + return "dup2"; + } + + /* start the logging thread */ + pthread_t thr; + if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { + return "pthread_create"; + } + if ((errno = pthread_detach(thr))) { + return "pthread_detach"; + } + return 0; +} + +JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToLogcat( + JNIEnv *env, jobject obj +) { + STREAMS[0].file = stdout; + STREAMS[1].file = stderr; + for (StreamInfo *si = STREAMS; si->file; si++) { + char *error_prefix; + if ((error_prefix = redirect_stream(si))) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); + return; + } + } +} + + +// --- Python initialization --------------------------------------------------- + +static PyStatus set_config_string( + JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value +) { + const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); + PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); + (*env)->ReleaseStringUTFChars(env, value, value_utf8); + return status; +} + +static void throw_status(JNIEnv *env, PyStatus status) { + throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); +} + +JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( + JNIEnv *env, jobject obj, jstring home, jstring runModule +) { + PyConfig config; + PyStatus status; + PyConfig_InitIsolatedConfig(&config); + + status = set_config_string(env, &config, &config.home, home); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + status = set_config_string(env, &config, &config.run_module, runModule); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. + config.install_signal_handlers = 1; + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return 1; + } + + return Py_RunMain(); +} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt new file mode 100644 index 00000000000000..c4bf6cbe83d8cd --- /dev/null +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -0,0 +1,79 @@ +package org.python.testbed + +import android.content.Context +import android.os.* +import android.system.Os +import android.widget.TextView +import androidx.appcompat.app.* +import java.io.* + + +// Launching the tests from an activity is OK for a quick check, but for +// anything more complicated it'll be more convenient to use `android.py test` +// to launch the tests via PythonSuite. +class MainActivity : AppCompatActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + val status = PythonTestRunner(this).run("-W -uall") + findViewById<TextView>(R.id.tvHello).text = "Exit status $status" + } +} + + +class PythonTestRunner(val context: Context) { + /** @param args Extra arguments for `python -m test`. + * @return The Python exit status: zero if the tests passed, nonzero if + * they failed. */ + fun run(args: String = "") : Int { + Os.setenv("PYTHON_ARGS", args, true) + + // Python needs this variable to help it find the temporary directory, + // but Android only sets it on API level 33 and later. + Os.setenv("TMPDIR", context.cacheDir.toString(), false) + + val pythonHome = extractAssets() + System.loadLibrary("main_activity") + redirectStdioToLogcat() + + // The main module is in src/main/python/main.py. + return runPython(pythonHome.toString(), "main") + } + + private fun extractAssets() : File { + val pythonHome = File(context.filesDir, "python") + if (pythonHome.exists() && !pythonHome.deleteRecursively()) { + throw RuntimeException("Failed to delete $pythonHome") + } + extractAssetDir("python", context.filesDir) + return pythonHome + } + + private fun extractAssetDir(path: String, targetDir: File) { + val names = context.assets.list(path) + ?: throw RuntimeException("Failed to list $path") + val targetSubdir = File(targetDir, path) + if (!targetSubdir.mkdirs()) { + throw RuntimeException("Failed to create $targetSubdir") + } + + for (name in names) { + val subPath = "$path/$name" + val input: InputStream + try { + input = context.assets.open(subPath) + } catch (e: FileNotFoundException) { + extractAssetDir(subPath, targetDir) + continue + } + input.use { + File(targetSubdir, name).outputStream().use { output -> + input.copyTo(output) + } + } + } + } + + private external fun redirectStdioToLogcat() + private external fun runPython(home: String, runModule: String) : Int +} diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py new file mode 100644 index 00000000000000..d6941b14412fcc --- /dev/null +++ b/Android/testbed/app/src/main/python/main.py @@ -0,0 +1,32 @@ +import os +import runpy +import shlex +import signal +import sys + +# Some tests use SIGUSR1, but that's blocked by default in an Android app in +# order to make it available to `sigwait` in the Signal Catcher thread. +# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). +# That thread's functionality is only useful for debugging the JVM, so disabling +# it should not weaken the tests. +# +# There's no safe way of stopping the thread completely (#123982), but simply +# unblocking SIGUSR1 is enough to fix most tests. +# +# However, in tests that generate multiple different signals in quick +# succession, it's possible for SIGUSR1 to arrive while the main thread is busy +# running the C-level handler for a different signal. In that case, the SIGUSR1 +# may be sent to the Signal Catcher thread instead, which will generate a log +# message containing the text "reacting to signal". +# +# Such tests may need to be changed in one of the following ways: +# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in +# test_signal.py). +# * Send the signal to a specific thread rather than the whole process (e.g. +# test_signals in test_threadsignals.py. +signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) + +sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) + +# The test module will call sys.exit to indicate whether the tests passed. +runpy.run_module("test") diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 00000000000000..741d6580d60e05 Binary files /dev/null and b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png differ diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml new file mode 100644 index 00000000000000..21398609ec9c78 --- /dev/null +++ b/Android/testbed/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ +<?xml version="1.0" encoding="utf-8"?> +<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android" + xmlns:app="http://schemas.android.com/apk/res-auto" + xmlns:tools="http://schemas.android.com/tools" + android:layout_width="match_parent" + android:layout_height="match_parent" + tools:context=".MainActivity"> + + <TextView + android:id="@+id/tvHello" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="Hello World!" + app:layout_constraintBottom_toBottomOf="parent" + app:layout_constraintEnd_toEndOf="parent" + app:layout_constraintStart_toStartOf="parent" + app:layout_constraintTop_toTopOf="parent" /> + +</androidx.constraintlayout.widget.ConstraintLayout> \ No newline at end of file diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Android/testbed/app/src/main/res/values/strings.xml new file mode 100644 index 00000000000000..352d2f9e885a2a --- /dev/null +++ b/Android/testbed/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ +<resources> + <string name="app_name">Python testbed</string> +</resources> \ No newline at end of file diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts new file mode 100644 index 00000000000000..4d1d6f87594da3 --- /dev/null +++ b/Android/testbed/build.gradle.kts @@ -0,0 +1,5 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.6.1" apply false + id("org.jetbrains.kotlin.android") version "1.9.22" apply false +} diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties new file mode 100644 index 00000000000000..e9f345c8c26250 --- /dev/null +++ b/Android/testbed/gradle.properties @@ -0,0 +1,28 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true + +# By default, the app will be uninstalled after the tests finish (apparently +# after 10 seconds in case of an unclean shutdown). We disable this, because +# when using android.py it can conflict with the installation of the next run. +android.injected.androidTest.leaveApksInstalledAfterRun=true diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000000000..36529c896426b0 --- /dev/null +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Feb 19 20:29:06 GMT 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/Android/testbed/settings.gradle.kts b/Android/testbed/settings.gradle.kts new file mode 100644 index 00000000000000..5e08773e02450f --- /dev/null +++ b/Android/testbed/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "Python testbed" +include(":app") + \ No newline at end of file diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml new file mode 100644 index 00000000000000..3e676e13c3f41a --- /dev/null +++ b/Doc/.ruff.toml @@ -0,0 +1,41 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +target-version = "py312" # Align with the version in oldest_supported_sphinx +extend-exclude = [ + "includes/*", + # Temporary exclusions: + "tools/extensions/pyspecific.py", +] + +[lint] +preview = true +select = [ + "C4", # flake8-comprehensions + "B", # flake8-bugbear + "E", # pycodestyle + "F", # pyflakes + "FA", # flake8-future-annotations + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format + "I", # isort + "LOG", # flake8-logging + "N", # pep8-naming + "PERF", # perflint + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "TCH", # flake8-type-checking + "UP", # pyupgrade + "W", # pycodestyle +] +ignore = [ + "E501", # Ignore line length errors (we use auto-formatting) +] + +[format] +preview = true +quote-style = "preserve" +docstring-code-format = true +exclude = [ + "tools/extensions/lexers/*", +] diff --git a/Doc/Makefile b/Doc/Makefile index 22691895068fea..c8a749a02a89ec 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -6,22 +6,26 @@ # You can set these variables from the command line. PYTHON = python3 VENVDIR = ./venv +UV = uv SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build -SPHINXLINT = PATH=$(VENVDIR)/bin:$$PATH sphinx-lint BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb JOBS = auto PAPER = SOURCES = DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) REQUIREMENTS = requirements.txt -SPHINXERRORHANDLING = -W +SPHINXERRORHANDLING = --fail-on-warning # Internal variables. -PAPEROPT_a4 = -D latex_elements.papersize=a4paper -PAPEROPT_letter = -D latex_elements.papersize=letterpaper +PAPEROPT_a4 = --define latex_elements.papersize=a4paper +PAPEROPT_letter = --define latex_elements.papersize=letterpaper -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) +ALLSPHINXOPTS = --builder $(BUILDER) \ + --doctree-dir build/doctrees \ + --jobs $(JOBS) \ + $(PAPEROPT_$(PAPER)) \ + $(SPHINXOPTS) $(SPHINXERRORHANDLING) \ + . build/$(BUILDER) $(SOURCES) .PHONY: help help: @@ -29,7 +33,9 @@ help: @echo " clean to remove build files" @echo " venv to create a venv with necessary tools" @echo " html to make standalone HTML files" + @echo " gettext to generate POT files" @echo " htmlview to open the index page built by the html target in your browser" + @echo " htmllive to rebuild and reload HTML files in your browser" @echo " htmlhelp to make HTML files and a HTML help project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " text to make plain text files" @@ -136,10 +142,20 @@ pydoc-topics: build @echo "Building finished; now run this:" \ "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" +.PHONY: gettext +gettext: BUILDER = gettext +gettext: override SPHINXOPTS := --doctree-dir build/doctrees-gettext $(SPHINXOPTS) +gettext: build + .PHONY: htmlview htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" +.PHONY: htmllive +htmllive: SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-autobuild +htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 +htmllive: _ensure-sphinx-autobuild html + .PHONY: clean clean: clean-venv -rm -rf build/* @@ -154,92 +170,151 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ - $(PYTHON) -m venv $(VENVDIR); \ - $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ - $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + echo "Creating venv in $(VENVDIR)"; \ + if $(UV) --version >/dev/null 2>&1; then \ + $(UV) venv --python=$(PYTHON) $(VENVDIR); \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install -r $(REQUIREMENTS); \ + else \ + $(PYTHON) -m venv $(VENVDIR); \ + $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ + $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ + fi; \ echo "The venv has been created in the $(VENVDIR) directory"; \ fi +.PHONY: dist-no-html +dist-no-html: dist-text dist-pdf dist-epub dist-texinfo + .PHONY: dist dist: rm -rf dist mkdir -p dist - + $(MAKE) dist-html + $(MAKE) dist-text + $(MAKE) dist-pdf + $(MAKE) dist-epub + $(MAKE) dist-texinfo + +.PHONY: dist-html +dist-html: # archive the HTML - make html + @echo "Building HTML..." + mkdir -p dist + rm -rf build/html + find dist -name 'python-$(DISTVERSION)-docs-html*' -exec rm -rf {} \; + $(MAKE) html cp -pPR build/html dist/python-$(DISTVERSION)-docs-html + rm -rf dist/python-$(DISTVERSION)-docs-html/_images/social_previews/ tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) rm -r dist/python-$(DISTVERSION)-docs-html rm dist/python-$(DISTVERSION)-docs-html.tar + @echo "Build finished and archived!" +.PHONY: dist-text +dist-text: # archive the text build - make text + @echo "Building text..." + mkdir -p dist + rm -rf build/text + find dist -name 'python-$(DISTVERSION)-docs-text*' -exec rm -rf {} \; + $(MAKE) text cp -pPR build/text dist/python-$(DISTVERSION)-docs-text tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) rm -r dist/python-$(DISTVERSION)-docs-text rm dist/python-$(DISTVERSION)-docs-text.tar + @echo "Build finished and archived!" +.PHONY: dist-pdf +dist-pdf: # archive the A4 latex + @echo "Building LaTeX (A4 paper)..." + mkdir -p dist rm -rf build/latex - make latex PAPER=a4 - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) + find dist -name 'python-$(DISTVERSION)-docs-pdf*' -exec rm -rf {} \; + $(MAKE) latex PAPER=a4 + # remove zip & bz2 dependency on all-pdf, + # as otherwise the full latexmk process is run twice. + # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) + -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 + @echo "Build finished and archived!" - # archive the letter latex - rm -rf build/latex - make latex PAPER=letter - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) - cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip - cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - +.PHONY: dist-epub +dist-epub: # copy the epub build + @echo "Building EPUB..." + mkdir -p dist rm -rf build/epub - make epub + rm -f dist/python-$(DISTVERSION)-docs.epub + $(MAKE) epub cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub + @echo "Build finished and archived!" +.PHONY: dist-texinfo +dist-texinfo: # archive the texinfo build + @echo "Building Texinfo..." + mkdir -p dist rm -rf build/texinfo - make texinfo - make info --directory=build/texinfo + find dist -name 'python-$(DISTVERSION)-docs-texinfo*' -exec rm -rf {} \; + $(MAKE) texinfo + $(MAKE) info --directory=build/texinfo cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) rm -r dist/python-$(DISTVERSION)-docs-texinfo rm dist/python-$(DISTVERSION)-docs-texinfo.tar + @echo "Build finished and archived!" + +.PHONY: _ensure-package +_ensure-package: venv + if $(UV) --version >/dev/null 2>&1; then \ + VIRTUAL_ENV=$(VENVDIR) $(UV) pip install $(PACKAGE); \ + else \ + $(VENVDIR)/bin/python3 -m pip install $(PACKAGE); \ + fi + +.PHONY: _ensure-pre-commit +_ensure-pre-commit: + $(MAKE) _ensure-package PACKAGE=pre-commit + +.PHONY: _ensure-sphinx-autobuild +_ensure-sphinx-autobuild: + $(MAKE) _ensure-package PACKAGE=sphinx-autobuild .PHONY: check -check: venv - $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit +check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files .PHONY: serve serve: - @echo "The serve target was removed, use htmlview instead (see bpo-36329)" + @echo "The serve target was removed, use htmllive instead (see gh-80510)" # Targets for daily automated doc build # By default, Sphinx only rebuilds pages where the page content has changed. # This means it doesn't always pick up changes to preferred link targets, etc # To ensure such changes are picked up, we build the published docs with -# `-E` (to ignore the cached environment) and `-a` (to ignore already existing -# output files) +# ``--fresh-env`` (to ignore the cached environment) and ``--write-all`` +# (to ignore already existing output files) # for development releases: always build .PHONY: autobuild-dev +autobuild-dev: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-no-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION) -# for quick rebuilds (HTML only) +# for HTML-only rebuilds .PHONY: autobuild-dev-html +autobuild-dev-html: DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py --short) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' + $(MAKE) dist-html SPHINXOPTS='$(SPHINXOPTS) --fresh-env --write-all --html-define daily=1' DISTVERSION=$(DISTVERSION) # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage @@ -249,7 +324,7 @@ autobuild-stable: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev + @$(MAKE) autobuild-dev .PHONY: autobuild-stable-html autobuild-stable-html: @@ -257,4 +332,4 @@ autobuild-stable-html: echo "Not building; $(DISTVERSION) is not a release version."; \ exit 1;; \ esac - @make autobuild-dev-html + @$(MAKE) autobuild-dev-html diff --git a/Doc/README.rst b/Doc/README.rst index a3bb5fa5445c23..2d1148753e0c6b 100644 --- a/Doc/README.rst +++ b/Doc/README.rst @@ -28,7 +28,7 @@ install the tools into there. Using make ---------- -To get started on UNIX, you can create a virtual environment and build +To get started on Unix, you can create a virtual environment and build documentation with the commands:: make venv @@ -40,13 +40,13 @@ If you'd like to create the virtual environment in a different location, you can specify it using the ``VENVDIR`` variable. You can also skip creating the virtual environment altogether, in which case -the Makefile will look for instances of ``sphinx-build`` and ``blurb`` +the ``Makefile`` will look for instances of ``sphinx-build`` and ``blurb`` installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and ``BLURB`` variables). -On Windows, we try to emulate the Makefile as closely as possible with a +On Windows, we try to emulate the ``Makefile`` as closely as possible with a ``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable. +set the ``PYTHON`` environment variable. Available make targets are: @@ -62,15 +62,19 @@ Available make targets are: * "htmlview", which re-uses the "html" builder, but then opens the main page in your default web browser. +* "htmllive", which re-uses the "html" builder, rebuilds the docs, + starts a local server, and automatically reloads the page in your browser + when you make changes to reST files (Unix only). + * "htmlhelp", which builds HTML files and a HTML Help project file usable to convert them into a single Compiled HTML (.chm) file -- these are popular under Microsoft Windows, but very handy on every platform. To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for + over the generated project (.hhp) file. The ``make.bat`` script does this for you on Windows. -* "latex", which builds LaTeX source files as input to "pdflatex" to produce +* "latex", which builds LaTeX source files as input to ``pdflatex`` to produce PDF documents. * "text", which builds a plain text file for each source file. @@ -95,8 +99,6 @@ Available make targets are: * "check", which checks for frequent markup errors. -* "serve", which serves the build/html directory on port 8000. - * "dist", (Unix only) which creates distributable archives of HTML, text, PDF, and EPUB builds. @@ -131,8 +133,5 @@ Bugs in the content should be reported to the Bugs in the toolset should be reported to the tools themselves. -You can also send a mail to the Python Documentation Team at docs@python.org, -and we will process your request as soon as possible. - -If you want to help the Documentation Team, you are always welcome. Just send -a mail to docs@python.org. +To help with the documentation, or report any problems, please leave a message +on `discuss.python.org <https://discuss.python.org/c/documentation>`_. diff --git a/Doc/about.rst b/Doc/about.rst index 5e6160ff2700ed..8f635d7f743a98 100644 --- a/Doc/about.rst +++ b/Doc/about.rst @@ -1,10 +1,11 @@ -===================== -About these documents -===================== +======================== +About this documentation +======================== -These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a -document processor specifically written for the Python documentation. +Python's documentation is generated from `reStructuredText`_ sources +using `Sphinx`_, a documentation generator originally created for Python +and now maintained as an independent project. .. _reStructuredText: https://docutils.sourceforge.io/rst.html .. _Sphinx: https://www.sphinx-doc.org/ @@ -20,14 +21,14 @@ volunteers are always welcome! Many thanks go to: * Fred L. Drake, Jr., the creator of the original Python documentation toolset - and writer of much of the content; + and author of much of the content; * the `Docutils <https://docutils.sourceforge.io/>`_ project for creating reStructuredText and the Docutils suite; * Fredrik Lundh for his Alternative Python Reference project from which Sphinx got many good ideas. -Contributors to the Python Documentation +Contributors to the Python documentation ---------------------------------------- Many people have contributed to the Python language, the Python standard diff --git a/Doc/bugs.rst b/Doc/bugs.rst index d98192b369603e..5d0f68ca69675e 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -16,16 +16,15 @@ Documentation bugs ================== If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker <using-the-tracker>`. If you +please submit a bug report on the :ref:`issue tracker <using-the-tracker>`. If you have a suggestion on how to fix it, include that as well. You can also open a discussion item on our `Documentation Discourse forum <https://discuss.python.org/c/documentation/26>`_. -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. +If you find a bug in the theme (HTML / CSS / JavaScript) of the +documentation, please submit a bug report on the `python-doc-theme issue +tracker <https://github.com/python/python-docs-theme>`_. .. seealso:: @@ -38,7 +37,7 @@ though it may take a while to be processed. `Helping with Documentation <https://devguide.python.org/docquality/#helping-with-documentation>`_ Comprehensive guide for individuals that are interested in contributing to Python documentation. - `Documentation Translations <https://devguide.python.org/documenting/#translating>`_ + `Documentation Translations <https://devguide.python.org/documentation/translating/>`_ A list of GitHub pages for documentation translation and their primary contacts. diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index b3609c233156b6..59d913a0462382 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -15,10 +15,21 @@ Allocating Objects on the Heap .. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) Initialize a newly allocated object *op* with its type and initial - reference. Returns the initialized object. If *type* indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected. + reference. Returns the initialized object. Other fields of the object are + not initialized. Despite its name, this function is unrelated to the + object's :meth:`~object.__init__` method (:c:member:`~PyTypeObject.tp_init` + slot). Specifically, this function does **not** call the object's + :meth:`!__init__` method. + + In general, consider this function to be a low-level routine. Use + :c:member:`~PyTypeObject.tp_alloc` where possible. + For implementing :c:member:`!tp_alloc` for your type, prefer + :c:func:`PyType_GenericAlloc` or :c:func:`PyObject_New`. + + .. note:: + + This function only initializes the object's memory corresponding to the + initial :c:type:`PyObject` structure. It does not zero the rest. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) @@ -26,40 +37,112 @@ Allocating Objects on the Heap This does everything :c:func:`PyObject_Init` does, and also initializes the length information for a variable-size object. + .. note:: + + This function only initializes some of the object's memory. It does not + zero the rest. + .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* - and the Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header are not initialized. - The caller will own the only reference to the object - (i.e. its reference count will be one). - The size of the memory allocation is determined from the - :c:member:`~PyTypeObject.tp_basicsize` field of the type object. + Allocates a new Python object using the C structure type *TYPE* and the + Python type object *typeobj* (``PyTypeObject*``) by calling + :c:func:`PyObject_Malloc` to allocate memory and initializing it like + :c:func:`PyObject_Init`. The caller will own the only reference to the + object (i.e. its reference count will be one). + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This macro does not call :c:member:`~PyTypeObject.tp_alloc`, + :c:member:`~PyTypeObject.tp_new` (:meth:`~object.__new__`), or + :c:member:`~PyTypeObject.tp_init` (:meth:`~object.__init__`). + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_New` instead. + + Memory allocated by this macro must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *foo = PyObject_CallNoArgs((PyObject *)&PyFoo_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* (``Py_ssize_t``) fields of the size - given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *typeobj*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. + Like :c:macro:`PyObject_New` except: + * It allocates enough memory for the *TYPE* structure plus *size* + (``Py_ssize_t``) fields of the size given by the + :c:member:`~PyTypeObject.tp_itemsize` field of *typeobj*. + * The memory is initialized like :c:func:`PyObject_InitVar`. -.. c:function:: void PyObject_Del(void *op) + This is useful for implementing objects like tuples, which are able to + determine their size at construction time. Embedding the array of fields + into the same allocation decreases the number of allocations, improving the + memory management efficiency. + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_NewVar` + instead. + + Memory allocated by this function must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. - Releases memory allocated to an object using :c:macro:`PyObject_New` or - :c:macro:`PyObject_NewVar`. This is normally called from the - :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of - the object should not be accessed after this call as the memory is no - longer a valid Python object. + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *list_instance = PyObject_CallNoArgs((PyObject *)&PyList_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + +.. c:function:: void PyObject_Del(void *op) + Same as :c:func:`PyObject_Free`. .. c:var:: PyObject _Py_NoneStruct @@ -70,6 +153,6 @@ Allocating Objects on the Heap .. seealso:: - :c:func:`PyModule_Create` + :ref:`moduleobjects` To allocate and create extension modules. diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index f6c8284daeacb0..96050f59bd5250 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -6,9 +6,13 @@ API and ABI Versioning *********************** + +Build-time version constants +---------------------------- + CPython exposes its version number in the following macros. -Note that these correspond to the version code is **built** with, -not necessarily the version used at **run time**. +Note that these correspond to the version code is **built** with. +See :c:var:`Py_Version` for the version used at **run time**. See :ref:`stable` for a discussion of API and ABI stability across versions. @@ -37,37 +41,83 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. .. c:macro:: PY_VERSION_HEX The Python version number encoded in a single integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. - The underlying version information can be found by treating it as a 32 bit - number in the following manner: - - +-------+-------------------------+-------------------------+--------------------------+ - | Bytes | Bits (big endian order) | Meaning | Value for ``3.4.1a2`` | - +=======+=========================+=========================+==========================+ - | 1 | 1-8 | ``PY_MAJOR_VERSION`` | ``0x03`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 2 | 9-16 | ``PY_MINOR_VERSION`` | ``0x04`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 3 | 17-24 | ``PY_MICRO_VERSION`` | ``0x01`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 4 | 25-28 | ``PY_RELEASE_LEVEL`` | ``0xA`` | - + +-------------------------+-------------------------+--------------------------+ - | | 29-32 | ``PY_RELEASE_SERIAL`` | ``0x2`` | - +-------+-------------------------+-------------------------+--------------------------+ + Use this for numeric comparisons, for example, + ``#if PY_VERSION_HEX >= ...``. - Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is - hexversion ``0x030a00f0``. - Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. - - This version is also available via the symbol :c:var:`Py_Version`. +Run-time version +---------------- .. c:var:: const unsigned long Py_Version - The Python runtime version number encoded in a single constant integer, with - the same format as the :c:macro:`PY_VERSION_HEX` macro. + The Python runtime version number encoded in a single constant integer. + See :c:func:`Py_PACK_FULL_VERSION` for the encoding details. This contains the Python version used at run time. + Use this for numeric comparisons, for example, ``if (Py_Version >= ...)``. + .. versionadded:: 3.11 -All the given macros are defined in :source:`Include/patchlevel.h`. + +Bit-packing macros +------------------ + +.. c:function:: uint32_t Py_PACK_FULL_VERSION(int major, int minor, int micro, int release_level, int release_serial) + + Return the given version, encoded as a single 32-bit integer with + the following structure: + + +------------------+-------+----------------+-----------+--------------------------+ + | | No. | | | Example values | + | | of | | +-------------+------------+ + | Argument | bits | Bit mask | Bit shift | ``3.4.1a2`` | ``3.10.0`` | + +==================+=======+================+===========+=============+============+ + | *major* | 8 | ``0xFF000000`` | 24 | ``0x03`` | ``0x03`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *minor* | 8 | ``0x00FF0000`` | 16 | ``0x04`` | ``0x0A`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *micro* | 8 | ``0x0000FF00`` | 8 | ``0x01`` | ``0x00`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_level* | 4 | ``0x000000F0`` | 4 | ``0xA`` | ``0xF`` | + +------------------+-------+----------------+-----------+-------------+------------+ + | *release_serial* | 4 | ``0x0000000F`` | 0 | ``0x2`` | ``0x0`` | + +------------------+-------+----------------+-----------+-------------+------------+ + + For example: + + +-------------+------------------------------------+-----------------+ + | Version | ``Py_PACK_FULL_VERSION`` arguments | Encoded version | + +=============+====================================+=================+ + | ``3.4.1a2`` | ``(3, 4, 1, 0xA, 2)`` | ``0x030401a2`` | + +-------------+------------------------------------+-----------------+ + | ``3.10.0`` | ``(3, 10, 0, 0xF, 0)`` | ``0x030a00f0`` | + +-------------+------------------------------------+-----------------+ + + Out-of range bits in the arguments are ignored. + That is, the macro can be defined as: + + .. code-block:: c + + #ifndef Py_PACK_FULL_VERSION + #define Py_PACK_FULL_VERSION(X, Y, Z, LEVEL, SERIAL) ( \ + (((X) & 0xff) << 24) | \ + (((Y) & 0xff) << 16) | \ + (((Z) & 0xff) << 8) | \ + (((LEVEL) & 0xf) << 4) | \ + (((SERIAL) & 0xf) << 0)) + #endif + + ``Py_PACK_FULL_VERSION`` is primarily a macro, intended for use in + ``#if`` directives, but it is also available as an exported function. + + .. versionadded:: 3.14 + +.. c:function:: uint32_t Py_PACK_VERSION(int major, int minor) + + Equivalent to ``Py_PACK_FULL_VERSION(major, minor, 0, 0, 0)``. + The result does not correspond to any Python release, but is useful + in numeric comparisons. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index c43dd0f4303cd4..3bbc990b6329c0 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -5,7 +5,7 @@ Parsing arguments and building values ===================================== -These functions are useful when creating your own extensions functions and +These functions are useful when creating your own extension functions and methods. Additional information and examples are available in :ref:`extending-index`. @@ -113,14 +113,18 @@ There are three ways strings and buffers can be converted to C: ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. + It is the same as ``s?`` with the C pointer was initialized to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. + It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer` + structure was initialized to ``NULL``. ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. + It is the same as ``s#?`` with the C pointer was initialized to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a @@ -229,12 +233,24 @@ There are three ways strings and buffers can be converted to C: Numbers ------- +These formats allow representing Python numbers or single characters as C numbers. +Formats that require :class:`int`, :class:`float` or :class:`complex` can +also use the corresponding special methods :meth:`~object.__index__`, +:meth:`~object.__float__` or :meth:`~object.__complex__` to convert +the Python object to the required type. + +For signed integer formats, :exc:`OverflowError` is raised if the value +is out of range for the C type. +For unsigned integer formats, no range checking is done --- the +most significant bits are silently truncated when the receiving field is too +small to receive the value. + ``b`` (:class:`int`) [unsigned char] - Convert a nonnegative Python integer to an unsigned tiny int, stored in a C + Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C :c:expr:`unsigned char`. ``B`` (:class:`int`) [unsigned char] - Convert a Python integer to a tiny int without overflow checking, stored in a C + Convert a Python integer to a tiny integer without overflow checking, stored in a C :c:expr:`unsigned char`. ``h`` (:class:`int`) [short int] @@ -258,6 +274,9 @@ Numbers Convert a Python integer to a C :c:expr:`unsigned long` without overflow checking. + .. versionchanged:: 3.14 + Use :meth:`~object.__index__` if available. + ``L`` (:class:`int`) [long long] Convert a Python integer to a C :c:expr:`long long`. @@ -265,6 +284,9 @@ Numbers Convert a Python integer to a C :c:expr:`unsigned long long` without overflow checking. + .. versionchanged:: 3.14 + Use :meth:`~object.__index__` if available. + ``n`` (:class:`int`) [:c:type:`Py_ssize_t`] Convert a Python integer to a C :c:type:`Py_ssize_t`. @@ -280,10 +302,10 @@ Numbers length 1, to a C :c:expr:`int`. ``f`` (:class:`float`) [float] - Convert a Python floating point number to a C :c:expr:`float`. + Convert a Python floating-point number to a C :c:expr:`float`. ``d`` (:class:`float`) [double] - Convert a Python floating point number to a C :c:expr:`double`. + Convert a Python floating-point number to a C :c:expr:`double`. ``D`` (:class:`complex`) [Py_complex] Convert a Python complex number to a C :c:type:`Py_complex` structure. @@ -307,7 +329,7 @@ Other objects .. _o_ampersand: -``O&`` (object) [*converter*, *anything*] +``O&`` (object) [*converter*, *address*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C variable (of arbitrary type), converted to :c:expr:`void *`. The *converter* @@ -321,14 +343,20 @@ Other objects the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. - If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a + .. c:macro:: Py_CLEANUP_SUPPORTED + :no-typesetting: + + If the *converter* returns :c:macro:`!Py_CLEANUP_SUPPORTED`, it may get called a second time if the argument parsing eventually fails, giving the converter a chance to release any memory that it had already allocated. In this second call, the *object* parameter will be ``NULL``; *address* will have the same value as in the original call. + Examples of converters: :c:func:`PyUnicode_FSConverter` and + :c:func:`PyUnicode_FSDecoder`. + .. versionchanged:: 3.1 - ``Py_CLEANUP_SUPPORTED`` was added. + :c:macro:`!Py_CLEANUP_SUPPORTED` was added. ``p`` (:class:`bool`) [int] Tests the value passed in for truth (a boolean **p**\ redicate) and converts @@ -339,16 +367,36 @@ Other objects .. versionadded:: 3.3 -``(items)`` (:class:`tuple`) [*matching-items*] - The object must be a Python sequence whose length is the number of format units +``(items)`` (sequence) [*matching-items*] + The object must be a Python sequence (except :class:`str`, :class:`bytes` + or :class:`bytearray`) whose length is the number of format units in *items*. The C arguments must correspond to the individual format units in *items*. Format units for sequences may be nested. -It is possible to pass "long" integers (integers whose value exceeds the -platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the -most significant bits are silently truncated when the receiving field is too -small to receive the value (actually, the semantics are inherited from downcasts -in C --- your mileage may vary). + If *items* contains format units which store a :ref:`borrowed buffer + <c-arg-borrowed-buffer>` (``s``, ``s#``, ``z``, ``z#``, ``y``, or ``y#``) + or a :term:`borrowed reference` (``S``, ``Y``, ``U``, ``O``, or ``O!``), + the object must be a Python tuple. + The *converter* for the ``O&`` format unit in *items* must not store + a borrowed buffer or a borrowed reference. + + .. versionchanged:: 3.14 + :class:`str` and :class:`bytearray` objects no longer accepted as a sequence. + + .. deprecated:: 3.14 + Non-tuple sequences are deprecated if *items* contains format units + which store a borrowed buffer or a borrowed reference. + +``unit?`` (anything or ``None``) [*matching-variable(s)*] + ``?`` modifies the behavior of the preceding format unit. + The C variable(s) corresponding to that parameter should be initialized + to their default value --- when the argument is ``None``, + :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding + C variable(s). + If the argument is not ``None``, it is parsed according to the specified + format unit. + + .. versionadded:: 3.14 A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: @@ -413,21 +461,35 @@ API Functions than a variable number of arguments. -.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) +.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, ...) Parse the parameters of a function that takes both positional and keyword - parameters into local variables. The *keywords* argument is a - ``NULL``-terminated array of keyword parameter names. Empty names denote + parameters into local variables. + The *keywords* argument is a ``NULL``-terminated array of keyword parameter + names specified as null-terminated ASCII or UTF-8 encoded C strings. + Empty names denote :ref:`positional-only parameters <positional-only_parameter>`. Returns true on success; on failure, it returns false and raises the appropriate exception. + .. note:: + + The *keywords* parameter declaration is :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + .. versionchanged:: 3.6 Added support for :ref:`positional-only parameters <positional-only_parameter>`. + .. versionchanged:: 3.13 + The *keywords* parameter has now type :c:expr:`char * const *` in C and + :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. + Added support for non-ASCII keyword parameter names. + + -.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) +.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char * const *keywords, va_list vargs) Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a va_list rather than a variable number of arguments. @@ -500,6 +562,19 @@ API Functions PyArg_ParseTuple(args, "O|O:ref", &object, &callback) +.. c:macro:: PY_CXX_CONST + + The value to be inserted, if any, before :c:expr:`char * const *` + in the *keywords* parameter declaration of + :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords`. + Default empty for C and ``const`` for C++ + (:c:expr:`const char * const *`). + To override, define it to the desired value before including + :file:`Python.h`. + + .. versionadded:: 3.13 + --------------- Building values @@ -600,12 +675,18 @@ Building values ``L`` (:class:`int`) [long long] Convert a C :c:expr:`long long` to a Python integer object. + .. _capi-py-buildvalue-format-K: + ``K`` (:class:`int`) [unsigned long long] Convert a C :c:expr:`unsigned long long` to a Python integer object. ``n`` (:class:`int`) [:c:type:`Py_ssize_t`] Convert a C :c:type:`Py_ssize_t` to a Python integer. + ``p`` (:class:`bool`) [int] + Convert a C :c:expr:`int` to a Python :class:`bool` object. + .. versionadded:: 3.14 + ``c`` (:class:`bytes` of length 1) [char] Convert a C :c:expr:`int` representing a byte to a Python :class:`bytes` object of length 1. @@ -615,10 +696,10 @@ Building values object of length 1. ``d`` (:class:`float`) [double] - Convert a C :c:expr:`double` to a Python floating point number. + Convert a C :c:expr:`double` to a Python floating-point number. ``f`` (:class:`float`) [float] - Convert a C :c:expr:`float` to a Python floating point number. + Convert a C :c:expr:`float` to a Python floating-point number. ``D`` (:class:`complex`) [Py_complex \*] Convert a C :c:type:`Py_complex` structure to a Python complex number. diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index b14fa6a0a982e2..b4dc4849d044e1 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -26,19 +26,19 @@ are available, however. .. c:var:: PyObject* Py_False The Python ``False`` object. This object has no methods and is - `immortal <https://peps.python.org/pep-0683/>`_. + :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_False` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_False` is :term:`immortal`. .. c:var:: PyObject* Py_True The Python ``True`` object. This object has no methods and is - `immortal <https://peps.python.org/pep-0683/>`_. + :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_True` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_True` is :term:`immortal`. .. c:macro:: Py_RETURN_FALSE diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8ca1c190dab9a9..d3081894eadaf5 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -26,17 +26,19 @@ characteristic of being backed by a possibly large memory buffer. It is then desirable, in some situations, to access that buffer directly and without intermediate copying. -Python provides such a facility at the C level in the form of the :ref:`buffer -protocol <bufferobjects>`. This protocol has two sides: +Python provides such a facility at the C and Python level in the form of the +:ref:`buffer protocol <bufferobjects>`. This protocol has two sides: -.. index:: single: PyBufferProcs +.. index:: single: PyBufferProcs (C type) - on the producer side, a type can export a "buffer interface" which allows objects of that type to expose information about their underlying buffer. - This interface is described in the section :ref:`buffer-structs`; + This interface is described in the section :ref:`buffer-structs`; for + Python see :ref:`python-buffer-protocol`. - on the consumer side, several means are available to obtain a pointer to - the raw underlying data of an object (for example a method parameter). + the raw underlying data of an object (for example a method parameter). For + Python see :class:`memoryview`. Simple objects such as :class:`bytes` and :class:`bytearray` expose their underlying buffer in byte-oriented form. Other forms are possible; for example, @@ -62,6 +64,10 @@ In both cases, :c:func:`PyBuffer_Release` must be called when the buffer isn't needed anymore. Failure to do so could lead to various issues such as resource leaks. +.. versionadded:: 3.12 + + The buffer protocol is now accessible in Python, see + :ref:`python-buffer-protocol` and :class:`memoryview`. .. _buffer-structure: @@ -147,9 +153,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char *format + .. c:member:: char *format - A *NUL* terminated string in :mod:`struct` module style syntax describing + A *NULL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. @@ -161,10 +167,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. - - The macro :c:macro:`PyBUF_MAX_NDIM` limits the maximum number of dimensions - to 64. Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. + The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. .. c:member:: Py_ssize_t *shape @@ -217,6 +220,17 @@ a buffer, see :c:func:`PyObject_GetBuffer`. freed when the buffer is released. The consumer MUST NOT alter this value. + +Constants: + +.. c:macro:: PyBUF_MAX_NDIM + + The maximum number of dimensions the memory represents. + Exporters MUST respect this limit, consumers of multi-dimensional + buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. + Currently set to 64. + + .. _buffer-request-types: Buffer request types @@ -236,7 +250,6 @@ The following fields are not influenced by *flags* and must always be filled in with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`, :c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`. - readonly, format ~~~~~~~~~~~~~~~~ @@ -245,7 +258,8 @@ readonly, format Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter MUST provide a writable buffer or else report failure. Otherwise, the exporter MAY provide either a read-only or writable buffer, but the choice - MUST be consistent for all consumers. + MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` + can be used to request a simple writable buffer. .. c:macro:: PyBUF_FORMAT @@ -257,8 +271,9 @@ readonly, format Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE` can be used as a stand-alone flag to request a simple writable buffer. -:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`. -The latter already implies format ``B`` (unsigned bytes). +:c:macro:`PyBUF_FORMAT` must be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`, because +the latter already implies format ``B`` (unsigned bytes). :c:macro:`!PyBUF_FORMAT` cannot be +used on its own. shape, strides, suboffsets @@ -440,7 +455,7 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and + :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. On success, fill in *view*, set ``view->obj`` to a new reference @@ -527,7 +542,7 @@ Buffer-related functions and :c:macro:`PyBUF_WRITABLE` is set in *flags*. On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set + return 0. Otherwise, raise :exc:`BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc <buffer-structs>`, diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 456f7d89bca03c..e2b22ec3c794ae 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -42,17 +42,22 @@ Direct API functions Return a new bytearray object from any object, *o*, that implements the :ref:`buffer protocol <bufferobjects>`. + On failure, return ``NULL`` with an exception set. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) - Create a new bytearray object from *string* and its length, *len*. On - failure, ``NULL`` is returned. + Create a new bytearray object from *string* and its length, *len*. + + On failure, return ``NULL`` with an exception set. .. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) Concat bytearrays *a* and *b* and return a new bytearray with the result. + On failure, return ``NULL`` with an exception set. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) @@ -69,6 +74,11 @@ Direct API functions .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) Resize the internal buffer of *bytearray* to *len*. + Failure is a ``-1`` return with an exception set. + + .. versionchanged:: 3.14 + A negative *len* will now result in an exception being set and -1 returned. + Macros ^^^^^^ diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 61a68f52773882..d47beee68eaa33 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -155,6 +155,7 @@ called with a non-bytes parameter. Return the null-terminated contents of the object *obj* through the output variables *buffer* and *length*. + Returns ``0`` on success. If *length* is ``NULL``, the bytes object may not contain embedded null bytes; @@ -188,12 +189,30 @@ called with a non-bytes parameter. to *newpart* (i.e. decrements its reference count). +.. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) + + Similar to ``sep.join(iterable)`` in Python. + + *sep* must be Python :class:`bytes` object. + (Note that :c:func:`PyUnicode_Join` accepts ``NULL`` separator and treats + it as a space, whereas :c:func:`PyBytes_Join` doesn't accept ``NULL`` + separator.) + + *iterable* must be an iterable object yielding objects that implement the + :ref:`buffer protocol <bufferobjects>`. + + On success, return a new :class:`bytes` object. + On error, set an exception and return ``NULL``. + + .. versionadded:: 3.14 + + .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) - A way to resize a bytes object even though it is "immutable". Only use this - to build up a brand new bytes object; don't use this if the bytes may already - be known in other parts of the code. It is an error to call this function if - the refcount on the input bytes object is not one. Pass the address of an + Resize a bytes object. *newsize* will be the new length of the bytes object. + You can think of it as creating a new bytes object and destroying the old + one, only more efficiently. + Pass the address of an existing bytes object as an lvalue (it may be written into), and the new size desired. On success, *\*bytes* holds the resized bytes object and ``0`` is returned; the address in *\*bytes* may differ from its input value. If the diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index aed4ae44c76eea..7198d6bc056eb4 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -108,6 +108,8 @@ This is a pointer to a function with the following signature: Doing so will allow callables such as bound methods to make their onward calls (which include a prepended *self* argument) very efficiently. + .. versionadded:: 3.8 + To call an object that implements vectorcall, use a :ref:`call API <capi-call>` function as with any other callable. :c:func:`PyObject_Vectorcall` will usually be most efficient. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index f8cd0344fdd1c0..61eb994c370946 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -39,7 +39,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: PyObject* PyCell_Get(PyObject *cell) - Return the contents of the cell *cell*. + Return the contents of the cell *cell*, which can be ``NULL``. + If *cell* is not a cell object, returns ``NULL`` with an exception set. .. c:function:: PyObject* PyCell_GET(PyObject *cell) @@ -52,8 +53,10 @@ Cell objects are not likely to be useful elsewhere. Set the contents of the cell object *cell* to *value*. This releases the reference to any current content of the cell. *value* may be ``NULL``. *cell* - must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. + must be non-``NULL``. + + On success, return ``0``. + If *cell* is not a cell object, set an exception and return ``-1``. .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 5082b0cb6ad3f3..42594f063b0709 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -22,16 +22,29 @@ bound into a function. .. c:var:: PyTypeObject PyCode_Type This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. + :ref:`code object <code-objects>`. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. This function always succeeds. + Return true if *co* is a :ref:`code object <code-objects>`. + This function always succeeds. -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) +.. c:function:: Py_ssize_t PyCode_GetNumFree(PyCodeObject *co) - Return the number of free variables in *co*. + Return the number of :term:`free (closure) variables <closure variable>` + in a code object. + +.. c:function:: int PyUnstable_Code_GetFirstFree(PyCodeObject *co) + + Return the position of the first :term:`free (closure) variable <closure variable>` + in a code object. + + .. versionchanged:: 3.13 + + Renamed from ``PyCode_GetFirstFree`` as part of :ref:`unstable-c-api`. + The old name is deprecated, but will remain available until the + signature changes again. .. c:function:: PyCodeObject* PyUnstable_Code_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) @@ -48,7 +61,7 @@ bound into a function. .. versionchanged:: 3.11 Added ``qualname`` and ``exceptiontable`` parameters. - .. index:: single: PyCode_New + .. index:: single: PyCode_New (C function) .. versionchanged:: 3.12 @@ -61,7 +74,7 @@ bound into a function. Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs + .. index:: single: PyCode_NewWithPosOnlyArgs (C function) .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` @@ -85,8 +98,8 @@ bound into a function. Return the line number of the instruction that occurs on or before ``byte_offset`` and ends after it. If you just need the line number of a frame, use :c:func:`PyFrame_GetLineNumber` instead. - For efficiently iterating over the line numbers in a code object, use `the API described in PEP 626 - <https://peps.python.org/pep-0626/#out-of-process-debuggers-and-profilers>`_. + For efficiently iterating over the line numbers in a code object, use :pep:`the API described in PEP 626 + <0626#out-of-process-debuggers-and-profilers>`. .. c:function:: int PyCode_Addr2Location(PyObject *co, int byte_offset, int *start_line, int *start_column, int *end_line, int *end_column) @@ -133,7 +146,8 @@ bound into a function. Equivalent to the Python code ``getattr(co, 'co_freevars')``. Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the free variables. On error, ``NULL`` is returned and an exception is raised. + the :term:`free (closure) variables <closure variable>`. On error, ``NULL`` is returned + and an exception is raised. .. versionadded:: 3.11 @@ -168,7 +182,7 @@ bound into a function. Type of a code object watcher callback function. If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked - after `co` has been fully initialized. Otherwise, the callback is invoked + after *co* has been fully initialized. Otherwise, the callback is invoked before the destruction of *co* takes place, so the prior state of *co* can be inspected. @@ -220,7 +234,7 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex + .. index:: single: _PyEval_RequestCodeExtraIndex (C function) .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` @@ -238,7 +252,7 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra + .. index:: single: _PyCode_GetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_GetExtra`` @@ -253,7 +267,7 @@ may change without deprecation warnings. Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra + .. index:: single: _PyCode_SetExtra (C function) .. versionadded:: 3.6 as ``_PyCode_SetExtra`` diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index e3fd001c599c80..16bd79475dc1e6 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -25,12 +25,16 @@ pointers. This is consistent throughout the API. The C structure which corresponds to the value portion of a Python complex number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. It is - defined as:: + use structures of this type as input or output values, as appropriate. + + .. c:member:: double real + double imag + + The structure is defined as:: typedef struct { - double real; - double imag; + double real; + double imag; } Py_complex; @@ -75,6 +79,8 @@ pointers. This is consistent throughout the API. If *num* is null and *exp* is not a positive real number, this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. + Set :c:data:`errno` to :c:macro:`!ERANGE` on overflows. + Complex Numbers as Python Objects ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -106,22 +112,46 @@ Complex Numbers as Python Objects .. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) Create a new Python complex number object from a C :c:type:`Py_complex` value. + Return ``NULL`` with an exception set on error. .. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) Return a new :c:type:`PyComplexObject` object from *real* and *imag*. + Return ``NULL`` with an exception set on error. .. c:function:: double PyComplex_RealAsDouble(PyObject *op) Return the real part of *op* as a C :c:expr:`double`. + If *op* is not a Python complex number object but has a + :meth:`~object.__complex__` method, this method will first be called to + convert *op* to a Python complex number object. If :meth:`!__complex__` is + not defined then it falls back to call :c:func:`PyFloat_AsDouble` and + returns its result. + + Upon failure, this method returns ``-1.0`` with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. + + .. versionchanged:: 3.13 + Use :meth:`~object.__complex__` if available. .. c:function:: double PyComplex_ImagAsDouble(PyObject *op) Return the imaginary part of *op* as a C :c:expr:`double`. + If *op* is not a Python complex number object but has a + :meth:`~object.__complex__` method, this method will first be called to + convert *op* to a Python complex number object. If :meth:`!__complex__` is + not defined then it falls back to call :c:func:`PyFloat_AsDouble` and + returns ``0.0`` on success. + + Upon failure, this method returns ``-1.0`` with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. + + .. versionchanged:: 3.13 + Use :meth:`~object.__complex__` if available. .. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) @@ -131,8 +161,11 @@ Complex Numbers as Python Objects method, this method will first be called to convert *op* to a Python complex number object. If :meth:`!__complex__` is not defined then it falls back to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back - to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real - value. + to :meth:`~object.__index__`. + + Upon failure, this method returns :c:type:`Py_complex` + with :c:member:`~Py_complex.real` set to ``-1.0`` and with an exception set, so one + should call :c:func:`PyErr_Occurred` to check for errors. .. versionchanged:: 3.8 Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index d970f5443b1df5..b7c6550ff34aac 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -6,6 +6,8 @@ Context Variables Objects ------------------------- .. _contextvarsobjects_pointertype_change: +.. versionadded:: 3.7 + .. versionchanged:: 3.7.1 .. note:: @@ -24,8 +26,6 @@ Context Variables Objects See :issue:`34762` for more details. -.. versionadded:: 3.7 - This section details the public C API for the :mod:`contextvars` module. .. c:type:: PyContext @@ -101,6 +101,52 @@ Context object management functions: current context for the current thread. Returns ``0`` on success, and ``-1`` on error. +.. c:function:: int PyContext_AddWatcher(PyContext_WatchCallback callback) + + Register *callback* as a context object watcher for the current interpreter. + Return an ID which may be passed to :c:func:`PyContext_ClearWatcher`. + In case of error (e.g. no more watcher IDs available), + return ``-1`` and set an exception. + + .. versionadded:: 3.14 + +.. c:function:: int PyContext_ClearWatcher(int watcher_id) + + Clear watcher identified by *watcher_id* previously returned from + :c:func:`PyContext_AddWatcher` for the current interpreter. + Return ``0`` on success, or ``-1`` and set an exception on error + (e.g. if the given *watcher_id* was never registered.) + + .. versionadded:: 3.14 + +.. c:type:: PyContextEvent + + Enumeration of possible context object watcher events: + + - ``Py_CONTEXT_SWITCHED``: The :term:`current context` has switched to a + different context. The object passed to the watch callback is the + now-current :class:`contextvars.Context` object, or None if no context is + current. + + .. versionadded:: 3.14 + +.. c:type:: int (*PyContext_WatchCallback)(PyContextEvent event, PyObject *obj) + + Context object watcher callback function. The object passed to the callback + is event-specific; see :c:type:`PyContextEvent` for details. + + If the callback returns with an exception set, it must return ``-1``; this + exception will be printed as an unraisable exception using + :c:func:`PyErr_FormatUnraisable`. Otherwise it should return ``0``. + + There may already be a pending exception set on entry to the callback. In + this case, the callback should return ``0`` with the same exception still + set. This means the callback may not call any other API that can set an + exception unless it saves and clears the exception state first, and restores + it before returning. + + .. versionadded:: 3.14 + Context variable functions: diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c5350123dfdfdc..c92ef4c653a675 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -48,6 +48,42 @@ The return value (*rv*) for these functions should be interpreted as follows: The following functions provide locale-independent string to number conversions. +.. c:function:: unsigned long PyOS_strtoul(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`unsigned + long` value according to the given ``base``, which must be between ``2`` and + ``36`` inclusive, or be the special value ``0``. + + Leading white space and case of characters are ignored. If ``base`` is zero + it looks for a leading ``0b``, ``0o`` or ``0x`` to tell which base. If + these are absent it defaults to ``10``. Base must be 0 or between 2 and 36 + (inclusive). If ``ptr`` is non-``NULL`` it will contain a pointer to the + end of the scan. + + If the converted value falls out of range of corresponding return type, + range error occurs (:c:data:`errno` is set to :c:macro:`!ERANGE`) and + :c:macro:`!ULONG_MAX` is returned. If no conversion can be performed, ``0`` + is returned. + + See also the Unix man page :manpage:`strtoul(3)`. + + .. versionadded:: 3.2 + + +.. c:function:: long PyOS_strtol(const char *str, char **ptr, int base) + + Convert the initial part of the string in ``str`` to an :c:expr:`long` value + according to the given ``base``, which must be between ``2`` and ``36`` + inclusive, or be the special value ``0``. + + Same as :c:func:`PyOS_strtoul`, but return a :c:expr:`long` value instead + and :c:macro:`LONG_MAX` on overflows. + + See also the Unix man page :manpage:`strtol(3)`. + + .. versionadded:: 3.2 + + .. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) Convert a string ``s`` to a :c:expr:`double`, raising a Python @@ -69,7 +105,7 @@ The following functions provide locale-independent string to number conversions. If ``s`` represents a value that is too large to store in a float (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with + if ``overflow_exception`` is ``NULL`` return ``Py_INFINITY`` (with an appropriate sign) and don't set any exception. Otherwise, ``overflow_exception`` must point to a Python exception object; raise that exception and return ``-1.0``. In both cases, set diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index 72fc07afbf1f4d..d2d4d5309c7098 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,54 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following macros. +.. c:type:: PyDateTime_Date + + This subtype of :c:type:`PyObject` represents a Python date object. + +.. c:type:: PyDateTime_DateTime + + This subtype of :c:type:`PyObject` represents a Python datetime object. + +.. c:type:: PyDateTime_Time + + This subtype of :c:type:`PyObject` represents a Python time object. + +.. c:type:: PyDateTime_Delta + + This subtype of :c:type:`PyObject` represents the difference between two datetime values. + +.. c:var:: PyTypeObject PyDateTime_DateType + + This instance of :c:type:`PyTypeObject` represents the Python date type; + it is the same object as :class:`datetime.date` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DateTimeType + + This instance of :c:type:`PyTypeObject` represents the Python datetime type; + it is the same object as :class:`datetime.datetime` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TimeType + + This instance of :c:type:`PyTypeObject` represents the Python time type; + it is the same object as :class:`datetime.time` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_DeltaType + + This instance of :c:type:`PyTypeObject` represents Python type for + the difference between two datetime values; + it is the same object as :class:`datetime.timedelta` in the Python layer. + +.. c:var:: PyTypeObject PyDateTime_TZInfoType + + This instance of :c:type:`PyTypeObject` represents the Python time zone info type; + it is the same object as :class:`datetime.tzinfo` in the Python layer. + + Macro for access to the UTC singleton: .. c:var:: PyObject* PyDateTime_TimeZone_UTC @@ -28,7 +71,7 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always succeeds. @@ -41,7 +84,7 @@ Type-check macros: .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -54,7 +97,7 @@ Type-check macros: .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always succeeds. @@ -67,7 +110,7 @@ Type-check macros: .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always succeeds. @@ -80,7 +123,7 @@ Type-check macros: .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always succeeds. @@ -133,7 +176,7 @@ Macros to create objects: :class:`datetime.timedelta` objects. -.. c:function:: PyObject* PyTimeZone_FromOffset(PyDateTime_DeltaType* offset) +.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) Return a :class:`datetime.timezone` object with an unnamed fixed offset represented by the *offset* argument. @@ -141,7 +184,7 @@ Macros to create objects: .. versionadded:: 3.7 -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyDateTime_DeltaType* offset, PyUnicode* name) +.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) Return a :class:`datetime.timezone` object with a fixed offset represented by the *offset* argument and with tzname *name*. @@ -150,8 +193,8 @@ Macros to create objects: Macros to extract fields from date objects. The argument must be an instance of -:c:data:`PyDateTime_Date`, including subclasses (such as -:c:data:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is +:c:type:`PyDateTime_Date`, including subclasses (such as +:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) @@ -170,7 +213,7 @@ not checked: Macros to extract fields from datetime objects. The argument must be an -instance of :c:data:`PyDateTime_DateTime`, including subclasses. The argument +instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) @@ -208,7 +251,7 @@ must not be ``NULL``, and the type is not checked: Macros to extract fields from time objects. The argument must be an instance of -:c:data:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, +:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) @@ -246,7 +289,7 @@ and the type is not checked: Macros to extract fields from time delta objects. The argument must be an -instance of :c:data:`PyDateTime_Delta`, including subclasses. The argument must +instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must not be ``NULL``, and the type is not checked: .. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) @@ -275,10 +318,10 @@ Macros for the convenience of modules implementing the DB API: .. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) Create and return a new :class:`datetime.datetime` object given an argument - tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp()`. + tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp`. .. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) Create and return a new :class:`datetime.date` object given an argument - tuple suitable for passing to :meth:`datetime.date.fromtimestamp()`. + tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e4c1d71a413a68..e55c5c80cb83c0 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -55,6 +55,15 @@ Dictionary Objects This is equivalent to the Python expression ``key in p``. +.. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) + + This is the same as :c:func:`PyDict_Contains`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Copy(PyObject *p) Return a new dictionary that contains the same key-value pairs as *p*. @@ -70,12 +79,9 @@ Dictionary Objects .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) - .. index:: single: PyUnicode_FromString() - - Insert *val* into the dictionary *p* using *key* as a key. *key* should - be a :c:expr:`const char*`. The key object is created using - ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on - failure. This function *does not* steal a reference to *val*. + This is the same as :c:func:`PyDict_SetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) @@ -88,9 +94,9 @@ Dictionary Objects .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - Remove the entry in dictionary *p* which has a key specified by the string *key*. - If *key* is not in the dictionary, :exc:`KeyError` is raised. - Return ``0`` on success or ``-1`` on failure. + This is the same as :c:func:`PyDict_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyDict_GetItemRef(PyObject *p, PyObject *key, PyObject **result) @@ -121,7 +127,7 @@ Dictionary Objects Prefer the :c:func:`PyDict_GetItemWithError` function instead. .. versionchanged:: 3.10 - Calling this API without :term:`GIL` held had been allowed for historical + Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. @@ -136,7 +142,8 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. .. note:: @@ -149,8 +156,9 @@ Dictionary Objects .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) - Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a - :c:expr:`const char*`, rather than a :c:expr:`PyObject*`. + Similar to :c:func:`PyDict_GetItemRef`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. .. versionadded:: 3.13 @@ -165,6 +173,54 @@ Dictionary Objects .. versionadded:: 3.4 + +.. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) + + Inserts *default_value* into the dictionary *p* with a key of *key* if the + key is not already present in the dictionary. If *result* is not ``NULL``, + then *\*result* is set to a :term:`strong reference` to either + *default_value*, if the key was not present, or the existing value, if *key* + was already present in the dictionary. + Returns ``1`` if the key was present and *default_value* was not inserted, + or ``0`` if the key was not present and *default_value* was inserted. + On failure, returns ``-1``, sets an exception, and sets ``*result`` + to ``NULL``. + + For clarity: if you have a strong reference to *default_value* before + calling this function, then after it returns, you hold a strong reference + to both *default_value* and *\*result* (if it's not ``NULL``). + These may refer to the same object: in that case you hold two separate + references to it. + + .. versionadded:: 3.13 + + +.. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) + + Remove *key* from dictionary *p* and optionally return the removed value. + Do not raise :exc:`KeyError` if the key missing. + + - If the key is present, set *\*result* to a new reference to the removed + value if *result* is not ``NULL``, and return ``1``. + - If the key is missing, set *\*result* to ``NULL`` if *result* is not + ``NULL``, and return ``0``. + - On error, raise an exception and return ``-1``. + + Similar to :meth:`dict.pop`, but without the default value and + not raising :exc:`KeyError` if the key missing. + + .. versionadded:: 3.13 + + +.. c:function:: int PyDict_PopString(PyObject *p, const char *key, PyObject **result) + + Similar to :c:func:`PyDict_Pop`, but *key* is specified as a + :c:expr:`const char*` UTF-8 encoded bytes string, rather than a + :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: PyObject* PyDict_Items(PyObject *p) Return a :c:type:`PyListObject` containing all the items from the dictionary. @@ -234,6 +290,17 @@ Dictionary Objects Py_DECREF(o); } + The function is not thread-safe in the :term:`free-threaded <free threading>` + build without external synchronization. You can use + :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating + over it:: + + Py_BEGIN_CRITICAL_SECTION(self->dict); + while (PyDict_Next(self->dict, &pos, &key, &value)) { + ... + } + Py_END_CRITICAL_SECTION(); + .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index f1d6c995188abb..c8e1b5c2461738 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -34,7 +34,7 @@ propagated, additional calls into the Python/C API may not behave as intended and may fail in mysterious ways. .. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. + The error indicator is **not** the result of :func:`sys.exc_info`. The former corresponds to an exception that is not yet caught (and is therefore still propagating), while the latter returns an exception after it is caught (and has therefore stopped propagating). @@ -88,9 +88,29 @@ Printing and clearing The function is called with a single argument *obj* that identifies the context in which the unraisable exception occurred. If possible, the repr of *obj* will be printed in the warning message. + If *obj* is ``NULL``, only the traceback is printed. An exception must be set when calling this function. + .. versionchanged:: 3.4 + Print a traceback. Print only traceback if *obj* is ``NULL``. + + .. versionchanged:: 3.8 + Use :func:`sys.unraisablehook`. + + +.. c:function:: void PyErr_FormatUnraisable(const char *format, ...) + + Similar to :c:func:`PyErr_WriteUnraisable`, but the *format* and subsequent + parameters help format the warning message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. + ``PyErr_WriteUnraisable(obj)`` is roughly equivalent to + ``PyErr_FormatUnraisable("Exception ignored in: %R", obj)``. + If *format* is ``NULL``, only the traceback is printed. + + .. versionadded:: 3.13 + + .. c:function:: void PyErr_DisplayException(PyObject *exc) Print the standard traceback display of ``exc`` to ``sys.stderr``, including @@ -98,6 +118,7 @@ Printing and clearing .. versionadded:: 3.12 + Raising exceptions ================== @@ -159,7 +180,7 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - .. index:: single: strerror() + .. index:: single: strerror (C function) This is a convenience function to raise an exception when a C library function has returned an error and set the C variable :c:data:`errno`. It constructs a @@ -200,13 +221,14 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - This is a convenience function to raise :exc:`WindowsError`. If called with + This is a convenience function to raise :exc:`OSError`. If called with *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError` is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve the Windows description of error code given by *ierr* or :c:func:`!GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + then it constructs a :exc:`OSError` object with the :attr:`~OSError.winerror` + attribute set to the error code, the :attr:`~OSError.strerror` attribute + set to the corresponding error message (gotten from + :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_OSError, object)``. This function always returns ``NULL``. .. availability:: Windows. @@ -222,17 +244,21 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). + Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is decoded from the filesystem + encoding (:func:`os.fsdecode`) and passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. .. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. + Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior + that if *filename* is not ``NULL``, it is passed to the constructor of + :exc:`OSError` as a third parameter to be used to define the + :attr:`!filename` attribute of the exception instance. .. availability:: Windows. @@ -371,7 +397,7 @@ an error value). .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + :exc:`ResourceWarning` and it passes *source* to :class:`!warnings.WarningMessage`. .. versionadded:: 3.6 @@ -387,7 +413,7 @@ Querying the error indicator own a reference to the return value, so you do not need to :c:func:`Py_DECREF` it. - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. .. note:: @@ -415,7 +441,7 @@ Querying the error indicator .. c:function:: PyObject *PyErr_GetRaisedException(void) Return the exception currently being raised, clearing the error indicator at - the same time. + the same time. Return ``NULL`` if the error indicator is not set. This function is used by code that needs to catch exceptions, or code that needs to save and restore the error indicator temporarily. @@ -516,7 +542,8 @@ Querying the error indicator .. note:: - This function *does not* implicitly set the ``__traceback__`` + This function *does not* implicitly set the + :attr:`~BaseException.__traceback__` attribute on the exception value. If setting the traceback appropriately is desired, the following additional snippet is needed:: @@ -609,7 +636,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) This function interacts with Python's signal handling. @@ -640,7 +667,7 @@ Signal Handling .. index:: pair: module; signal - single: SIGINT + single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) Simulate the effect of a :c:macro:`!SIGINT` signal arriving. @@ -648,7 +675,7 @@ Signal Handling .. note:: This function is async-signal-safe. It can be called without - the :term:`GIL` and from a C signal handler. + an :term:`attached thread state` and from a C signal handler. .. c:function:: int PyErr_SetInterruptEx(int signum) @@ -675,7 +702,7 @@ Signal Handling .. note:: This function is async-signal-safe. It can be called without - the :term:`GIL` and from a C signal handler. + an :term:`attached thread state` and from a C signal handler. .. versionadded:: 3.10 @@ -706,7 +733,7 @@ Exception Classes This creates a class object derived from :exc:`Exception` (accessible in C as :c:data:`PyExc_Exception`). - The :attr:`__module__` attribute of the new class is set to the first part (up + The :attr:`~type.__module__` attribute of the new class is set to the first part (up to the last dot) of the *name* argument, and the class name is set to the last part (after the last dot). The *base* argument can be used to specify alternate base classes; it can either be only one class or a tuple of classes. The *dict* @@ -728,7 +755,8 @@ Exception Objects .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no + accessible from Python through the :attr:`~BaseException.__traceback__` + attribute. If there is no traceback associated, this returns ``NULL``. @@ -742,8 +770,8 @@ Exception Objects Return the context (another exception instance during whose handling *ex* was raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. + Python through the :attr:`~BaseException.__context__` attribute. + If there is no context associated, this returns ``NULL``. .. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) @@ -757,7 +785,8 @@ Exception Objects Return the cause (either an exception instance, or ``None``, set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. + reference, as accessible from Python through the + :attr:`~BaseException.__cause__` attribute. .. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) @@ -766,7 +795,8 @@ Exception Objects it. There is no type check to make sure that *cause* is either an exception instance or ``None``. This steals a reference to *cause*. - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + The :attr:`~BaseException.__suppress_context__` attribute is implicitly set + to ``True`` by this function. .. c:function:: PyObject* PyException_GetArgs(PyObject *ex) @@ -782,7 +812,7 @@ Exception Objects Implement part of the interpreter's implementation of :keyword:`!except*`. *orig* is the original exception that was caught, and *excs* is the list of - the exceptions that need to be raised. This list contains the the unhandled + the exceptions that need to be raised. This list contains the unhandled part of *orig*, if any, as well as the exceptions that were raised from the :keyword:`!except*` clauses (so they have a different traceback from *orig*) and those that were reraised (and have the same traceback as *orig*). @@ -823,12 +853,23 @@ The following functions are used to create and modify Unicode exceptions from C. *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *start* is ``0``. Otherwise, it is clipped to ``[0, len(object) - 1]``. + + .. seealso:: :attr:`UnicodeError.start` + .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. + Set the *start* attribute of the given exception object to *start*. + Return ``0`` on success, ``-1`` on failure. + + .. note:: + + While passing a negative *start* does not raise an exception, + the corresponding getters will not consider it as a relative + offset. .. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) @@ -838,6 +879,9 @@ The following functions are used to create and modify Unicode exceptions from C. *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *end* is ``0``. Otherwise, it is clipped to ``[1, len(object)]``. + .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -845,6 +889,8 @@ The following functions are used to create and modify Unicode exceptions from C. Set the *end* attribute of the given exception object to *end*. Return ``0`` on success, ``-1`` on failure. + .. seealso:: :attr:`UnicodeError.end` + .. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) @@ -875,11 +921,7 @@ because the :ref:`call protocol <call>` takes care of recursion handling. Marks a point where a recursive C-level call is about to be performed. - If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the + The function then checks if the stack limit is reached. If this is the case, a :exc:`RecursionError` is set and a nonzero value is returned. Otherwise, zero is returned. @@ -939,59 +981,60 @@ All standard Python exceptions are available as global variables whose names are the variables: .. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError + single: PyExc_BaseException (C var) + single: PyExc_Exception (C var) + single: PyExc_ArithmeticError (C var) + single: PyExc_AssertionError (C var) + single: PyExc_AttributeError (C var) + single: PyExc_BlockingIOError (C var) + single: PyExc_BrokenPipeError (C var) + single: PyExc_BufferError (C var) + single: PyExc_ChildProcessError (C var) + single: PyExc_ConnectionAbortedError (C var) + single: PyExc_ConnectionError (C var) + single: PyExc_ConnectionRefusedError (C var) + single: PyExc_ConnectionResetError (C var) + single: PyExc_EOFError (C var) + single: PyExc_FileExistsError (C var) + single: PyExc_FileNotFoundError (C var) + single: PyExc_FloatingPointError (C var) + single: PyExc_GeneratorExit (C var) + single: PyExc_ImportError (C var) + single: PyExc_IndentationError (C var) + single: PyExc_IndexError (C var) + single: PyExc_InterruptedError (C var) + single: PyExc_IsADirectoryError (C var) + single: PyExc_KeyError (C var) + single: PyExc_KeyboardInterrupt (C var) + single: PyExc_LookupError (C var) + single: PyExc_MemoryError (C var) + single: PyExc_ModuleNotFoundError (C var) + single: PyExc_NameError (C var) + single: PyExc_NotADirectoryError (C var) + single: PyExc_NotImplementedError (C var) + single: PyExc_OSError (C var) + single: PyExc_OverflowError (C var) + single: PyExc_PermissionError (C var) + single: PyExc_ProcessLookupError (C var) + single: PyExc_PythonFinalizationError (C var) + single: PyExc_RecursionError (C var) + single: PyExc_ReferenceError (C var) + single: PyExc_RuntimeError (C var) + single: PyExc_StopAsyncIteration (C var) + single: PyExc_StopIteration (C var) + single: PyExc_SyntaxError (C var) + single: PyExc_SystemError (C var) + single: PyExc_SystemExit (C var) + single: PyExc_TabError (C var) + single: PyExc_TimeoutError (C var) + single: PyExc_TypeError (C var) + single: PyExc_UnboundLocalError (C var) + single: PyExc_UnicodeDecodeError (C var) + single: PyExc_UnicodeEncodeError (C var) + single: PyExc_UnicodeError (C var) + single: PyExc_UnicodeTranslateError (C var) + single: PyExc_ValueError (C var) + single: PyExc_ZeroDivisionError (C var) +-----------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | @@ -1066,6 +1109,8 @@ the variables: +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | +-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | ++-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | +-----------------------------------------+---------------------------------+----------+ | :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | @@ -1122,18 +1167,18 @@ the variables: These are compatibility aliases to :c:data:`PyExc_OSError`: .. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError + single: PyExc_EnvironmentError (C var) + single: PyExc_IOError (C var) + single: PyExc_WindowsError (C var) +-------------------------------------+----------+ | C Name | Notes | +=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | +| :c:data:`!PyExc_EnvironmentError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | +| :c:data:`!PyExc_IOError` | | +-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | [2]_ | +| :c:data:`!PyExc_WindowsError` | [2]_ | +-------------------------------------+----------+ .. versionchanged:: 3.3 @@ -1159,17 +1204,17 @@ names are ``PyExc_`` followed by the Python exception name. These have the type the variables: .. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning + single: PyExc_Warning (C var) + single: PyExc_BytesWarning (C var) + single: PyExc_DeprecationWarning (C var) + single: PyExc_FutureWarning (C var) + single: PyExc_ImportWarning (C var) + single: PyExc_PendingDeprecationWarning (C var) + single: PyExc_ResourceWarning (C var) + single: PyExc_RuntimeWarning (C var) + single: PyExc_SyntaxWarning (C var) + single: PyExc_UnicodeWarning (C var) + single: PyExc_UserWarning (C var) +------------------------------------------+---------------------------------+----------+ | C Name | Python Name | Notes | diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index b36c800e00444a..e9019a0d500f7e 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -65,8 +65,14 @@ the :mod:`io` APIs instead. Overrides the normal behavior of :func:`io.open_code` to pass its parameter through the provided handler. - The handler is a function of type :c:expr:`PyObject *(\*)(PyObject *path, - void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. + The *handler* is a function of type: + + .. c:namespace:: NULL + .. c:type:: PyObject * (*Py_OpenCodeHookFunction)(PyObject *, void *) + + Equivalent of :c:expr:`PyObject *(\*)(PyObject *path, + void *userData)`, where *path* is guaranteed to be + :c:type:`PyUnicodeObject`. The *userData* pointer is passed into the hook function. Since hook functions may be called from different runtimes, this pointer should not @@ -90,7 +96,7 @@ the :mod:`io` APIs instead. .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) - .. index:: single: Py_PRINT_RAW + .. index:: single: Py_PRINT_RAW (C macro) Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 4f6ac0d8175c6b..c5a7653efca26b 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -2,20 +2,20 @@ .. _floatobjects: -Floating Point Objects +Floating-Point Objects ====================== -.. index:: pair: object; floating point +.. index:: pair: object; floating-point .. c:type:: PyFloatObject - This subtype of :c:type:`PyObject` represents a Python floating point object. + This subtype of :c:type:`PyObject` represents a Python floating-point object. .. c:var:: PyTypeObject PyFloat_Type - This instance of :c:type:`PyTypeObject` represents the Python floating point + This instance of :c:type:`PyTypeObject` represents the Python floating-point type. This is the same object as :class:`float` in the Python layer. @@ -45,7 +45,7 @@ Floating Point Objects .. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` + *pyfloat* is not a Python floating-point object but has a :meth:`~object.__float__` method, this method will first be called to convert *pyfloat* into a float. If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. This method returns ``-1.0`` upon failure, so one should call @@ -96,6 +96,9 @@ NaNs (if such things exist on the platform) isn't handled correctly, and attempting to unpack a bytes string containing an IEEE INF or NaN will raise an exception. +Note that NaNs type may not be preserved on IEEE platforms (silent NaN become +quiet), for example on x86 systems in 32-bit mode. + On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less precision, or smaller dynamic range, not all values can be unpacked. What diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1accee2767a485..1a52e146a69751 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -50,7 +50,7 @@ See also :ref:`Reflection <reflection>`. .. c:function:: PyObject* PyFrame_GetBuiltins(PyFrameObject *frame) - Get the *frame*'s ``f_builtins`` attribute. + Get the *frame*'s :attr:`~frame.f_builtins` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -81,7 +81,7 @@ See also :ref:`Reflection <reflection>`. .. c:function:: PyObject* PyFrame_GetGlobals(PyFrameObject *frame) - Get the *frame*'s ``f_globals`` attribute. + Get the *frame*'s :attr:`~frame.f_globals` attribute. Return a :term:`strong reference`. The result cannot be ``NULL``. @@ -90,7 +90,7 @@ See also :ref:`Reflection <reflection>`. .. c:function:: int PyFrame_GetLasti(PyFrameObject *frame) - Get the *frame*'s ``f_lasti`` attribute. + Get the *frame*'s :attr:`~frame.f_lasti` attribute. Returns -1 if ``frame.f_lasti`` is ``None``. @@ -120,18 +120,46 @@ See also :ref:`Reflection <reflection>`. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s ``f_locals`` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute. + If the frame refers to an :term:`optimized scope`, this returns a + write-through proxy object that allows modifying the locals. + In all other cases (classes, modules, :func:`exec`, :func:`eval`) it returns + the mapping representing the frame locals directly (as described for + :func:`locals`). Return a :term:`strong reference`. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + As part of :pep:`667`, return an instance of :c:var:`PyFrameLocalsProxy_Type`. + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) Return the line number that *frame* is currently executing. +Frame Locals Proxies +^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.13 + +The :attr:`~frame.f_locals` attribute on a :ref:`frame object <frame-objects>` +is an instance of a "frame-locals proxy". The proxy object exposes a +write-through view of the underlying locals dictionary for the frame. This +ensures that the variables exposed by ``f_locals`` are always up to date with +the live local variables in the frame itself. + +See :pep:`667` for more information. + +.. c:var:: PyTypeObject PyFrameLocalsProxy_Type + + The type of frame :func:`locals` proxy objects. + +.. c:function:: int PyFrameLocalsProxy_Check(PyObject *obj) + + Return non-zero if *obj* is a frame :func:`locals` proxy. Internal Frames ^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 5857dba82c11c6..63b78f677674e9 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -34,18 +34,20 @@ There are a few functions specific to Python functions. Return a new function object associated with the code object *code*. *globals* must be a dictionary with the global variables accessible to the function. - The function's docstring and name are retrieved from the code object. *__module__* + The function's docstring and name are retrieved from the code object. + :attr:`~function.__module__` is retrieved from *globals*. The argument defaults, annotations and closure are - set to ``NULL``. *__qualname__* is set to the same value as the code object's - ``co_qualname`` field. + set to ``NULL``. :attr:`~function.__qualname__` is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; - if ``NULL``, the ``__qualname__`` attribute is set to the same value as the - code object's ``co_qualname`` field. + :attr:`~function.__qualname__` attribute. + *qualname* should be a unicode object or ``NULL``; + if ``NULL``, the :attr:`!__qualname__` attribute is set to the same value as + the code object's :attr:`~codeobject.co_qualname` field. .. versionadded:: 3.3 @@ -62,11 +64,12 @@ There are a few functions specific to Python functions. .. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - Return a :term:`borrowed reference` to the *__module__* attribute of the - function object *op*. It can be *NULL*. + Return a :term:`borrowed reference` to the :attr:`~function.__module__` + attribute of the :ref:`function object <user-defined-funcs>` *op*. + It can be *NULL*. - This is normally a string containing the module name, but can be set to any - other object by Python code. + This is normally a :class:`string <str>` containing the module name, + but can be set to any other object by Python code. .. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) @@ -142,12 +145,13 @@ There are a few functions specific to Python functions. .. c:type:: PyFunction_WatchEvent - Enumeration of possible function watcher events: - - ``PyFunction_EVENT_CREATE`` - - ``PyFunction_EVENT_DESTROY`` - - ``PyFunction_EVENT_MODIFY_CODE`` - - ``PyFunction_EVENT_MODIFY_DEFAULTS`` - - ``PyFunction_EVENT_MODIFY_KWDEFAULTS`` + Enumeration of possible function watcher events: + + - ``PyFunction_EVENT_CREATE`` + - ``PyFunction_EVENT_DESTROY`` + - ``PyFunction_EVENT_MODIFY_CODE`` + - ``PyFunction_EVENT_MODIFY_DEFAULTS`` + - ``PyFunction_EVENT_MODIFY_KWDEFAULTS`` .. versionadded:: 3.12 @@ -165,7 +169,7 @@ There are a few functions specific to Python functions. unpredictable effects, including infinite recursion. If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked - after `func` has been fully initialized. Otherwise, the callback is invoked + after *func* has been fully initialized. Otherwise, the callback is invoked before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 6b2494ee4f0ed4..f6fa52b36c5ab3 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -57,11 +57,49 @@ rules: Analogous to :c:macro:`PyObject_New` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) Analogous to :c:macro:`PyObject_NewVar` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size* @@ -73,6 +111,10 @@ rules: The extra data will be deallocated with the object, but otherwise it is not managed by Python. + Memory allocated by this function must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + .. warning:: The function is marked as unstable because the final mechanism for reserving extra data after an instance is not yet decided. @@ -83,10 +125,15 @@ rules: .. versionadded:: 3.12 -.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) +.. c:macro:: PyObject_GC_Resize(TYPE, op, newsize) + + Resize an object allocated by :c:macro:`PyObject_NewVar`. + Returns the resized object of type ``TYPE*`` (refers to any C type) + or ``NULL`` on failure. - Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the - resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. + *op* must be of type :c:expr:`PyVarObject *` + and must not be tracked by the collector yet. + *newsize* must be of type :c:type:`Py_ssize_t`. .. c:function:: void PyObject_GC_Track(PyObject *op) @@ -131,6 +178,21 @@ rules: Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_New`, + :c:macro:`PyObject_NewVar`, or related allocation functions; use + :c:func:`PyObject_Free` instead. + + .. seealso:: + + * :c:func:`PyObject_Free` is the non-GC equivalent of this function. + * :c:macro:`PyObject_GC_New` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. c:function:: void PyObject_GC_UnTrack(void *op) @@ -175,9 +237,9 @@ provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` must name its arguments exactly *visit* and *arg*: -.. c:function:: void Py_VISIT(PyObject *o) +.. c:macro:: Py_VISIT(o) - If *o* is not ``NULL``, call the *visit* callback, with arguments *o* + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: @@ -272,7 +334,7 @@ the garbage collector. Type of the visitor function to be passed to :c:func:`PyUnstable_GC_VisitObjects`. *arg* is the same as the *arg* passed to ``PyUnstable_GC_VisitObjects``. - Return ``0`` to continue iteration, return ``1`` to stop iteration. Other return + Return ``1`` to continue iteration, return ``0`` to stop iteration. Other return values are reserved for now so behavior on returning anything else is undefined. .. versionadded:: 3.12 diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst new file mode 100644 index 00000000000000..00f8cb887dc7eb --- /dev/null +++ b/Doc/c-api/hash.rst @@ -0,0 +1,120 @@ +.. highlight:: c + +PyHash API +---------- + +See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. + +.. c:type:: Py_hash_t + + Hash value type: signed integer. + + .. versionadded:: 3.2 + +.. c:type:: Py_uhash_t + + Hash value type: unsigned integer. + + .. versionadded:: 3.2 + +.. c:macro:: PyHASH_MODULUS + + The `Mersenne prime <https://en.wikipedia.org/wiki/Mersenne_prime>`_ ``P = 2**n -1``, used for numeric hash scheme. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_BITS + + The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_MULTIPLIER + + Prime multiplier used in string and various other hashes. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_INF + + The hash value returned for a positive infinity. + + .. versionadded:: 3.13 + +.. c:macro:: PyHASH_IMAG + + The multiplier used for the imaginary part of a complex number. + + .. versionadded:: 3.13 + +.. c:type:: PyHash_FuncDef + + Hash function definition used by :c:func:`PyHash_GetFuncDef`. + + .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + + Hash function. + + .. c:member:: const char *name + + Hash function name (UTF-8 encoded string). + + .. c:member:: const int hash_bits + + Internal size of the hash value in bits. + + .. c:member:: const int seed_bits + + Size of seed input in bits. + + .. versionadded:: 3.4 + + +.. c:function:: PyHash_FuncDef* PyHash_GetFuncDef(void) + + Get the hash function definition. + + .. seealso:: + :pep:`456` "Secure and interchangeable hash algorithm". + + .. versionadded:: 3.4 + + +.. c:function:: Py_hash_t Py_HashPointer(const void *ptr) + + Hash a pointer value: process the pointer value as an integer (cast it to + ``uintptr_t`` internally). The pointer is not dereferenced. + + The function cannot fail: it cannot return ``-1``. + + .. versionadded:: 3.13 + + +.. c:function:: Py_hash_t Py_HashBuffer(const void *ptr, Py_ssize_t len) + + Compute and return the hash value of a buffer of *len* bytes + starting at address *ptr*. The hash is guaranteed to match that of + :class:`bytes`, :class:`memoryview`, and other built-in objects + that implement the :ref:`buffer protocol <bufferobjects>`. + + Use this function to implement hashing for immutable objects whose + :c:member:`~PyTypeObject.tp_richcompare` function compares to another + object's buffer. + + *len* must be greater than or equal to ``0``. + + This function always succeeds. + + .. versionadded:: 3.14 + + +.. c:function:: Py_hash_t PyObject_GenericHash(PyObject *obj) + + Generic hashing function that is meant to be put into a type + object's ``tp_hash`` slot. + Its result only depends on the object's identity. + + .. impl-detail:: + In CPython, it is equivalent to :c:func:`Py_HashPointer`. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 137780cc359cf9..8eabc0406b11ce 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -13,33 +13,8 @@ Importing Modules single: __all__ (package variable) single: modules (in module sys) - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or ``NULL`` with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - - -.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) - - This function is a deprecated alias of :c:func:`PyImport_ImportModule`. - - .. versionchanged:: 3.3 - This function used to fail immediately when the import lock was held - by another thread. In Python 3.3 though, the locking scheme switched - to per-module locks for most purposes, so this function's special - behaviour isn't needed anymore. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyImport_ImportModule` instead. + This is a wrapper around :c:func:`PyImport_Import()` which takes a + :c:expr:`const char *` as an argument instead of a :c:expr:`PyObject *`. .. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) @@ -148,14 +123,14 @@ Importing Modules such modules have no way to know that the module object is an unknown (and probably damaged with respect to the module author's intents) state. - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`~importlib.machinery.SourceFileLoader` otherwise. + The module's :attr:`~module.__spec__` and :attr:`~module.__loader__` will be + set, if not set already, with the appropriate values. The spec's loader + will be set to the module's :attr:`!__loader__` (if set) and to an instance + of :class:`~importlib.machinery.SourceFileLoader` otherwise. - The module's :attr:`__file__` attribute will be set to the code object's - :attr:`!co_filename`. If applicable, :attr:`__cached__` will also - be set. + The module's :attr:`~module.__file__` attribute will be set to the code + object's :attr:`~codeobject.co_filename`. If applicable, + :attr:`~module.__cached__` will also be set. This function will reload the module if it was already imported. See :c:func:`PyImport_ReloadModule` for the intended way to reload a module. @@ -167,29 +142,29 @@ Importing Modules :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. versionchanged:: 3.12 - The setting of :attr:`__cached__` and :attr:`__loader__` is - deprecated. See :class:`~importlib.machinery.ModuleSpec` for + The setting of :attr:`~module.__cached__` and :attr:`~module.__loader__` + is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. .. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) - Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of - the module object is set to *pathname* if it is non-``NULL``. + Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`~module.__file__` + attribute of the module object is set to *pathname* if it is non-``NULL``. See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. .. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` + Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`~module.__cached__` attribute of the module object is set to *cpathname* if it is non-``NULL``. Of the three functions, this is the preferred one to use. .. versionadded:: 3.3 .. versionchanged:: 3.12 - Setting :attr:`__cached__` is deprecated. See + Setting :attr:`~module.__cached__` is deprecated. See :class:`~importlib.machinery.ModuleSpec` for alternatives. @@ -202,7 +177,7 @@ Importing Modules .. versionadded:: 3.2 .. versionchanged:: 3.3 - Uses :func:`!imp.source_from_cache()` in calculating the source path if + Uses :func:`!imp.source_from_cache` in calculating the source path if only the bytecode path is provided. .. versionchanged:: 3.12 No longer uses the removed :mod:`!imp` module. @@ -320,7 +295,7 @@ Importing Modules The module name, as an ASCII encoded string. - .. c: member:: PyObject* (*initfunc)(void) + .. c:member:: PyObject* (*initfunc)(void) Initialization function for a module built into the interpreter. @@ -337,3 +312,24 @@ Importing Modules If Python is initialized multiple times, :c:func:`PyImport_AppendInittab` or :c:func:`PyImport_ExtendInittab` must be called before each Python initialization. + + +.. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) + + Import the module *mod_name* and get its attribute *attr_name*. + + Names must be Python :class:`str` objects. + + Helper function combining :c:func:`PyImport_Import` and + :c:func:`PyObject_GetAttr`. For example, it can raise :exc:`ImportError` if + the module is not found, and :exc:`AttributeError` if the attribute doesn't + exist. + + .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_ImportModuleAttrString(const char *mod_name, const char *attr_name) + + Similar to :c:func:`PyImport_ImportModuleAttr`, but names are UTF-8 encoded + strings instead of Python :class:`str` objects. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 9a8f1507b3f4cc..ba56b03c6ac8e7 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -25,3 +25,4 @@ document the API functions in detail. memory.rst objimpl.rst apiabiversion.rst + monitoring.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dd53fe2bc95696..9c866438b487d0 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -7,7 +7,8 @@ Initialization, Finalization, and Threads ***************************************** -See also :ref:`Python Initialization Configuration <init-config>`. +See :ref:`Python Initialization Configuration <init-config>` for details +on how to configure the interpreter prior to initialization. .. _pre-init-safe: @@ -21,6 +22,15 @@ a few functions and the :ref:`global configuration variables The following functions can be safely called before Python is initialized: +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + * Configuration functions: * :c:func:`PyImport_AppendInittab` @@ -29,7 +39,10 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_SetAllocator` * :c:func:`PyMem_SetupDebugHooks` * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` * :c:func:`PySys_ResetWarnOptions` + * the configuration functions covered in :ref:`init-config` * Informative functions: @@ -41,10 +54,12 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_GetCopyright` * :c:func:`Py_GetPlatform` * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` * Utilities: * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` * Memory allocators: @@ -53,13 +68,17 @@ The following functions can be safely called before Python is initialized: * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + .. note:: - The following functions **should not be called** before - :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, - :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - and :c:func:`Py_GetProgramName`. + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. .. _global-conf-vars: @@ -87,7 +106,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_DebugFlag @@ -101,7 +120,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_DontWriteBytecodeFlag @@ -115,7 +134,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_FrozenFlag @@ -123,12 +142,9 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. :c:member:`PyConfig.pathconfig_warnings` should be used instead, see :ref:`Python Initialization Configuration <init-config>`. - Suppress error messages when calculating the module search path in - :c:func:`Py_GetPath`. - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_HashRandomizationFlag @@ -143,7 +159,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_IgnoreEnvironmentFlag @@ -156,7 +172,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-E` and :option:`-I` options. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_InspectFlag @@ -171,7 +187,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_InteractiveFlag @@ -181,7 +197,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option. - .. deprecated:: 3.12 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_IsolatedFlag @@ -196,7 +212,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_LegacyWindowsFSEncodingFlag @@ -215,7 +231,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_LegacyWindowsStdioFlag @@ -233,7 +249,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_NoSiteFlag @@ -248,7 +264,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_NoUserSiteDirectory @@ -262,7 +278,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_OptimizeFlag @@ -273,7 +289,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_QuietFlag @@ -287,7 +303,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_UnbufferedStdioFlag @@ -300,7 +316,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 .. c:var:: int Py_VerboseFlag @@ -316,7 +332,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment variable. - .. deprecated-removed:: 3.12 3.14 + .. deprecated-removed:: 3.12 3.15 Initializing and finalizing the interpreter @@ -326,46 +342,55 @@ Initializing and finalizing the interpreter .. c:function:: void Py_Initialize() .. index:: + single: PyEval_InitThreads() single: modules (in module sys) single: path (in module sys) pair: module; builtins pair: module; __main__ pair: module; sys triple: module; search; path - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) Initialize the Python interpreter. In an application embedding Python, this should be called before using any other Python/C API functions; see :ref:`Before Python Initialization <pre-init-safe>` for the few exceptions. - This initializes - the table of loaded modules (``sys.modules``), and creates the fundamental - modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes - the module search path (``sys.path``). It does not set ``sys.argv``; use - the new :c:type:`PyConfig` API of the :ref:`Python Initialization - Configuration <init-config>` for that. This is a no-op when called for a - second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a - fatal error if the initialization fails. - - Use the :c:func:`Py_InitializeFromConfig` function to customize the + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration <init-config>` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration <init-config>`. .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, which will - also affect non-Python uses of the console using the C Runtime. + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. .. c:function:: void Py_InitializeEx(int initsigs) This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, which - might be useful when Python is embedded. + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. - Use the :c:func:`Py_InitializeFromConfig` function to customize the + Use :c:func:`Py_InitializeFromConfig` to customize the :ref:`Python Initialization Configuration <init-config>`. +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + .. c:function:: int Py_IsInitialized() Return true (nonzero) when the Python interpreter has been initialized, false @@ -373,16 +398,36 @@ Initializing and finalizing the interpreter :c:func:`Py_Initialize` is called again. +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down <interpreter shutdown>`. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + .. c:function:: int Py_FinalizeEx() Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of Python/C API functions, and destroy all sub-interpreters (see :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is ``0``. If there were errors during finalization - (flushing buffered data), ``-1`` is returned. + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the preveiously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -398,154 +443,163 @@ Initializing and finalizing the interpreter loaded extension modules loaded by Python are not unloaded. Small amounts of memory allocated by the Python interpreter may not be freed (if you find a leak, please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx .. versionadded:: 3.6 + .. c:function:: void Py_Finalize() This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that disregards the return value. -Process-wide parameters -======================= - - -.. c:function:: wchar_t* Py_GetProgramName() - - Return the program name set with :c:member:`PyConfig.program_name`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. +.. c:function:: int Py_BytesMain(int argc, char **argv) - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.executable` instead. + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + .. versionadded:: 3.8 -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.prefix``. - It is only useful on Unix. See also the next function. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.prefix` instead. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.exec_prefix``. It is only useful on Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.exec_prefix` instead. - - -.. c:function:: wchar_t* Py_GetProgramFullPath() - - .. index:: - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:member:`PyConfig.program_name`). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value will be ``0`` if the interpreter exits normally (i.e., + without an exception), ``1`` if the interpreter exits due to an exception, + or ``2`` if the argument list does not represent a valid Python command + line. + + Note that if an otherwise unhandled :exc:`SystemExit` is raised, this + function will not return ``1``, but exit the process, as long as + ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will + drop into the interactive Python prompt, at which point a second otherwise + unhandled :exc:`SystemExit` will still exit the process, while any other + means of exiting will set the return value as described above. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration <init-config>` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), or ``1`` if the interpreter exits due to an exception. If an + otherwise unhandled :exc:`SystemExit` is raised, the function will immediately + exit the process instead of returning ``1``. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: returning ``0`` + if the session terminates without raising an unhandled exception, exiting + immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for + any other unhandled exception. + + This function always finalizes the Python interpreter regardless of whether + it returns a value or immediately exits the process due to an unhandled + :exc:`SystemExit` exception. + + See :ref:`Python Configuration <init-python-config>` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.executable` instead. +Process-wide parameters +======================= -.. c:function:: wchar_t* Py_GetPath() +.. c:function:: void Py_SetProgramName(const wchar_t *name) .. index:: - triple: module; search; path - single: path (in module sys) - - Return the default module search path; this is computed from the program name - (set by :c:member:`PyConfig.program_name`) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and macOS, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. + single: Py_Initialize() + single: main() - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration <init-config>`. - .. XXX should give the exact rules + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. - .. deprecated-removed:: 3.13 3.15 - Get :data:`sys.path` instead. + .. deprecated-removed:: 3.11 3.15 .. c:function:: const char* Py_GetVersion() @@ -618,21 +672,104 @@ Process-wide parameters ``sys.version``. -.. c:function:: wchar_t* Py_GetPythonHome() +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration <init-config>`. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration <init-config>`. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; + check w/ Guido. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration <init-config>`. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration <init-config>`. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.15 - Return the default "home", that is, the value set by - :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration <init-config>`. - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. - .. deprecated-removed:: 3.13 3.15 - Get :c:member:`PyConfig.home` or :envvar:`PYTHONHOME` environment - variable instead. + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 .. _threads: @@ -645,7 +782,8 @@ Thread State and the Global Interpreter Lock single: interpreter lock single: lock, interpreter -The Python interpreter is not fully thread-safe. In order to support +Unless on a :term:`free-threaded <free threading>` build of :term:`CPython`, +the Python interpreter is not fully thread-safe. In order to support multi-threaded Python programs, there's a global lock, called the :term:`global interpreter lock` or :term:`GIL`, that must be held by the current thread before it can safely access Python objects. Without the lock, even the simplest @@ -653,7 +791,7 @@ operations could cause problems in a multi-threaded program: for example, when two threads simultaneously increment the reference count of the same object, the reference count could end up being incremented only once instead of twice. -.. index:: single: setswitchinterval() (in module sys) +.. index:: single: setswitchinterval (in module sys) Therefore, the rule exists that only the thread that has acquired the :term:`GIL` may operate on Python objects or call Python/C API functions. @@ -663,24 +801,33 @@ released around potentially blocking I/O operations like reading or writing a file, so that other Python threads can run in the meantime. .. index:: - single: PyThreadState - single: PyThreadState + single: PyThreadState (C type) The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`. There's also one -global variable pointing to the current :c:type:`PyThreadState`: it can -be retrieved using :c:func:`PyThreadState_Get`. - -Releasing the GIL from extension code -------------------------------------- - -Most extension code manipulating the :term:`GIL` has the following simple +inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached <attached thread state>`. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the :term:`GIL`, except on +:term:`free-threaded <free threading>` builds. On builds with the :term:`GIL` enabled, +:term:`attaching <attached thread state>` a thread state will block until the :term:`GIL` +can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required +to have an attached thread state to call most of the C API. + +In general, there will always be an :term:`attached thread state` when using Python's C API. +Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the +thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns +``NULL``. + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple structure:: Save the thread state in a local variable. - Release the global interpreter lock. ... Do some blocking I/O operation ... - Reacquire the global interpreter lock. Restore the thread state from the local variable. This is so common that a pair of macros exists to simplify it:: @@ -690,8 +837,8 @@ This is so common that a pair of macros exists to simplify it:: Py_END_ALLOW_THREADS .. index:: - single: Py_BEGIN_ALLOW_THREADS - single: Py_END_ALLOW_THREADS + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the @@ -706,24 +853,33 @@ The block above expands to the following code:: PyEval_RestoreThread(_save); .. index:: - single: PyEval_RestoreThread() - single: PyEval_SaveThread() - -Here is how these functions work: the global interpreter lock is used to protect the pointer to the -current thread state. When releasing the lock and saving the thread state, -the current thread state pointer must be retrieved before the lock is released -(since another thread could immediately acquire the lock and store its own thread -state in the global variable). Conversely, when acquiring the lock and restoring -the thread state, the lock must be acquired before storing the thread state -pointer. + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching +the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach +a thread state to their own thread, thus getting the :term:`GIL` and can start executing. +The pointer to the prior :term:`attached thread state` is stored as a local variable. +Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was +previously :term:`attached <attached thread state>` is passed to :c:func:`PyEval_RestoreThread`. +This function will block until another releases its :term:`thread state <attached thread state>`, +thus allowing the old :term:`thread state <attached thread state>` to get re-attached and the +C API can be called again. + +For :term:`free-threaded <free threading>` builds, the :term:`GIL` is normally +out of the question, but detaching the :term:`thread state <attached thread state>` is still required +for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` +to be released to attach their thread state, allowing true multi-core parallelism. .. note:: - Calling system I/O functions is the most common use case for releasing - the GIL, but it can also be useful before calling long-running computations - which don't need access to Python objects, such as compression or - cryptographic functions operating over memory buffers. For example, the - standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when - compressing or hashing data. + Calling system I/O functions is the most common use case for detaching + the :term:`thread state <attached thread state>`, but it can also be useful before calling + long-running computations which don't need access to Python objects, such + as compression or cryptographic functions operating over memory buffers. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state <attached thread state>` when compressing or hashing data. .. _gilstate: @@ -735,16 +891,15 @@ When threads are created using the dedicated Python APIs (such as the :mod:`threading` module), a thread state is automatically associated to them and the code showed above is therefore correct. However, when threads are created from C (for example by a third-party library with its own thread -management), they don't hold the GIL, nor is there a thread state structure -for them. +management), they don't hold the :term:`GIL`, because they don't have an +:term:`attached thread state`. If you need to call Python code from these threads (often this will be part of a callback API provided by the aforementioned third-party library), you must first register these threads with the interpreter by -creating a thread state data structure, then acquiring the GIL, and finally -storing their thread state pointer, before you can start using the Python/C -API. When you are done, you should reset the thread state pointer, release -the GIL, and finally free the thread state data structure. +creating an :term:`attached thread state` before you can start using the Python/C +API. When you are done, you should detach the :term:`thread state <attached thread state>`, and +finally free it. The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do all of the above automatically. The typical idiom for calling into Python @@ -764,8 +919,36 @@ Note that the ``PyGILState_*`` functions assume there is only one global interpreter (created automatically by :c:func:`Py_Initialize`). Python supports the creation of additional interpreters (using :c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. +``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` +and similar functions default to :term:`attaching <attached thread state>` a +:term:`thread state` for the main interpreter, meaning that the thread can't safely +interact with the calling subinterpreter. + +Supporting subinterpreters in non-Python threads +------------------------------------------------ + +If you would like to support subinterpreters with non-Python created threads, you +must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` +API. + +In particular, you must store the interpreter state from the calling +function and pass it to :c:func:`PyThreadState_New`, which will ensure that +the :term:`thread state` is targeting the correct interpreter:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = ThreadData->interp; + PyThreadState *tstate = PyThreadState_New(interp); + PyThreadState_Swap(tstate); + + /* GIL of the subinterpreter is now held. + Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); .. _fork-and-threads: @@ -804,6 +987,34 @@ thread, where the CPython global runtime was originally initialized. The only exception is if :c:func:`exec` will be called immediately after. +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to attach a :term:`thread state` +during finalization, either explicitly or +implicitly, the thread enters **a permanently blocked state** +where it remains until the program exits. In most cases this is harmless, but this can result +in deadlock if a later stage of finalization attempts to acquire a lock owned by the +blocked thread, or otherwise waits on the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs +have never had any error reporting or handling expectations at :term:`thread state` +attachment time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + High-level API -------------- @@ -834,40 +1045,89 @@ code, or when embedding the Python interpreter: This thread's interpreter state. +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + .. c:function:: PyThreadState* PyEval_SaveThread() - Release the global interpreter lock (if it has been created) and reset the - thread state to ``NULL``, returning the previous thread state (which is not - ``NULL``). If the lock has been created, the current thread must have - acquired it. + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. .. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - Acquire the global interpreter lock (if it has been created) and set the - thread state to *tstate*, which must not be ``NULL``. If the lock has been - created, the current thread must not have acquired it, otherwise deadlock - ensues. + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached <attached thread state>`, + otherwise deadlock ensues. *tstate* will be attached upon returning. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: PyThreadState* PyThreadState_Get() - Return the current thread state. The global interpreter lock must be held. - When the current thread state is ``NULL``, this issues a fatal error (so that - the caller needn't check for ``NULL``). + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. .. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - Swap the current thread state with the thread state given by the argument - *tstate*, which may be ``NULL``. The global interpreter lock must be held - and is not released. + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. The following functions use thread-local storage, and are not compatible @@ -876,7 +1136,7 @@ with sub-interpreters: .. c:function:: PyGILState_STATE PyGILState_Ensure() Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the global interpreter lock. This may + of the current state of Python, or of the :term:`attached thread state`. This may be called as many times as desired by a thread as long as each call is matched with a call to :c:func:`PyGILState_Release`. In general, other thread-related APIs may be used between :c:func:`PyGILState_Ensure` and @@ -885,22 +1145,25 @@ with sub-interpreters: :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is acceptable. - The return value is an opaque "handle" to the thread state when + The return value is an opaque "handle" to the :term:`attached thread state` when :c:func:`PyGILState_Ensure` was called, and must be passed to :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even though recursive calls are allowed, these handles *cannot* be shared - each unique call to :c:func:`PyGILState_Ensure` must save the handle for its call to :c:func:`PyGILState_Release`. - When the function returns, the current thread will hold the GIL and be able - to call arbitrary Python code. Failure is a fatal error. + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. Failure is a fatal error. - .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. .. c:function:: void PyGILState_Release(PyGILState_STATE) @@ -912,26 +1175,37 @@ with sub-interpreters: Every call to :c:func:`PyGILState_Ensure` must be matched by a call to :c:func:`PyGILState_Release` on the same thread. - .. c:function:: PyThreadState* PyGILState_GetThisThreadState() - Get the current thread state for this thread. May return ``NULL`` if no + Get the :term:`attached thread state` for this thread. May return ``NULL`` if no GILState API has been used on the current thread. Note that the main thread always has such a thread-state, even if no auto-thread-state call has been made on the main thread. This is mainly a helper/diagnostic function. + .. note:: + This function does not account for :term:`thread states <thread state>` created + by something other than :c:func:`PyGILState_Ensure` (such as :c:func:`PyThreadState_New`). + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. seealso: :c:func:`PyThreadState_Get`` .. c:function:: int PyGILState_Check() - Return ``1`` if the current thread is holding the GIL and ``0`` otherwise. + Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the GIL will it return ``1``. + Only if it has had its :term:`thread state <attached thread state>` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. This is mainly a helper/diagnostic function. It can be useful for example in callback contexts or memory allocation functions when - knowing that the GIL is locked can allow the caller to perform sensitive + knowing that the :term:`GIL` is locked can allow the caller to perform sensitive actions or otherwise behave differently. + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + .. versionadded:: 3.4 @@ -974,13 +1248,14 @@ Low-level API All of the following functions must be called after :c:func:`Py_Initialize`. .. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL`. + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. .. c:function:: PyInterpreterState* PyInterpreterState_New() - Create a new interpreter state object. The global interpreter lock need not - be held, but may be held if it is necessary to serialize calls to this + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this function. .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New @@ -988,50 +1263,52 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - Reset all information in an interpreter state object. The global interpreter - lock must be held. + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the the interpreter. .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear .. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - Destroy an interpreter state object. The global interpreter lock need not be - held. The interpreter state must have been reset with a previous call to - :c:func:`PyInterpreterState_Clear`. + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. .. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) Create a new thread state object belonging to the given interpreter object. - The global interpreter lock need not be held, but may be held if it is - necessary to serialize calls to this function. - + An :term:`attached thread state` is not needed. .. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - Reset all information in a thread state object. The global interpreter lock - must be held. + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached <attached thread state>` .. versionchanged:: 3.9 This function now calls the :c:member:`PyThreadState.on_delete` callback. Previously, that happened in :c:func:`PyThreadState_Delete`. + .. versionchanged:: 3.13 + The :c:member:`PyThreadState.on_delete` callback was removed. + .. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - Destroy a thread state object. The global interpreter lock need not be held. - The thread state must have been reset with a previous call to + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached <attached thread state>` to any thread. + *tstate* must have been reset with a previous call to :c:func:`PyThreadState_Clear`. .. c:function:: void PyThreadState_DeleteCurrent(void) - Destroy the current thread state and release the global interpreter lock. - Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not - be held. The thread state must have been reset with a previous call - to :c:func:`PyThreadState_Clear`. + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + No :term:`thread state` will be :term:`attached <attached thread state>` upon + returning. .. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) @@ -1042,16 +1319,16 @@ All of the following functions must be called after :c:func:`Py_Initialize`. See also :c:func:`PyEval_GetFrame`. - *tstate* must not be ``NULL``. + *tstate* must not be ``NULL``, and must be :term:`attached <attached thread state>`. .. versionadded:: 3.9 .. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - Get the unique thread state identifier of the Python thread state *tstate*. + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - *tstate* must not be ``NULL``. + *tstate* must not be ``NULL``, and must be :term:`attached <attached thread state>`. .. versionadded:: 3.9 @@ -1060,7 +1337,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Get the interpreter of the Python thread state *tstate*. - *tstate* must not be ``NULL``. + *tstate* must not be ``NULL``, and must be :term:`attached <attached thread state>`. .. versionadded:: 3.9 @@ -1089,10 +1366,8 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Get the current interpreter. - Issue a fatal error if there no current Python thread state or no current - interpreter. It cannot return NULL. - - The caller must hold the GIL. + Issue a fatal error if there no :term:`attached thread state`. + It cannot return NULL. .. versionadded:: 3.9 @@ -1102,7 +1377,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Return the interpreter's unique ID. If there was any error in doing so then ``-1`` is returned and an error is set. - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. .. versionadded:: 3.7 @@ -1118,6 +1393,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.8 + .. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) Type of a frame evaluation function. @@ -1152,9 +1428,10 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Return a dictionary in which extensions can store thread-specific state information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no current thread state - is available. If this function returns ``NULL``, no exception has been raised and - the caller should assume no current thread state is available. + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached <attached thread state>`. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. .. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) @@ -1162,7 +1439,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Asynchronously raise an exception in a thread. The *id* argument is the thread id of the target thread; *exc* is the exception object to be raised. This function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with the GIL held. + must write your own C extension to call this. Must be called with an :term:`attached thread state`. Returns the number of thread states modified; this is normally one, but will be zero if the thread id isn't found. If *exc* is ``NULL``, the pending exception (if any) for the thread is cleared. This raises no exceptions. @@ -1173,32 +1450,35 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - Acquire the global interpreter lock and set the current thread state to - *tstate*, which must not be ``NULL``. The lock must have been created earlier. - If this thread already has the lock, deadlock ensues. + :term:`Attach <attached thread state>` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached <attached thread state>`. + + The calling thread must not already have an :term:`attached thread state`. .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.8 Updated to be consistent with :c:func:`PyEval_RestoreThread`, :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, and terminate the current thread if called while the interpreter is finalizing. + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + :c:func:`PyEval_RestoreThread` is a higher-level function which is always available (even when threads have not been initialized). .. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - Reset the current thread state to ``NULL`` and release the global interpreter - lock. The lock must have been created earlier and must be held by the current - thread. The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the current thread state --- if it isn't, a fatal error is + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is reported. :c:func:`PyEval_SaveThread` is a higher-level function which is always @@ -1338,23 +1618,23 @@ function. You can create and destroy them using the following functions: The given *config* controls the options with which the interpreter is initialized. - Upon success, *tstate_p* will be set to the first thread state - created in the new - sub-interpreter. This thread state is made in the current thread state. + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached <attached thread state>`. Note that no actual thread is created; see the discussion of thread states below. If creation of the new interpreter is unsuccessful, *tstate_p* is set to ``NULL``; no exception is set since the exception state is stored in the - current thread state and there may not be a current thread state. + :term:`attached thread state`, which might not exist. - Like all other Python/C API functions, the global interpreter lock - must be held before calling this function and is still held when it - returns. Likewise a current thread state must be set on entry. On - success, the returned thread state will be set as current. If the - sub-interpreter is created with its own GIL then the GIL of the - calling interpreter will be released. When the function returns, - the new interpreter's GIL will be held by the current thread and - the previously interpreter's GIL will remain released here. + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached <attached thread state>`. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached <attached thread state>` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. .. versionadded:: 3.12 @@ -1370,7 +1650,11 @@ function. You can create and destroy them using the following functions: .check_multi_interp_extensions = 1, .gil = PyInterpreterConfig_OWN_GIL, }; - PyThreadState *tstate = Py_NewInterpreterFromConfig(&config); + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } Note that the config is used only briefly and does not get modified. During initialization the config's values are converted into various @@ -1378,8 +1662,8 @@ function. You can create and destroy them using the following functions: may be stored internally on the :c:type:`PyInterpreterState`. .. index:: - single: Py_FinalizeEx() - single: Py_Initialize() + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) Extension modules are shared between (sub-)interpreters as follows: @@ -1407,7 +1691,7 @@ function. You can create and destroy them using the following functions: As with multi-phase initialization, this means that only C-level static and global variables are shared between these modules. - .. index:: single: close() (in module os) + .. index:: single: close (in module os) .. c:function:: PyThreadState* Py_NewInterpreter(void) @@ -1430,15 +1714,12 @@ function. You can create and destroy them using the following functions: .. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - .. index:: single: Py_FinalizeEx() + .. index:: single: Py_FinalizeEx (C function) - Destroy the (sub-)interpreter represented by the given thread state. - The given thread state must be the current thread state. See the - discussion of thread states below. When the call returns, - the current thread state is ``NULL``. All thread states associated - with this interpreter are destroyed. The global interpreter lock - used by the target interpreter must be held before calling this - function. No GIL is held when it returns. + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached <attached thread state>`. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that haven't been explicitly destroyed at that point. @@ -1464,7 +1745,7 @@ otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared because of the refcount. One simple but less-efficient approach around this is to use a global lock around all use of some state (or object). Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them "immortal". +can be made safe in spite of their refcounts by making them :term:`immortal`. In fact, this has been done for the builtin singletons, small integers, and a number of other builtin objects. @@ -1522,8 +1803,6 @@ pointer and a void pointer argument. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - .. index:: single: Py_AddPendingCall() - Schedule a function to be called from the main interpreter thread. On success, ``0`` is returned and *func* is queued for being called in the main thread. On failure, ``-1`` is returned without setting any exception. @@ -1534,20 +1813,17 @@ pointer and a void pointer argument. both these conditions met: * on a :term:`bytecode` boundary; - * with the main thread holding the :term:`global interpreter lock` + * with the main thread holding an :term:`attached thread state` (*func* can therefore use the full C API). *func* must return ``0`` on success, or ``-1`` on failure with an exception set. *func* won't be interrupted to perform another asynchronous notification recursively, but it can still be interrupted to switch - threads if the global interpreter lock is released. - - This function doesn't need a current thread state to run, and it doesn't - need the global interpreter lock. + threads if the :term:`thread state <attached thread state>` is detached. - To call this function in a subinterpreter, the caller must hold the GIL. - Otherwise, the function *func* can be scheduled to be called from the wrong - interpreter. + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. .. warning:: This is a low-level function, only useful for very special cases. @@ -1557,14 +1833,14 @@ pointer and a void pointer argument. function is generally **not** suitable for calling Python code from arbitrary C threads. Instead, use the :ref:`PyGILState API<gilstate>`. + .. versionadded:: 3.1 + .. versionchanged:: 3.9 If this function is called in a subinterpreter, the function *func* is now scheduled to be called from the subinterpreter, rather than being called from the main interpreter. Each subinterpreter now has its own list of scheduled calls. - .. versionadded:: 3.1 - .. _profiling: Profiling and Tracing @@ -1641,7 +1917,8 @@ Python-level trace functions in previous versions. The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. .. c:var:: int PyTrace_RETURN @@ -1673,7 +1950,7 @@ Python-level trace functions in previous versions. The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not profiling functions) when a new opcode is about to be executed. This event is not emitted by default: it must be explicitly requested by setting - :attr:`f_trace_opcodes` to *1* on the frame. + :attr:`~frame.f_trace_opcodes` to *1* on the frame. .. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) @@ -1687,14 +1964,14 @@ Python-level trace functions in previous versions. See also the :func:`sys.setprofile` function. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. .. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads belonging to the current interpreter instead of the setting it only on the current thread. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while setting the profile functions in all threads. @@ -1713,20 +1990,72 @@ Python-level trace functions in previous versions. See also the :func:`sys.settrace` function. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. .. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads belonging to the current interpreter instead of the setting it only on the current thread. - The caller must hold the :term:`GIL`. + The caller must have an :term:`attached thread state`. As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while setting the trace functions in all threads. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: @@ -1781,8 +2110,8 @@ CPython C level APIs are similar to those offered by pthreads and Windows: use a thread key and functions to associate a :c:expr:`void*` value per thread. -The GIL does *not* need to be held when calling these functions; they supply -their own locking. +A :term:`thread state` does *not* need to be :term:`attached <attached thread state>` +when calling these functions; they suppl their own locking. Note that :file:`Python.h` does not include the declaration of the TLS APIs, you need to include :file:`pythread.h` to use thread-local storage. @@ -1923,3 +2252,145 @@ be used in new code. .. c:function:: void PyThread_delete_key_value(int key) .. c:function:: void PyThread_ReInitTLS() +Synchronization Primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily detach the :term:`thread state <attached thread state>` if one exists. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. _python-critical-section-api: + +Python Critical Section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded <free threading>` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. +When :c:func:`PyEval_RestoreThread` is called, the most recent critical section +is resumed, and its locks reacquired. This means the critical section API +provides weaker guarantees than traditional locks -- they are useful because +their behavior is similar to the :term:`GIL`. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-objects locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 3ad2d435665f89..4fd10224262488 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -6,6 +6,629 @@ Python Initialization Configuration *********************************** + +.. _pyinitconfig_api: + +PyInitConfig C API +================== + +.. versionadded:: 3.14 + +Python can be initialized with :c:func:`Py_InitializeFromInitConfig`. + +The :c:func:`Py_RunMain` function can be used to write a customized Python +program. + +See also :ref:`Initialization, Finalization, and Threads <initialization>`. + +.. seealso:: + :pep:`741` "Python Configuration C API". + + +Example +------- + +Example of customized Python always running with the :ref:`Python Development +Mode <devmode>` enabled; return ``-1`` on error: + +.. code-block:: c + + int init_python(void) + { + PyInitConfig *config = PyInitConfig_Create(); + if (config == NULL) { + printf("PYTHON INIT ERROR: memory allocation failed\n"); + return -1; + } + + // Enable the Python Development Mode + if (PyInitConfig_SetInt(config, "dev_mode", 1) < 0) { + goto error; + } + + // Initialize Python with the configuration + if (Py_InitializeFromInitConfig(config) < 0) { + goto error; + } + PyInitConfig_Free(config); + return 0; + + error: + { + // Display the error message. + // + // This uncommon braces style is used, because you cannot make + // goto targets point to variable declarations. + const char *err_msg; + (void)PyInitConfig_GetError(config, &err_msg); + printf("PYTHON INIT ERROR: %s\n", err_msg); + PyInitConfig_Free(config); + return -1; + } + } + +Create Config +------------- + +.. c:struct:: PyInitConfig + + Opaque structure to configure the Python initialization. + + +.. c:function:: PyInitConfig* PyInitConfig_Create(void) + + Create a new initialization configuration using :ref:`Isolated Configuration + <init-isolated-conf>` default values. + + It must be freed by :c:func:`PyInitConfig_Free`. + + Return ``NULL`` on memory allocation failure. + + +.. c:function:: void PyInitConfig_Free(PyInitConfig *config) + + Free memory of the initialization configuration *config*. + + If *config* is ``NULL``, no operation is performed. + + +Error Handling +-------------- + +.. c:function:: int PyInitConfig_GetError(PyInitConfig* config, const char **err_msg) + + Get the *config* error message. + + * Set *\*err_msg* and return ``1`` if an error is set. + * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. + + An error message is an UTF-8 encoded string. + + If *config* has an exit code, format the exit code as an error + message. + + The error message remains valid until another ``PyInitConfig`` + function is called with *config*. The caller doesn't have to free the + error message. + + +.. c:function:: int PyInitConfig_GetExitCode(PyInitConfig* config, int *exitcode) + + Get the *config* exit code. + + * Set *\*exitcode* and return ``1`` if *config* has an exit code set. + * Return ``0`` if *config* has no exit code set. + + Only the ``Py_InitializeFromInitConfig()`` function can set an exit + code if the ``parse_argv`` option is non-zero. + + An exit code can be set when parsing the command line failed (exit + code ``2``) or when a command line option asks to display the command + line help (exit code ``0``). + + +Get Options +----------- + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options <pyinitconfig-opts>`. + +.. c:function:: int PyInitConfig_HasOption(PyInitConfig *config, const char *name) + + Test if the configuration has an option called *name*. + + Return ``1`` if the option exists, or return ``0`` otherwise. + + +.. c:function:: int PyInitConfig_GetInt(PyInitConfig *config, const char *name, int64_t *value) + + Get an integer configuration option. + + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_GetStr(PyInitConfig *config, const char *name, char **value) + + Get a string configuration option as a null-terminated UTF-8 + encoded string. + + * Set *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + *\*value* can be set to ``NULL`` if the option is an optional string and the + option is unset. + + On success, the string must be released with ``free(value)`` if it's not + ``NULL``. + + +.. c:function:: int PyInitConfig_GetStrList(PyInitConfig *config, const char *name, size_t *length, char ***items) + + Get a string list configuration option as an array of + null-terminated UTF-8 encoded strings. + + * Set *\*length* and *\*value*, and return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + On success, the string list must be released with + ``PyInitConfig_FreeStrList(length, items)``. + + +.. c:function:: void PyInitConfig_FreeStrList(size_t length, char **items) + + Free memory of a string list created by + ``PyInitConfig_GetStrList()``. + + +Set Options +----------- + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options <pyinitconfig-opts>`. + +Some configuration options have side effects on other options. This logic is +only implemented when ``Py_InitializeFromInitConfig()`` is called, not by the +"Set" functions below. For example, setting ``dev_mode`` to ``1`` does not set +``faulthandler`` to ``1``. + +.. c:function:: int PyInitConfig_SetInt(PyInitConfig *config, const char *name, int64_t value) + + Set an integer configuration option. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char *value) + + Set a string configuration option from a null-terminated UTF-8 + encoded string. The string is copied. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +.. c:function:: int PyInitConfig_SetStrList(PyInitConfig *config, const char *name, size_t length, char * const *items) + + Set a string list configuration option from an array of + null-terminated UTF-8 encoded strings. The string list is copied. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + +Module +------ + +.. c:function:: int PyInitConfig_AddModule(PyInitConfig *config, const char *name, PyObject* (*initfunc)(void)) + + Add a built-in extension module to the table of built-in modules. + + The new module can be imported by the name *name*, and uses the function + *initfunc* as the initialization function called on the first attempted + import. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + + If Python is initialized multiple times, ``PyInitConfig_AddModule()`` must + be called at each Python initialization. + + Similar to the :c:func:`PyImport_AppendInittab` function. + + +Initialize Python +----------------- + +.. c:function:: int Py_InitializeFromInitConfig(PyInitConfig *config) + + Initialize Python from the initialization configuration. + + * Return ``0`` on success. + * Set an error in *config* and return ``-1`` on error. + * Set an exit code in *config* and return ``-1`` if Python wants to + exit. + + See ``PyInitConfig_GetExitcode()`` for the exit code case. + + +.. _pyinitconfig-opts: + +Configuration Options +===================== + +.. list-table:: + :header-rows: 1 + + * - Option + - PyConfig/PyPreConfig member + - Type + - Visibility + * - ``"allocator"`` + - :c:member:`allocator <PyPreConfig.allocator>` + - ``int`` + - Read-only + * - ``"argv"`` + - :c:member:`argv <PyConfig.argv>` + - ``list[str]`` + - Public + * - ``"base_exec_prefix"`` + - :c:member:`base_exec_prefix <PyConfig.base_exec_prefix>` + - ``str`` + - Public + * - ``"base_executable"`` + - :c:member:`base_executable <PyConfig.base_executable>` + - ``str`` + - Public + * - ``"base_prefix"`` + - :c:member:`base_prefix <PyConfig.base_prefix>` + - ``str`` + - Public + * - ``"buffered_stdio"`` + - :c:member:`buffered_stdio <PyConfig.buffered_stdio>` + - ``bool`` + - Read-only + * - ``"bytes_warning"`` + - :c:member:`bytes_warning <PyConfig.bytes_warning>` + - ``int`` + - Public + * - ``"check_hash_pycs_mode"`` + - :c:member:`check_hash_pycs_mode <PyConfig.check_hash_pycs_mode>` + - ``str`` + - Read-only + * - ``"code_debug_ranges"`` + - :c:member:`code_debug_ranges <PyConfig.code_debug_ranges>` + - ``bool`` + - Read-only + * - ``"coerce_c_locale"`` + - :c:member:`coerce_c_locale <PyPreConfig.coerce_c_locale>` + - ``bool`` + - Read-only + * - ``"coerce_c_locale_warn"`` + - :c:member:`coerce_c_locale_warn <PyPreConfig.coerce_c_locale_warn>` + - ``bool`` + - Read-only + * - ``"configure_c_stdio"`` + - :c:member:`configure_c_stdio <PyConfig.configure_c_stdio>` + - ``bool`` + - Read-only + * - ``"configure_locale"`` + - :c:member:`configure_locale <PyPreConfig.configure_locale>` + - ``bool`` + - Read-only + * - ``"cpu_count"`` + - :c:member:`cpu_count <PyConfig.cpu_count>` + - ``int`` + - Public + * - ``"dev_mode"`` + - :c:member:`dev_mode <PyConfig.dev_mode>` + - ``bool`` + - Read-only + * - ``"dump_refs"`` + - :c:member:`dump_refs <PyConfig.dump_refs>` + - ``bool`` + - Read-only + * - ``"dump_refs_file"`` + - :c:member:`dump_refs_file <PyConfig.dump_refs_file>` + - ``str`` + - Read-only + * - ``"exec_prefix"`` + - :c:member:`exec_prefix <PyConfig.exec_prefix>` + - ``str`` + - Public + * - ``"executable"`` + - :c:member:`executable <PyConfig.executable>` + - ``str`` + - Public + * - ``"faulthandler"`` + - :c:member:`faulthandler <PyConfig.faulthandler>` + - ``bool`` + - Read-only + * - ``"filesystem_encoding"`` + - :c:member:`filesystem_encoding <PyConfig.filesystem_encoding>` + - ``str`` + - Read-only + * - ``"filesystem_errors"`` + - :c:member:`filesystem_errors <PyConfig.filesystem_errors>` + - ``str`` + - Read-only + * - ``"hash_seed"`` + - :c:member:`hash_seed <PyConfig.hash_seed>` + - ``int`` + - Read-only + * - ``"home"`` + - :c:member:`home <PyConfig.home>` + - ``str`` + - Read-only + * - ``"import_time"`` + - :c:member:`import_time <PyConfig.import_time>` + - ``int`` + - Read-only + * - ``"inspect"`` + - :c:member:`inspect <PyConfig.inspect>` + - ``bool`` + - Public + * - ``"install_signal_handlers"`` + - :c:member:`install_signal_handlers <PyConfig.install_signal_handlers>` + - ``bool`` + - Read-only + * - ``"int_max_str_digits"`` + - :c:member:`int_max_str_digits <PyConfig.int_max_str_digits>` + - ``int`` + - Public + * - ``"interactive"`` + - :c:member:`interactive <PyConfig.interactive>` + - ``bool`` + - Public + * - ``"isolated"`` + - :c:member:`isolated <PyConfig.isolated>` + - ``bool`` + - Read-only + * - ``"legacy_windows_fs_encoding"`` + - :c:member:`legacy_windows_fs_encoding <PyPreConfig.legacy_windows_fs_encoding>` + - ``bool`` + - Read-only + * - ``"legacy_windows_stdio"`` + - :c:member:`legacy_windows_stdio <PyConfig.legacy_windows_stdio>` + - ``bool`` + - Read-only + * - ``"malloc_stats"`` + - :c:member:`malloc_stats <PyConfig.malloc_stats>` + - ``bool`` + - Read-only + * - ``"module_search_paths"`` + - :c:member:`module_search_paths <PyConfig.module_search_paths>` + - ``list[str]`` + - Public + * - ``"optimization_level"`` + - :c:member:`optimization_level <PyConfig.optimization_level>` + - ``int`` + - Public + * - ``"orig_argv"`` + - :c:member:`orig_argv <PyConfig.orig_argv>` + - ``list[str]`` + - Read-only + * - ``"parse_argv"`` + - :c:member:`parse_argv <PyConfig.parse_argv>` + - ``bool`` + - Read-only + * - ``"parser_debug"`` + - :c:member:`parser_debug <PyConfig.parser_debug>` + - ``bool`` + - Public + * - ``"pathconfig_warnings"`` + - :c:member:`pathconfig_warnings <PyConfig.pathconfig_warnings>` + - ``bool`` + - Read-only + * - ``"perf_profiling"`` + - :c:member:`perf_profiling <PyConfig.perf_profiling>` + - ``bool`` + - Read-only + * - ``"platlibdir"`` + - :c:member:`platlibdir <PyConfig.platlibdir>` + - ``str`` + - Public + * - ``"prefix"`` + - :c:member:`prefix <PyConfig.prefix>` + - ``str`` + - Public + * - ``"program_name"`` + - :c:member:`program_name <PyConfig.program_name>` + - ``str`` + - Read-only + * - ``"pycache_prefix"`` + - :c:member:`pycache_prefix <PyConfig.pycache_prefix>` + - ``str`` + - Public + * - ``"quiet"`` + - :c:member:`quiet <PyConfig.quiet>` + - ``bool`` + - Public + * - ``"run_command"`` + - :c:member:`run_command <PyConfig.run_command>` + - ``str`` + - Read-only + * - ``"run_filename"`` + - :c:member:`run_filename <PyConfig.run_filename>` + - ``str`` + - Read-only + * - ``"run_module"`` + - :c:member:`run_module <PyConfig.run_module>` + - ``str`` + - Read-only + * - ``"run_presite"`` + - :c:member:`run_presite <PyConfig.run_presite>` + - ``str`` + - Read-only + * - ``"safe_path"`` + - :c:member:`safe_path <PyConfig.safe_path>` + - ``bool`` + - Read-only + * - ``"show_ref_count"`` + - :c:member:`show_ref_count <PyConfig.show_ref_count>` + - ``bool`` + - Read-only + * - ``"site_import"`` + - :c:member:`site_import <PyConfig.site_import>` + - ``bool`` + - Read-only + * - ``"skip_source_first_line"`` + - :c:member:`skip_source_first_line <PyConfig.skip_source_first_line>` + - ``bool`` + - Read-only + * - ``"stdio_encoding"`` + - :c:member:`stdio_encoding <PyConfig.stdio_encoding>` + - ``str`` + - Read-only + * - ``"stdio_errors"`` + - :c:member:`stdio_errors <PyConfig.stdio_errors>` + - ``str`` + - Read-only + * - ``"stdlib_dir"`` + - :c:member:`stdlib_dir <PyConfig.stdlib_dir>` + - ``str`` + - Public + * - ``"tracemalloc"`` + - :c:member:`tracemalloc <PyConfig.tracemalloc>` + - ``int`` + - Read-only + * - ``"use_environment"`` + - :c:member:`use_environment <PyConfig.use_environment>` + - ``bool`` + - Public + * - ``"use_frozen_modules"`` + - :c:member:`use_frozen_modules <PyConfig.use_frozen_modules>` + - ``bool`` + - Read-only + * - ``"use_hash_seed"`` + - :c:member:`use_hash_seed <PyConfig.use_hash_seed>` + - ``bool`` + - Read-only + * - ``"use_system_logger"`` + - :c:member:`use_system_logger <PyConfig.use_system_logger>` + - ``bool`` + - Read-only + * - ``"user_site_directory"`` + - :c:member:`user_site_directory <PyConfig.user_site_directory>` + - ``bool`` + - Read-only + * - ``"utf8_mode"`` + - :c:member:`utf8_mode <PyPreConfig.utf8_mode>` + - ``bool`` + - Read-only + * - ``"verbose"`` + - :c:member:`verbose <PyConfig.verbose>` + - ``int`` + - Public + * - ``"warn_default_encoding"`` + - :c:member:`warn_default_encoding <PyConfig.warn_default_encoding>` + - ``bool`` + - Read-only + * - ``"warnoptions"`` + - :c:member:`warnoptions <PyConfig.warnoptions>` + - ``list[str]`` + - Public + * - ``"write_bytecode"`` + - :c:member:`write_bytecode <PyConfig.write_bytecode>` + - ``bool`` + - Public + * - ``"xoptions"`` + - :c:member:`xoptions <PyConfig.xoptions>` + - ``dict[str, str]`` + - Public + * - ``"_pystats"`` + - :c:member:`_pystats <PyConfig._pystats>` + - ``bool`` + - Read-only + +Visibility: + +* Public: Can by get by :c:func:`PyConfig_Get` and set by + :c:func:`PyConfig_Set`. +* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by + :c:func:`PyConfig_Set`. + + +Runtime Python configuration API +================================ + +At runtime, it's possible to get and set configuration options using +:c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions. + +The configuration option *name* parameter must be a non-NULL null-terminated +UTF-8 encoded string. See :ref:`Configuration Options <pyinitconfig-opts>`. + +Some options are read from the :mod:`sys` attributes. For example, the option +``"argv"`` is read from :data:`sys.argv`. + + +.. c:function:: PyObject* PyConfig_Get(const char *name) + + Get the current runtime value of a configuration option as a Python object. + + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. + + The object type depends on the configuration option. It can be: + + * ``bool`` + * ``int`` + * ``str`` + * ``list[str]`` + * ``dict[str, str]`` + + The caller must have an :term:`attached thread state`. The function cannot + be called before Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + + +.. c:function:: int PyConfig_GetInt(const char *name, int *value) + + Similar to :c:func:`PyConfig_Get`, but get the value as a C int. + + * Return ``0`` on success. + * Set an exception and return ``-1`` on error. + + .. versionadded:: 3.14 + + +.. c:function:: PyObject* PyConfig_Names(void) + + Get all configuration option names as a ``frozenset``. + + * Return a new reference on success. + * Set an exception and return ``NULL`` on error. + + The caller must have an :term:`attached thread state`. The function cannot + be called before Python initialization nor after Python finalization. + + .. versionadded:: 3.14 + + +.. c:function:: int PyConfig_Set(const char *name, PyObject *value) + + Set the current runtime value of a configuration option. + + * Raise a :exc:`ValueError` if there is no option *name*. + * Raise a :exc:`ValueError` if *value* is an invalid value. + * Raise a :exc:`ValueError` if the option is read-only (cannot be set). + * Raise a :exc:`TypeError` if *value* has not the proper type. + + The caller must have an :term:`attached thread state`. The function cannot + be called before Python initialization nor after Python finalization. + + .. audit-event:: cpython.PyConfig_Set name,value c.PyConfig_Set + + .. versionadded:: 3.14 + + +.. _pyconfig_api: + +PyConfig C API +============== + .. versionadded:: 3.8 Python can be initialized with :c:func:`Py_InitializeFromConfig` and the @@ -34,7 +657,7 @@ See also :ref:`Initialization, Finalization, and Threads <initialization>`. Example -======= +------- Example of customized Python always running in isolated mode:: @@ -73,7 +696,7 @@ Example of customized Python always running in isolated mode:: PyWideStringList -================ +---------------- .. c:type:: PyWideStringList @@ -82,6 +705,8 @@ PyWideStringList If *length* is non-zero, *items* must be non-``NULL`` and all strings must be non-``NULL``. + .. c:namespace:: NULL + Methods: .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) @@ -101,6 +726,8 @@ PyWideStringList Python must be preinitialized to call this function. + .. c:namespace:: PyWideStringList + Structure fields: .. c:member:: Py_ssize_t length @@ -112,7 +739,7 @@ PyWideStringList List items. PyStatus -======== +-------- .. c:type:: PyStatus @@ -206,7 +833,7 @@ Example:: PyPreConfig -=========== +----------- .. c:type:: PyPreConfig @@ -249,11 +876,21 @@ PyPreConfig * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc memory allocator <pymalloc>` with :ref:`debug hooks <pymem-debug-hooks>`. + * ``PYMEM_ALLOCATOR_MIMALLOC`` (``6``): use ``mimalloc``, a fast + malloc replacement. + * ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` (``7``): use ``mimalloc``, a fast + malloc replacement with :ref:`debug hooks <pymem-debug-hooks>`. + ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are not supported if Python is :option:`configured using --without-pymalloc <--without-pymalloc>`. + ``PYMEM_ALLOCATOR_MIMALLOC`` and ``PYMEM_ALLOCATOR_MIMALLOC_DEBUG`` are + not supported if Python is :option:`configured using --without-mimalloc + <--without-mimalloc>` or if the underlying atomic support isn't + available. + See :ref:`Memory Management <memory>`. Default: ``PYMEM_ALLOCATOR_NOT_SET``. @@ -307,7 +944,7 @@ PyPreConfig * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. - Initialized the from :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + Initialized from the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable value. Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for @@ -346,7 +983,7 @@ PyPreConfig .. _c-preinit: Preinitialize Python with PyPreConfig -===================================== +------------------------------------- The preinitialization of Python: @@ -426,7 +1063,7 @@ the :ref:`Python UTF-8 Mode <utf8-mode>`:: PyConfig -======== +-------- .. c:type:: PyConfig @@ -495,7 +1132,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. :ref:`Preinitialize Python <c-preinit>` if needed. @@ -712,11 +1349,21 @@ PyConfig Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. - Need a special build of Python with the ``Py_TRACE_REFS`` macro defined: + Needs a special build of Python with the ``Py_TRACE_REFS`` macro defined: see the :option:`configure --with-trace-refs option <--with-trace-refs>`. Default: ``0``. + .. c:member:: wchar_t* dump_refs_file + + Filename where to dump Python references. + + Set by the :envvar:`PYTHONDUMPREFSFILE` environment variable. + + Default: ``NULL``. + + .. versionadded:: 3.11 + .. c:member:: wchar_t* exec_prefix The site-specific directory prefix where the platform-dependent Python @@ -795,6 +1442,15 @@ PyConfig See also the :c:member:`~PyConfig.filesystem_encoding` member. + .. c:member:: int use_frozen_modules + + If non-zero, use frozen modules. + + Set by the :envvar:`PYTHON_FROZEN_MODULES` environment variable. + + Default: ``1`` in a release build, or ``0`` in a :ref:`debug build + <debug-build>`. + .. c:member:: unsigned long hash_seed .. c:member:: int use_hash_seed @@ -821,13 +1477,19 @@ PyConfig .. c:member:: int import_time - If non-zero, profile import time. + If ``1``, profile import time. + If ``2``, include additional output that indicates + when an imported module has already been loaded. - Set the ``1`` by the :option:`-X importtime <-X>` option and the + Set by the :option:`-X importtime <-X>` option and the :envvar:`PYTHONPROFILEIMPORTTIME` environment variable. Default: ``0``. + .. versionchanged:: 3.14 + + Added support for ``import_time = 2`` + .. c:member:: int inspect Enter interactive mode after executing a script or a command. @@ -874,6 +1536,19 @@ PyConfig .. versionadded:: 3.12 + .. c:member:: int cpu_count + + If the value of :c:member:`~PyConfig.cpu_count` is not ``-1`` then it will + override the return values of :func:`os.cpu_count`, + :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. + + Configured by the :samp:`-X cpu_count={n|default}` command line + flag or the :envvar:`PYTHON_CPU_COUNT` environment variable. + + Default: ``-1``. + + .. versionadded:: 3.13 + .. c:member:: int isolated If greater than ``0``, enable isolated mode: @@ -1014,7 +1689,7 @@ PyConfig The :c:func:`PyConfig_Read` function only parses :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + stripped from :c:member:`PyConfig.argv`, parsing arguments twice would parse the application options as Python options. Default: ``1`` in Python mode, ``0`` in isolated mode. @@ -1031,7 +1706,7 @@ PyConfig Incremented by the :option:`-d` command line option. Set to the :envvar:`PYTHONDEBUG` environment variable value. - Need a :ref:`debug build of Python <debug-build>` (the ``Py_DEBUG`` macro + Needs a :ref:`debug build of Python <debug-build>` (the ``Py_DEBUG`` macro must be defined). Default: ``0``. @@ -1083,6 +1758,7 @@ PyConfig Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and the :envvar:`PYTHONPYCACHEPREFIX` environment variable. + The command-line option takes precedence. If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. @@ -1126,13 +1802,27 @@ PyConfig Default: ``NULL``. + .. c:member:: wchar_t* run_presite + + ``package.module`` path to module that should be imported before + ``site.py`` is run. + + Set by the :option:`-X presite=package.module <-X>` command-line + option and the :envvar:`PYTHON_PRESITE` environment variable. + The command-line option takes precedence. + + Needs a :ref:`debug build of Python <debug-build>` (the ``Py_DEBUG`` macro + must be defined). + + Default: ``NULL``. + .. c:member:: int show_ref_count - Show total reference count at exit (excluding immortal objects)? + Show total reference count at exit (excluding :term:`immortal` objects)? Set to ``1`` by :option:`-X showrefcount <-X>` command line option. - Need a :ref:`debug build of Python <debug-build>` (the ``Py_REF_DEBUG`` + Needs a :ref:`debug build of Python <debug-build>` (the ``Py_REF_DEBUG`` macro must be defined). Default: ``0``. @@ -1203,18 +1893,34 @@ PyConfig .. c:member:: int perf_profiling - Enable compatibility mode with the perf profiler? + Enable the Linux ``perf`` profiler support? + + If equals to ``1``, enable support for the Linux ``perf`` profiler. - If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` - for more information. + If equals to ``2``, enable support for the Linux ``perf`` profiler with + DWARF JIT support. - Set by :option:`-X perf <-X>` command line option and by the + Set to ``1`` by :option:`-X perf <-X>` command-line option and the :envvar:`PYTHONPERFSUPPORT` environment variable. + Set to ``2`` by the :option:`-X perf_jit <-X>` command-line option and + the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable. + Default: ``-1``. + .. seealso:: + See :ref:`perf_profiling` for more information. + .. versionadded:: 3.12 + .. c:member:: wchar_t* stdlib_dir + + Directory of the Python standard library. + + Default: ``NULL``. + + .. versionadded:: 3.11 + .. c:member:: int use_environment Use :ref:`environment variables <using-on-envvars>`? @@ -1226,6 +1932,18 @@ PyConfig Default: ``1`` in Python config and ``0`` in isolated config. + .. c:member:: int use_system_logger + + If non-zero, ``stdout`` and ``stderr`` will be redirected to the system + log. + + Only available on macOS 10.12 and later, and on iOS. + + Default: ``0`` (don't use the system log) on macOS; ``1`` on iOS (use the + system log). + + .. versionadded:: 3.14 + .. c:member:: int user_site_directory If non-zero, add the user site directory to :data:`sys.path`. @@ -1290,6 +2008,15 @@ PyConfig Default: empty list. + .. c:member:: int _pystats + + If non-zero, write performance statistics at Python exit. + + Need a special build with the ``Py_STATS`` macro: + see :option:`--enable-pystats`. + + Default: ``0``. + If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` arguments are parsed the same way the regular Python parses :ref:`command line arguments <using-on-cmdline>`, and Python arguments are stripped from @@ -1303,14 +2030,13 @@ the :option:`-X` command line option. The ``show_alloc_count`` field has been removed. -Initialization with PyConfig -============================ - -Function to initialize Python: +.. _init-from-config: -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) +Initialization with PyConfig +---------------------------- - Initialize Python from *config* configuration. +Initializing the interpreter from a populated configuration struct is handled +by calling :c:func:`Py_InitializeFromConfig`. The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -1385,7 +2111,7 @@ initialization:: /* Specify sys.path explicitly */ /* If you want to modify the default set of paths, finish - initialization first and then use PySys_GetObject("path") */ + initialization first and then use PySys_GetAttrString("path") */ config.module_search_paths_set = 1; status = PyWideStringList_Append(&config.module_search_paths, L"/path/to/stdlib"); @@ -1416,7 +2142,7 @@ initialization:: .. _init-isolated-conf: Isolated Configuration -====================== +---------------------- :c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to @@ -1436,7 +2162,7 @@ to avoid computing the default path configuration. .. _init-python-config: Python Configuration -==================== +-------------------- :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` functions create a configuration to build a customized Python which behaves as @@ -1454,7 +2180,7 @@ and :ref:`Python UTF-8 Mode <utf8-mode>` .. _init-path-config: Python Path Configuration -========================= +------------------------- :c:type:`PyConfig` contains multiple fields for the path configuration: @@ -1536,28 +2262,22 @@ If a ``._pth`` file is present: * Set :c:member:`~PyConfig.site_import` to ``0``. * Set :c:member:`~PyConfig.safe_path` to ``1``. +If :c:member:`~PyConfig.home` is not set and a ``pyvenv.cfg`` file is present in +the same directory as :c:member:`~PyConfig.executable`, or its parent, +:c:member:`~PyConfig.prefix` and :c:member:`~PyConfig.exec_prefix` are set that +location. When this happens, :c:member:`~PyConfig.base_prefix` and +:c:member:`~PyConfig.base_exec_prefix` still keep their value, pointing to the +base installation. See :ref:`sys-path-init-virtual-environments` for more +information. + The ``__PYVENV_LAUNCHER__`` environment variable is used to set :c:member:`PyConfig.base_executable`. +.. versionchanged:: 3.14 -Py_RunMain() -============ - -.. c:function:: int Py_RunMain(void) - - Execute the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. - - By default and when if :option:`-i` option is used, run the REPL. - - Finally, finalizes Python and returns an exit status that can be passed to - the ``exit()`` function. - -See :ref:`Python Configuration <init-python-config>` for an example of -customized Python always running in isolated mode using -:c:func:`Py_RunMain`. + :c:member:`~PyConfig.prefix`, and :c:member:`~PyConfig.exec_prefix`, are now + set to the ``pyvenv.cfg`` directory. This was previously done by :mod:`site`, + therefore affected by :option:`-S`. Py_GetArgcArgv() @@ -1569,89 +2289,13 @@ Py_GetArgcArgv() See also :c:member:`PyConfig.orig_argv` member. +Delaying main module execution +============================== -Multi-Phase Initialization Private Provisional API -================================================== - -This section is a private provisional API introducing multi-phase -initialization, the core feature of :pep:`432`: - -* "Core" initialization phase, "bare minimum Python": - - * Builtin types; - * Builtin exceptions; - * Builtin and frozen modules; - * The :mod:`sys` module is only partially initialized - (ex: :data:`sys.path` doesn't exist yet). - -* "Main" initialization phase, Python is fully initialized: - - * Install and configure :mod:`importlib`; - * Apply the :ref:`Path Configuration <init-path-config>`; - * Install signal handlers; - * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` - and :data:`sys.path`); - * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; - * Import the :mod:`site` module; - * etc. - -Private provisional API: - -* :c:member:`PyConfig._init_main`: if set to ``0``, - :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. +In some embedding use cases, it may be desirable to separate interpreter initialization +from the execution of the main module. -.. c:function:: PyStatus _Py_InitializeMain(void) - - Move to the "Main" initialization phase, finish the Python initialization. - -No module is imported during the "Core" phase and the ``importlib`` module is -not configured: the :ref:`Path Configuration <init-path-config>` is only -applied during the "Main" phase. It may allow to customize Python in Python to -override or tune the :ref:`Path Configuration <init-path-config>`, maybe -install a custom :data:`sys.meta_path` importer or an import hook, etc. - -It may become possible to calculate the :ref:`Path Configuration -<init-path-config>` in Python, after the Core phase and before the Main phase, -which is one of the :pep:`432` motivation. - -The "Core" phase is not properly defined: what should be and what should -not be available at this phase is not specified yet. The API is marked -as private and provisional: the API can be modified or even be removed -anytime until a proper public API is designed. - -Example running Python code between "Core" and "Main" initialization -phases:: - - void init_python(void) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config._init_main = 0; - - /* ... customize 'config' configuration ... */ - - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - /* Use sys.stderr because sys.stdout is only created - by _Py_InitializeMain() */ - int res = PyRun_SimpleString( - "import sys; " - "print('Run Python code before _Py_InitializeMain', " - "file=sys.stderr)"); - if (res < 0) { - exit(1); - } - - /* ... put more configuration code here ... */ - - status = _Py_InitializeMain(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - } +This separation can be achieved by setting ``PyConfig.run_command`` to the empty +string during initialization (to prevent the interpreter from dropping into the +interactive prompt), and then subsequently executing the desired main module +code using ``__main__.__dict__`` as the global namespace. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 5cf9914be8c8bf..41856922110eee 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -30,6 +30,16 @@ familiar with writing an extension before attempting to embed Python in a real application. +Language version compatibility +============================== + +Python's C API is compatible with C11 and C++11 versions of C and C++. + +This is a lower limit: the C API does not require features from later +C/C++ versions. +You do *not* need to enable your compiler's "c11 mode". + + Coding standards ================ @@ -105,6 +115,30 @@ defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). Others of a more general utility are defined here. This is not necessarily a complete listing. +.. c:macro:: PyMODINIT_FUNC + + Declare an extension module ``PyInit`` initialization function. The function + return type is :c:expr:`PyObject*`. The macro declares any special linkage + declarations required by the platform, and for C++ declares the function as + ``extern "C"``. + + The initialization function must be named :samp:`PyInit_{name}`, where + *name* is the name of the module, and should be the only non-\ ``static`` + item defined in the module file. Example:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + + .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. @@ -114,7 +148,7 @@ complete listing. .. c:macro:: Py_ALWAYS_INLINE Ask the compiler to always inline a static inline function. The compiler can - ignore it and decides to not inline the function. + ignore it and decide to not inline the function. It can be used to inline performance critical static inline functions when building Python in debug mode with function inlining disabled. For example, @@ -124,7 +158,7 @@ complete listing. worse performances (due to increased code size for example). The compiler is usually smarter than the developer for the cost/benefit analysis. - If Python is :ref:`built in debug mode <debug-build>` (if the ``Py_DEBUG`` + If Python is :ref:`built in debug mode <debug-build>` (if the :c:macro:`Py_DEBUG` macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. It must be specified before the function return type. Usage:: @@ -301,8 +335,8 @@ objects that reference each other here; for now, the solution is "don't do that.") .. index:: - single: Py_INCREF() - single: Py_DECREF() + single: Py_INCREF (C function) + single: Py_DECREF (C function) Reference counts are always manipulated explicitly. The normal way is to use the macro :c:func:`Py_INCREF` to take a new reference to an @@ -377,8 +411,8 @@ function, that function assumes that it now owns that reference, and you are not responsible for it any longer. .. index:: - single: PyList_SetItem() - single: PyTuple_SetItem() + single: PyList_SetItem (C function) + single: PyTuple_SetItem (C function) Few functions steal references; the two notable exceptions are :c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference @@ -467,8 +501,8 @@ using :c:func:`PySequence_GetItem` (which happens to take exactly the same arguments), you do own a reference to the returned object. .. index:: - single: PyList_GetItem() - single: PySequence_GetItem() + single: PyList_GetItem (C function) + single: PySequence_GetItem (C function) Here is an example of how you could write a function that computes the sum of the items in a list of integers; once using :c:func:`PyList_GetItem`, and once @@ -563,7 +597,7 @@ caller, then to the caller's caller, and so on, until they reach the top-level interpreter, where they are reported to the user accompanied by a stack traceback. -.. index:: single: PyErr_Occurred() +.. index:: single: PyErr_Occurred (C function) For C programmers, however, error checking always has to be explicit. All functions in the Python/C API can raise exceptions, unless an explicit claim is @@ -577,8 +611,8 @@ ambiguous return value, and require explicit testing for errors with :c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. .. index:: - single: PyErr_SetString() - single: PyErr_Clear() + single: PyErr_SetString (C function) + single: PyErr_Clear (C function) Exception state is maintained in per-thread storage (this is equivalent to using global storage in an unthreaded application). A thread can be in one of @@ -600,7 +634,7 @@ an exception is being passed on between C functions until it reaches the Python bytecode interpreter's main loop, which takes care of transferring it to ``sys.exc_info()`` and friends. -.. index:: single: exc_info() (in module sys) +.. index:: single: exc_info (in module sys) Note that starting with Python 1.5, the preferred, thread-safe way to access the exception state from Python code is to call the function :func:`sys.exc_info`, @@ -685,9 +719,9 @@ Here is the corresponding C code, in all its glory:: .. index:: single: incr_item() .. index:: - single: PyErr_ExceptionMatches() - single: PyErr_Clear() - single: Py_XDECREF() + single: PyErr_ExceptionMatches (C function) + single: PyErr_Clear (C function) + single: Py_XDECREF (C function) This example represents an endorsed use of the ``goto`` statement in C! It illustrates the use of :c:func:`PyErr_ExceptionMatches` and @@ -711,7 +745,7 @@ the finalization, of the Python interpreter. Most functionality of the interpreter can only be used after the interpreter has been initialized. .. index:: - single: Py_Initialize() + single: Py_Initialize (C function) pair: module; builtins pair: module; __main__ pair: module; sys @@ -745,22 +779,13 @@ found along :envvar:`PATH`.) The user can override this behavior by setting the environment variable :envvar:`PYTHONHOME`, or insert additional directories in front of the standard path by setting :envvar:`PYTHONPATH`. -.. index:: - single: Py_GetPath() - single: Py_GetPrefix() - single: Py_GetExecPrefix() - single: Py_GetProgramFullPath() - The embedding application can steer the search by setting :c:member:`PyConfig.program_name` *before* calling :c:func:`Py_InitializeFromConfig`. Note that :envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still -inserted in front of the standard path. An application that requires total -control has to provide its own implementation of :c:func:`Py_GetPath`, -:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and -:c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). +inserted in front of the standard path. -.. index:: single: Py_IsInitialized() +.. index:: single: Py_IsInitialized (C function) Sometimes, it is desirable to "uninitialize" Python. For instance, the application may want to start over (make another call to @@ -788,16 +813,21 @@ available that support tracing of reference counts, debugging the memory allocator, or low-level profiling of the main interpreter loop. Only the most frequently used builds will be described in the remainder of this section. -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces +.. c:macro:: Py_DEBUG + +Compiling the interpreter with the :c:macro:`!Py_DEBUG` macro defined produces what is generally meant by :ref:`a debug build of Python <debug-build>`. -:c:macro:`Py_DEBUG` is enabled in the Unix build by adding -:option:`--with-pydebug` to the :file:`./configure` command. -It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled -in the Unix build, compiler optimization is disabled. + +On Unix, :c:macro:`!Py_DEBUG` can be enabled by adding :option:`--with-pydebug` +to the :file:`./configure` command. This will also disable compiler optimization. + +On Windows, selecting a debug build (e.g., by passing the :option:`-d` option to +:file:`PCbuild/build.bat`) automatically enables :c:macro:`!Py_DEBUG`. +Additionally, the presence of the not-Python-specific :c:macro:`!_DEBUG` macro, +when defined by the compiler, will also implicitly enable :c:macro:`!Py_DEBUG`. In addition to the reference count debugging described below, extra checks are -performed, see :ref:`Python Debug Build <debug-build>`. +performed. See :ref:`Python Debug Build <debug-build>` for more details. Defining :c:macro:`Py_TRACE_REFS` enables reference tracing (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). @@ -809,3 +839,40 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. + +.. _c-api-tools: + +Recommended third party tools +============================= + +The following third party tools offer both simpler and more sophisticated +approaches to creating C, C++ and Rust extensions for Python: + +* `Cython <https://cython.org/>`_ +* `cffi <https://cffi.readthedocs.io>`_ +* `HPy <https://hpyproject.org/>`_ +* `nanobind <https://github.com/wjakob/nanobind>`_ (C++) +* `Numba <https://numba.pydata.org/>`_ +* `pybind11 <https://pybind11.readthedocs.io/>`_ (C++) +* `PyO3 <https://pyo3.rs/>`_ (Rust) +* `SWIG <https://www.swig.org>`_ + +Using tools such as these can help avoid writing code that is tightly bound to +a particular version of CPython, avoid reference counting errors, and focus +more on your own code than on using the CPython API. In general, new versions +of Python can be supported by updating the tool, and your code will often use +newer and more efficient APIs automatically. Some tools also support compiling +for other implementations of Python from a single set of sources. + +These projects are not supported by the same people who maintain Python, and +issues need to be raised with the projects directly. Remember to check that the +project is still maintained and supported, as the list above may become +outdated. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/guides/packaging-binary-extensions/>`_ + The Python Packaging User Guide not only covers several available + tools that simplify the creation of binary extensions, but also + discusses the various reasons why creating an extension module may be + desirable in the first place. diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 434d2021cea8e6..bf9df62c6f1706 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -10,7 +10,8 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) Return non-zero if the object *o* can be safely passed to - :c:func:`PyIter_Next`, and ``0`` otherwise. This function always succeeds. + :c:func:`PyIter_NextItem` and ``0`` otherwise. + This function always succeeds. .. c:function:: int PyAIter_Check(PyObject *o) @@ -19,41 +20,27 @@ There are two functions specifically for working with iterators. .. versionadded:: 3.10 +.. c:function:: int PyIter_NextItem(PyObject *iter, PyObject **item) + + Return ``1`` and set *item* to a :term:`strong reference` of the + next value of the iterator *iter* on success. + Return ``0`` and set *item* to ``NULL`` if there are no remaining values. + Return ``-1``, set *item* to ``NULL`` and set an exception on error. + + .. versionadded:: 3.14 + .. c:function:: PyObject* PyIter_Next(PyObject *o) + This is an older version of :c:func:`!PyIter_NextItem`, + which is retained for backwards compatibility. + Prefer :c:func:`PyIter_NextItem`. + Return the next value from the iterator *o*. The object must be an iterator according to :c:func:`PyIter_Check` (it is up to the caller to check this). If there are no remaining values, returns ``NULL`` with no exception set. If an error occurs while retrieving the item, returns ``NULL`` and passes along the exception. -To write a loop which iterates over an iterator, the C code should look -something like this:: - - PyObject *iterator = PyObject_GetIter(obj); - PyObject *item; - - if (iterator == NULL) { - /* propagate error */ - } - - while ((item = PyIter_Next(iterator))) { - /* do something with item */ - ... - /* release reference when done */ - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - /* propagate error */ - } - else { - /* continue doing useful work */ - } - - .. c:type:: PySendResult The enum value used to represent different results of :c:func:`PyIter_Send`. diff --git a/Doc/c-api/lifecycle.dot b/Doc/c-api/lifecycle.dot new file mode 100644 index 00000000000000..dca9f87e9e0aca --- /dev/null +++ b/Doc/c-api/lifecycle.dot @@ -0,0 +1,156 @@ +digraph "Life Events" { + graph [ + fontnames="svg" + fontsize=12.0 + id="life_events_graph" + layout="dot" + margin="0,0" + ranksep=0.25 + stylesheet="lifecycle.dot.css" + ] + node [ + fontname="Courier" + fontsize=12.0 + ] + edge [ + fontname="Times-Italic" + fontsize=12.0 + ] + + "start" [fontname="Times-Italic" shape=plain label=< start > style=invis] + { + rank="same" + "tp_new" [href="typeobj.html#c.PyTypeObject.tp_new" target="_top"] + "tp_alloc" [href="typeobj.html#c.PyTypeObject.tp_alloc" target="_top"] + } + "tp_init" [href="typeobj.html#c.PyTypeObject.tp_init" target="_top"] + "reachable" [fontname="Times-Italic" shape=box] + "tp_traverse" [ + href="typeobj.html#c.PyTypeObject.tp_traverse" + ordering="in" + target="_top" + ] + "finalized?" [ + fontname="Times-Italic" + label=<marked as<br/>finalized?> + ordering="in" + shape=diamond + tooltip="marked as finalized?" + ] + "tp_finalize" [ + href="typeobj.html#c.PyTypeObject.tp_finalize" + ordering="in" + target="_top" + ] + "tp_clear" [href="typeobj.html#c.PyTypeObject.tp_clear" target="_top"] + "uncollectable" [ + fontname="Times-Italic" + label=<uncollectable<br/>(leaked)> + shape=box + tooltip="uncollectable (leaked)" + ] + "tp_dealloc" [ + href="typeobj.html#c.PyTypeObject.tp_dealloc" + ordering="in" + target="_top" + ] + "tp_free" [href="typeobj.html#c.PyTypeObject.tp_free" target="_top"] + + "start" -> "tp_new" [ + label=< type call > + ] + "tp_new" -> "tp_alloc" [ + label=< direct call > arrowhead=empty + labeltooltip="tp_new to tp_alloc: direct call" + tooltip="tp_new to tp_alloc: direct call" + ] + "tp_new" -> "tp_init" [tooltip="tp_new to tp_init"] + "tp_init" -> "reachable" [tooltip="tp_init to reachable"] + "reachable" -> "tp_traverse" [ + dir="back" + label=< not in a <br/> cyclic <br/> isolate > + labeltooltip="tp_traverse to reachable: not in a cyclic isolate" + tooltip="tp_traverse to reachable: not in a cyclic isolate" + ] + "reachable" -> "tp_traverse" [ + label=< periodic <br/> cyclic isolate <br/> detection > + labeltooltip="reachable to tp_traverse: periodic cyclic isolate detection" + tooltip="reachable to tp_traverse: periodic cyclic isolate detection" + ] + "reachable" -> "tp_init" [tooltip="reachable to tp_init"] + "reachable" -> "tp_finalize" [ + dir="back" + label=< resurrected <br/> (maybe remove <br/> finalized mark) > + labeltooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + tooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + ] + "tp_traverse" -> "finalized?" [ + label=< cyclic <br/> isolate > + labeltooltip="tp_traverse to finalized?: cyclic isolate" + tooltip="tp_traverse to finalized?: cyclic isolate" + ] + "reachable" -> "finalized?" [ + label=< no refs > + labeltooltip="reachable to finalized?: no refs" + tooltip="reachable to finalized?: no refs" + ] + "finalized?" -> "tp_finalize" [ + label=< no (mark <br/> as finalized) > + labeltooltip="finalized? to tp_finalize: no (mark as finalized)" + tooltip="finalized? to tp_finalize: no (mark as finalized)" + ] + "finalized?" -> "tp_clear" [ + label=< yes > + labeltooltip="finalized? to tp_clear: yes" + tooltip="finalized? to tp_clear: yes" + ] + "tp_finalize" -> "tp_clear" [ + label=< no refs or <br/> cyclic isolate > + labeltooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + tooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + ] + "tp_finalize" -> "tp_dealloc" [ + arrowtail=empty + dir="back" + href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" + style=dashed + label=< recommended<br/> call (see<br/> explanation)> + labeltooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + target="_top" + tooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + ] + "tp_finalize" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_finalize to tp_dealloc: no refs" + tooltip="tp_finalize to tp_dealloc: no refs" + ] + "tp_clear" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_clear to tp_dealloc: no refs" + tooltip="tp_clear to tp_dealloc: no refs" + ] + "tp_clear" -> "uncollectable" [ + label=< cyclic <br/> isolate > + labeltooltip="tp_clear to uncollectable: cyclic isolate" + tooltip="tp_clear to uncollectable: cyclic isolate" + ] + "uncollectable" -> "tp_dealloc" [ + style=invis + tooltip="uncollectable to tp_dealloc" + ] + "reachable" -> "uncollectable" [ + label=< cyclic <br/> isolate <br/> (no GC <br/> support) > + labeltooltip="reachable to uncollectable: cyclic isolate (no GC support)" + tooltip="reachable to uncollectable: cyclic isolate (no GC support)" + ] + "reachable" -> "tp_dealloc" [ + label=< no refs> + labeltooltip="reachable to tp_dealloc: no refs" + ] + "tp_dealloc" -> "tp_free" [ + arrowhead=empty + label=< direct call > + labeltooltip="tp_dealloc to tp_free: direct call" + tooltip="tp_dealloc to tp_free: direct call" + ] +} diff --git a/Doc/c-api/lifecycle.dot.css b/Doc/c-api/lifecycle.dot.css new file mode 100644 index 00000000000000..3abf95b74da6ba --- /dev/null +++ b/Doc/c-api/lifecycle.dot.css @@ -0,0 +1,21 @@ +#life_events_graph { + --svg-fgcolor: currentcolor; + --svg-bgcolor: transparent; +} +#life_events_graph a { + color: inherit; +} +#life_events_graph [stroke="black"] { + stroke: var(--svg-fgcolor); +} +#life_events_graph text, +#life_events_graph [fill="black"] { + fill: var(--svg-fgcolor); +} +#life_events_graph [fill="white"] { + fill: var(--svg-bgcolor); +} +#life_events_graph [fill="none"] { + /* On links, setting fill will make the entire shape clickable */ + fill: var(--svg-bgcolor); +} diff --git a/Doc/c-api/lifecycle.dot.pdf b/Doc/c-api/lifecycle.dot.pdf new file mode 100644 index 00000000000000..ed5b5039c83e2c Binary files /dev/null and b/Doc/c-api/lifecycle.dot.pdf differ diff --git a/Doc/c-api/lifecycle.dot.svg b/Doc/c-api/lifecycle.dot.svg new file mode 100644 index 00000000000000..7ace27dfcba113 --- /dev/null +++ b/Doc/c-api/lifecycle.dot.svg @@ -0,0 +1,374 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<?xml-stylesheet href="lifecycle.dot.css" type="text/css"?> +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" + "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"> +<!-- Generated by graphviz version 12.2.0 (0) + --> +<!-- Title: Life Events Pages: 1 --> +<svg width="465pt" height="845pt" + viewBox="0.00 0.00 465.30 845.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink"> +<g id="life_events_graph" class="graph" transform="scale(1 1) rotate(0) translate(4 841)"> +<title>Life Events</title> +<polygon fill="white" stroke="none" points="-4,4 -4,-841 461.3,-841 461.3,4 -4,4"/> +<!-- start --> +<!-- tp_new --> +<g id="life_events_graph_node2" class="node"> +<title>tp_new</title> +<g id="a_life_events_graph_node2"><a xlink:href="typeobj.html#c.PyTypeObject.tp_new" xlink:title="tp_new" target="_top"> +<ellipse fill="none" stroke="black" cx="192.8" cy="-772.5" rx="38.8" ry="18"/> +<text text-anchor="middle" x="192.8" y="-768.23" font-family="monospace,monospace" font-size="12.00">tp_new</text> +</a> +</g> +</g> +<!-- start->tp_new --> +<g id="life_events_graph_edge1" class="edge"> +<title>start->tp_new</title> +<g id="a_life_events_graph_edge1"><a xlink:title="start to tp_new: type call"> +<path fill="none" stroke="black" d="M192.8,-822.95C192.8,-817.85 192.8,-810.09 192.8,-802.22"/> +<polygon fill="black" stroke="black" points="196.3,-802.42 192.8,-792.42 189.3,-802.42 196.3,-802.42"/> +</a> +</g> +<g id="a_life_events_graph_edge1-label"><a xlink:title="start to tp_new: type call"> +<text text-anchor="start" x="192.8" y="-802.35" font-family="serif,serif" font-style="italic" font-size="12.00">    type call  </text> +</a> +</g> +</g> +<!-- tp_alloc --> +<g id="life_events_graph_node3" class="node"> +<title>tp_alloc</title> +<g id="a_life_events_graph_node3"><a xlink:href="typeobj.html#c.PyTypeObject.tp_alloc" xlink:title="tp_alloc" target="_top"> +<ellipse fill="none" stroke="black" cx="373.8" cy="-772.5" rx="48.34" ry="18"/> +<text text-anchor="middle" x="373.8" y="-768.23" font-family="monospace,monospace" font-size="12.00">tp_alloc</text> +</a> +</g> +</g> +<!-- tp_new->tp_alloc --> +<g id="life_events_graph_edge2" class="edge"> +<title>tp_new->tp_alloc</title> +<g id="a_life_events_graph_edge2"><a xlink:title="tp_new to tp_alloc: direct call"> +<path fill="none" stroke="black" d="M232.07,-772.5C256,-772.5 287.05,-772.5 313.98,-772.5"/> +<polygon fill="none" stroke="black" points="313.73,-776 323.73,-772.5 313.73,-769 313.73,-776"/> +</a> +</g> +<g id="a_life_events_graph_edge2-label"><a xlink:title="tp_new to tp_alloc: direct call"> +<text text-anchor="start" x="240.65" y="-778.35" font-family="serif,serif" font-style="italic" font-size="12.00">  direct call  </text> +</a> +</g> +</g> +<!-- tp_init --> +<g id="life_events_graph_node4" class="node"> +<title>tp_init</title> +<g id="a_life_events_graph_node4"><a xlink:href="typeobj.html#c.PyTypeObject.tp_init" xlink:title="tp_init" target="_top"> +<ellipse fill="none" stroke="black" cx="192.8" cy="-717.5" rx="43.57" ry="18"/> +<text text-anchor="middle" x="192.8" y="-713.23" font-family="monospace,monospace" font-size="12.00">tp_init</text> +</a> +</g> +</g> +<!-- tp_new->tp_init --> +<g id="life_events_graph_edge3" class="edge"> +<title>tp_new->tp_init</title> +<g id="a_life_events_graph_edge3"><a xlink:title="tp_new to tp_init"> +<path fill="none" stroke="black" d="M192.8,-754.15C192.8,-751.83 192.8,-749.42 192.8,-746.98"/> +<polygon fill="black" stroke="black" points="196.3,-747.23 192.8,-737.23 189.3,-747.23 196.3,-747.23"/> +</a> +</g> +</g> +<!-- reachable --> +<g id="life_events_graph_node5" class="node"> +<title>reachable</title> +<polygon fill="none" stroke="black" points="230.8,-680.5 154.8,-680.5 154.8,-644.5 230.8,-644.5 230.8,-680.5"/> +<text text-anchor="middle" x="192.8" y="-658.23" font-family="serif,serif" font-style="italic" font-size="12.00">reachable</text> +</g> +<!-- tp_init->reachable --> +<g id="life_events_graph_edge4" class="edge"> +<title>tp_init->reachable</title> +<g id="a_life_events_graph_edge4"><a xlink:title="tp_init to reachable"> +<path fill="none" stroke="black" d="M186.44,-699.44C186.24,-697.12 186.11,-694.69 186.07,-692.24"/> +<polygon fill="black" stroke="black" points="189.56,-692.51 186.37,-682.41 182.56,-692.29 189.56,-692.51"/> +</a> +</g> +</g> +<!-- reachable->tp_init --> +<g id="life_events_graph_edge7" class="edge"> +<title>reachable->tp_init</title> +<g id="a_life_events_graph_edge7"><a xlink:title="reachable to tp_init"> +<path fill="none" stroke="black" d="M199.18,-680.89C199.37,-683.22 199.49,-685.65 199.53,-688.11"/> +<polygon fill="black" stroke="black" points="196.04,-687.81 199.2,-697.93 203.04,-688.05 196.04,-687.81"/> +</a> +</g> +</g> +<!-- tp_traverse --> +<g id="life_events_graph_node6" class="node"> +<title>tp_traverse</title> +<g id="a_life_events_graph_node6"><a xlink:href="typeobj.html#c.PyTypeObject.tp_traverse" xlink:title="tp_traverse" target="_top"> +<ellipse fill="none" stroke="black" cx="136.8" cy="-565.75" rx="62.65" ry="18"/> +<text text-anchor="middle" x="136.8" y="-561.48" font-family="monospace,monospace" font-size="12.00">tp_traverse</text> +</a> +</g> +</g> +<!-- reachable->tp_traverse --> +<g id="life_events_graph_edge5" class="edge"> +<title>reachable->tp_traverse</title> +<g id="a_life_events_graph_edge5"><a xlink:title="tp_traverse to reachable: not in a cyclic isolate"> +<path fill="none" stroke="black" d="M143.43,-658.77C108.3,-655.68 65.38,-649.16 54.05,-635.5 41.91,-620.88 42.8,-608.07 54.05,-592.75 60.55,-583.89 70.07,-577.97 80.37,-574.03"/> +<polygon fill="black" stroke="black" points="142.76,-662.23 153.01,-659.54 143.32,-655.25 142.76,-662.23"/> +</a> +</g> +<g id="a_life_events_graph_edge5-label"><a xlink:title="tp_traverse to reachable: not in a cyclic isolate"> +<text text-anchor="start" x="54.05" y="-624.1" font-family="serif,serif" font-style="italic" font-size="12.00">  not in a  </text> +<text text-anchor="start" x="59.67" y="-609.85" font-family="serif,serif" font-style="italic" font-size="12.00">  cyclic  </text> +<text text-anchor="start" x="57.05" y="-595.6" font-family="serif,serif" font-style="italic" font-size="12.00">  isolate  </text> +</a> +</g> +</g> +<!-- reachable->tp_traverse --> +<g id="life_events_graph_edge6" class="edge"> +<title>reachable->tp_traverse</title> +<g id="a_life_events_graph_edge6"><a xlink:title="reachable to tp_traverse: periodic cyclic isolate detection"> +<path fill="none" stroke="black" d="M154.41,-650.07C147.94,-646.44 142.04,-641.69 138.05,-635.5 130.52,-623.82 129.57,-608.56 130.79,-595.38"/> +<polygon fill="black" stroke="black" points="134.25,-595.91 132.17,-585.52 127.31,-594.94 134.25,-595.91"/> +</a> +</g> +<g id="a_life_events_graph_edge6-label"><a xlink:title="reachable to tp_traverse: periodic cyclic isolate detection"> +<text text-anchor="start" x="154.17" y="-624.1" font-family="serif,serif" font-style="italic" font-size="12.00">  periodic  </text> +<text text-anchor="start" x="138.05" y="-609.85" font-family="serif,serif" font-style="italic" font-size="12.00">  cyclic isolate   </text> +<text text-anchor="start" x="151.17" y="-595.6" font-family="serif,serif" font-style="italic" font-size="12.00">  detection  </text> +</a> +</g> +</g> +<!-- finalized? --> +<g id="life_events_graph_node7" class="node"> +<title>finalized?</title> +<g id="a_life_events_graph_node7"><a xlink:title="marked as finalized?"> +<polygon fill="none" stroke="black" points="191.8,-487 112.05,-450.5 191.8,-414 271.55,-450.5 191.8,-487"/> +<text text-anchor="start" x="159.92" y="-453.35" font-family="serif,serif" font-style="italic" font-size="12.00">marked as</text> +<text text-anchor="start" x="162.92" y="-439.1" font-family="serif,serif" font-style="italic" font-size="12.00">finalized?</text> +</a> +</g> +</g> +<!-- reachable->finalized? --> +<g id="life_events_graph_edge10" class="edge"> +<title>reachable->finalized?</title> +<g id="a_life_events_graph_edge10"><a xlink:title="reachable to finalized?: no refs"> +<path fill="none" stroke="black" d="M227.72,-644.32C230.51,-641.73 232.96,-638.8 234.8,-635.5 244.04,-618.9 235.48,-611.74 234.8,-592.75 233.24,-549.67 243.64,-536.1 227.8,-496 226.37,-492.38 224.53,-488.82 222.45,-485.4"/> +<polygon fill="black" stroke="black" points="225.47,-483.62 216.91,-477.39 219.72,-487.61 225.47,-483.62"/> +</a> +</g> +<g id="a_life_events_graph_edge10-label"><a xlink:title="reachable to finalized?: no refs"> +<text text-anchor="start" x="236.45" y="-561.48" font-family="serif,serif" font-style="italic" font-size="12.00">  no refs  </text> +</a> +</g> +</g> +<!-- tp_finalize --> +<g id="life_events_graph_node8" class="node"> +<title>tp_finalize</title> +<g id="a_life_events_graph_node8"><a xlink:href="typeobj.html#c.PyTypeObject.tp_finalize" xlink:title="tp_finalize" target="_top"> +<ellipse fill="none" stroke="black" cx="122.8" cy="-321" rx="62.65" ry="18"/> +<text text-anchor="middle" x="122.8" y="-316.73" font-family="monospace,monospace" font-size="12.00">tp_finalize</text> +</a> +</g> +</g> +<!-- reachable->tp_finalize --> +<g id="life_events_graph_edge8" class="edge"> +<title>reachable->tp_finalize</title> +<g id="a_life_events_graph_edge8"><a xlink:title="tp_finalize to reachable: resurrected (maybe remove finalized mark)"> +<path fill="none" stroke="black" d="M142.86,-659.6C103.8,-656.96 53.97,-650.63 40.8,-635.5 -37.32,-545.75 69.61,-390.31 109.14,-338.99"/> +<polygon fill="black" stroke="black" points="142.62,-663.09 152.82,-660.2 143.05,-656.11 142.62,-663.09"/> +</a> +</g> +<g id="a_life_events_graph_edge8-label"><a xlink:title="tp_finalize to reachable: resurrected (maybe remove finalized mark)"> +<text text-anchor="start" x="33.68" y="-527.35" font-family="serif,serif" font-style="italic" font-size="12.00">  resurrected  </text> +<text text-anchor="start" x="22.43" y="-513.1" font-family="serif,serif" font-style="italic" font-size="12.00">  (maybe remove  </text> +<text text-anchor="start" x="23.18" y="-498.85" font-family="serif,serif" font-style="italic" font-size="12.00">  finalized mark)  </text> +</a> +</g> +</g> +<!-- uncollectable --> +<g id="life_events_graph_node10" class="node"> +<title>uncollectable</title> +<g id="a_life_events_graph_node10"><a xlink:title="uncollectable (leaked)"> +<polygon fill="none" stroke="black" points="371.92,-159.75 275.67,-159.75 275.67,-123.25 371.92,-123.25 371.92,-159.75"/> +<text text-anchor="start" x="283.67" y="-144.35" font-family="serif,serif" font-style="italic" font-size="12.00">uncollectable</text> +<text text-anchor="start" x="299.42" y="-130.1" font-family="serif,serif" font-style="italic" font-size="12.00">(leaked)</text> +</a> +</g> +</g> +<!-- reachable->uncollectable --> +<g id="life_events_graph_edge19" class="edge"> +<title>reachable->uncollectable</title> +<g id="a_life_events_graph_edge19"><a xlink:title="reachable to uncollectable: cyclic isolate (no GC support)"> +<path fill="none" stroke="black" d="M231.2,-652.03C270.79,-639.69 326.8,-613.9 326.8,-566.75 326.8,-566.75 326.8,-566.75 326.8,-237.5 326.8,-215.3 325.97,-190.2 325.18,-171.37"/> +<polygon fill="black" stroke="black" points="328.68,-171.35 324.75,-161.52 321.69,-171.66 328.68,-171.35"/> +</a> +</g> +<g id="a_life_events_graph_edge19-label"><a xlink:title="reachable to uncollectable: cyclic isolate (no GC support)"> +<text text-anchor="start" x="335.05" y="-393.6" font-family="serif,serif" font-style="italic" font-size="12.00">  cyclic  </text> +<text text-anchor="start" x="332.42" y="-379.35" font-family="serif,serif" font-style="italic" font-size="12.00">  isolate  </text> +<text text-anchor="start" x="331.3" y="-365.1" font-family="serif,serif" font-style="italic" font-size="12.00">  (no GC  </text> +<text text-anchor="start" x="326.8" y="-350.85" font-family="serif,serif" font-style="italic" font-size="12.00">  support)  </text> +</a> +</g> +</g> +<!-- tp_dealloc --> +<g id="life_events_graph_node11" class="node"> +<title>tp_dealloc</title> +<g id="a_life_events_graph_node11"><a xlink:href="typeobj.html#c.PyTypeObject.tp_dealloc" xlink:title="tp_dealloc" target="_top"> +<ellipse fill="none" stroke="black" cx="200.8" cy="-86.25" rx="57.88" ry="18"/> +<text text-anchor="middle" x="200.8" y="-81.97" font-family="monospace,monospace" font-size="12.00">tp_dealloc</text> +</a> +</g> +</g> +<!-- reachable->tp_dealloc --> +<g id="life_events_graph_edge20" class="edge"> +<title>reachable->tp_dealloc</title> +<path fill="none" stroke="black" d="M231.23,-661.18C293.08,-658.43 407.8,-643.03 407.8,-566.75 407.8,-566.75 407.8,-566.75 407.8,-140.5 407.8,-111.22 329.12,-97.8 268.77,-91.82"/> +<polygon fill="black" stroke="black" points="269.15,-88.34 258.87,-90.89 268.5,-95.31 269.15,-88.34"/> +<g id="a_life_events_graph_edge20-label"><a xlink:title="reachable to tp_dealloc: no refs"> +<text text-anchor="start" x="407.8" y="-316.73" font-family="serif,serif" font-style="italic" font-size="12.00">  no refs</text> +</a> +</g> +</g> +<!-- tp_traverse->finalized? --> +<g id="life_events_graph_edge9" class="edge"> +<title>tp_traverse->finalized?</title> +<g id="a_life_events_graph_edge9"><a xlink:title="tp_traverse to finalized?: cyclic isolate"> +<path fill="none" stroke="black" d="M145.15,-547.55C152.4,-532.62 163.18,-510.43 172.55,-491.13"/> +<polygon fill="black" stroke="black" points="175.7,-492.66 176.92,-482.14 169.4,-489.61 175.7,-492.66"/> +</a> +</g> +<g id="a_life_events_graph_edge9-label"><a xlink:title="tp_traverse to finalized?: cyclic isolate"> +<text text-anchor="start" x="171.85" y="-520.23" font-family="serif,serif" font-style="italic" font-size="12.00">  cyclic  </text> +<text text-anchor="start" x="169.22" y="-505.98" font-family="serif,serif" font-style="italic" font-size="12.00">  isolate  </text> +</a> +</g> +</g> +<!-- finalized?->tp_finalize --> +<g id="life_events_graph_edge11" class="edge"> +<title>finalized?->tp_finalize</title> +<g id="a_life_events_graph_edge11"><a xlink:title="finalized? to tp_finalize: no (mark as finalized)"> +<path fill="none" stroke="black" d="M172.89,-422.6C169.14,-416.89 165.34,-410.82 162.05,-405 151.89,-387.08 141.99,-366.11 134.68,-349.73"/> +<polygon fill="black" stroke="black" points="137.89,-348.35 130.66,-340.61 131.48,-351.17 137.89,-348.35"/> +</a> +</g> +<g id="a_life_events_graph_edge11-label"><a xlink:title="finalized? to tp_finalize: no (mark as finalized)"> +<text text-anchor="start" x="170.67" y="-379.35" font-family="serif,serif" font-style="italic" font-size="12.00">  no (mark  </text> +<text text-anchor="start" x="162.05" y="-365.1" font-family="serif,serif" font-style="italic" font-size="12.00">  as finalized)  </text> +</a> +</g> +</g> +<!-- tp_clear --> +<g id="life_events_graph_node9" class="node"> +<title>tp_clear</title> +<g id="a_life_events_graph_node9"><a xlink:href="typeobj.html#c.PyTypeObject.tp_clear" xlink:title="tp_clear" target="_top"> +<ellipse fill="none" stroke="black" cx="222.8" cy="-238.5" rx="48.34" ry="18"/> +<text text-anchor="middle" x="222.8" y="-234.22" font-family="monospace,monospace" font-size="12.00">tp_clear</text> +</a> +</g> +</g> +<!-- finalized?->tp_clear --> +<g id="life_events_graph_edge12" class="edge"> +<title>finalized?->tp_clear</title> +<g id="a_life_events_graph_edge12"><a xlink:title="finalized? to tp_clear: yes"> +<path fill="none" stroke="black" d="M227.56,-430.1C236.46,-423.41 244.86,-415.02 249.8,-405 277.22,-349.39 274.06,-322.55 249.8,-265.5 249.7,-265.27 249.6,-265.04 249.49,-264.81"/> +<polygon fill="black" stroke="black" points="252.41,-262.88 243.93,-256.52 246.6,-266.78 252.41,-262.88"/> +</a> +</g> +<g id="a_life_events_graph_edge12-label"><a xlink:title="finalized? to tp_clear: yes"> +<text text-anchor="start" x="269.2" y="-316.73" font-family="serif,serif" font-style="italic" font-size="12.00">  yes  </text> +</a> +</g> +</g> +<!-- tp_finalize->tp_clear --> +<g id="life_events_graph_edge13" class="edge"> +<title>tp_finalize->tp_clear</title> +<g id="a_life_events_graph_edge13"><a xlink:title="tp_finalize to tp_clear: no refs or cyclic isolate"> +<path fill="none" stroke="black" d="M130.02,-302.72C135.75,-290.85 144.8,-275.49 156.8,-265.5 161.95,-261.21 167.9,-257.57 174.07,-254.49"/> +<polygon fill="black" stroke="black" points="175.46,-257.71 183.18,-250.45 172.62,-251.31 175.46,-257.71"/> +</a> +</g> +<g id="a_life_events_graph_edge13-label"><a xlink:title="tp_finalize to tp_clear: no refs or cyclic isolate"> +<text text-anchor="start" x="164.3" y="-282.6" font-family="serif,serif" font-style="italic" font-size="12.00">  no refs or   </text> +<text text-anchor="start" x="156.8" y="-268.35" font-family="serif,serif" font-style="italic" font-size="12.00">  cyclic isolate  </text> +</a> +</g> +</g> +<!-- tp_finalize->tp_dealloc --> +<g id="life_events_graph_edge14" class="edge"> +<title>tp_finalize->tp_dealloc</title> +<g id="a_life_events_graph_edge14"><a xlink:href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" xlink:title="tp_dealloc to tp_finalize: recommended call (see explanation)" target="_top"> +<path fill="none" stroke="black" stroke-dasharray="5,2" d="M85.85,-298.52C42.09,-270.18 -21.4,-218.11 7.8,-168.75 36.22,-120.7 99.95,-100.97 146.42,-92.87"/> +<polygon fill="none" stroke="black" points="83.78,-301.35 94.11,-303.72 87.52,-295.43 83.78,-301.35"/> +</a> +</g> +<g id="a_life_events_graph_edge14-label"><a xlink:href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" xlink:title="tp_dealloc to tp_finalize: recommended call (see explanation)" target="_top"> +<text text-anchor="start" x="7.8" y="-200.1" font-family="serif,serif" font-style="italic" font-size="12.00">  recommended</text> +<text text-anchor="start" x="25.8" y="-185.85" font-family="serif,serif" font-style="italic" font-size="12.00">  call (see</text> +<text text-anchor="start" x="13.05" y="-171.6" font-family="serif,serif" font-style="italic" font-size="12.00">  explanation)</text> +</a> +</g> +</g> +<!-- tp_finalize->tp_dealloc --> +<g id="life_events_graph_edge15" class="edge"> +<title>tp_finalize->tp_dealloc</title> +<g id="a_life_events_graph_edge15"><a xlink:title="tp_finalize to tp_dealloc: no refs"> +<path fill="none" stroke="black" d="M123.03,-302.58C123.95,-273.77 128.08,-214.78 146.05,-168.75 153.95,-148.5 167.56,-128.2 179.24,-112.92"/> +<polygon fill="black" stroke="black" points="181.81,-115.32 185.25,-105.3 176.31,-110.98 181.81,-115.32"/> +</a> +</g> +<g id="a_life_events_graph_edge15-label"><a xlink:title="tp_finalize to tp_dealloc: no refs"> +<text text-anchor="start" x="146.05" y="-185.85" font-family="serif,serif" font-style="italic" font-size="12.00">   no refs  </text> +</a> +</g> +</g> +<!-- tp_clear->uncollectable --> +<g id="life_events_graph_edge17" class="edge"> +<title>tp_clear->uncollectable</title> +<g id="a_life_events_graph_edge17"><a xlink:title="tp_clear to uncollectable: cyclic isolate"> +<path fill="none" stroke="black" d="M227.75,-220.38C232.99,-205 242.67,-182.74 258.05,-168.75 260.43,-166.58 263.02,-164.58 265.74,-162.73"/> +<polygon fill="black" stroke="black" points="267.27,-165.89 274.12,-157.81 263.73,-159.86 267.27,-165.89"/> +</a> +</g> +<g id="a_life_events_graph_edge17-label"><a xlink:title="tp_clear to uncollectable: cyclic isolate"> +<text text-anchor="start" x="260.67" y="-192.97" font-family="serif,serif" font-style="italic" font-size="12.00">  cyclic  </text> +<text text-anchor="start" x="258.05" y="-178.72" font-family="serif,serif" font-style="italic" font-size="12.00">  isolate  </text> +</a> +</g> +</g> +<!-- tp_clear->tp_dealloc --> +<g id="life_events_graph_edge16" class="edge"> +<title>tp_clear->tp_dealloc</title> +<g id="a_life_events_graph_edge16"><a xlink:title="tp_clear to tp_dealloc: no refs"> +<path fill="none" stroke="black" d="M219.7,-220.24C216.92,-204.51 212.83,-180.61 209.8,-159.75 207.7,-145.34 205.67,-129.26 204.07,-115.92"/> +<polygon fill="black" stroke="black" points="207.56,-115.59 202.91,-106.07 200.61,-116.41 207.56,-115.59"/> +</a> +</g> +<g id="a_life_events_graph_edge16-label"><a xlink:title="tp_clear to tp_dealloc: no refs"> +<text text-anchor="start" x="209.8" y="-137.22" font-family="serif,serif" font-style="italic" font-size="12.00">  no refs  </text> +</a> +</g> +</g> +<!-- uncollectable->tp_dealloc --> +<!-- tp_free --> +<g id="life_events_graph_node12" class="node"> +<title>tp_free</title> +<g id="a_life_events_graph_node12"><a xlink:href="typeobj.html#c.PyTypeObject.tp_free" xlink:title="tp_free" target="_top"> +<ellipse fill="none" stroke="black" cx="200.8" cy="-18" rx="43.57" ry="18"/> +<text text-anchor="middle" x="200.8" y="-13.72" font-family="monospace,monospace" font-size="12.00">tp_free</text> +</a> +</g> +</g> +<!-- tp_dealloc->tp_free --> +<g id="life_events_graph_edge21" class="edge"> +<title>tp_dealloc->tp_free</title> +<g id="a_life_events_graph_edge21"><a xlink:title="tp_dealloc to tp_free: direct call"> +<path fill="none" stroke="black" d="M200.8,-67.84C200.8,-61.63 200.8,-54.46 200.8,-47.56"/> +<polygon fill="none" stroke="black" points="204.3,-47.57 200.8,-37.57 197.3,-47.57 204.3,-47.57"/> +</a> +</g> +<g id="a_life_events_graph_edge21-label"><a xlink:title="tp_dealloc to tp_free: direct call"> +<text text-anchor="start" x="200.8" y="-47.85" font-family="serif,serif" font-style="italic" font-size="12.00">    direct call  </text> +</a> +</g> +</g> +</g> +</svg> diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst new file mode 100644 index 00000000000000..5a170862a26f44 --- /dev/null +++ b/Doc/c-api/lifecycle.rst @@ -0,0 +1,271 @@ +.. highlight:: c + +.. _life-cycle: + +Object Life Cycle +================= + +This section explains how a type's slots relate to each other throughout the +life of an object. It is not intended to be a complete canonical reference for +the slots; instead, refer to the slot-specific documentation in +:ref:`type-structs` for details about a particular slot. + + +Life Events +----------- + +The figure below illustrates the order of events that can occur throughout an +object's life. An arrow from *A* to *B* indicates that event *B* can occur +after event *A* has occurred, with the arrow's label indicating the condition +that must be true for *B* to occur after *A*. + +.. only:: html and not epub + + .. raw:: html + + <style type="text/css"> + + .. raw:: html + :file: lifecycle.dot.css + + .. raw:: html + + </style> + + .. raw:: html + :file: lifecycle.dot.svg + + .. raw:: html + + <script> + (() => { + const g = document.getElementById('life_events_graph'); + const title = g.querySelector(':scope > title'); + title.id = 'life-events-graph-title'; + const svg = g.closest('svg'); + svg.role = 'img'; + svg.setAttribute('aria-describedby', + 'life-events-graph-description'); + svg.setAttribute('aria-labelledby', 'life-events-graph-title'); + })(); + </script> + +.. only:: epub or not (html or latex) + + .. image:: lifecycle.dot.svg + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. only:: latex + + .. image:: lifecycle.dot.pdf + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. container:: + :name: life-events-graph-description + + Explanation: + + * When a new object is constructed by calling its type: + + #. :c:member:`~PyTypeObject.tp_new` is called to create a new object. + #. :c:member:`~PyTypeObject.tp_alloc` is directly called by + :c:member:`~PyTypeObject.tp_new` to allocate the memory for the new + object. + #. :c:member:`~PyTypeObject.tp_init` initializes the newly created object. + :c:member:`!tp_init` can be called again to re-initialize an object, if + desired. The :c:member:`!tp_init` call can also be skipped entirely, + for example by Python code calling :py:meth:`~object.__new__`. + + * After :c:member:`!tp_init` completes, the object is ready to use. + * Some time after the last reference to an object is removed: + + #. If an object is not marked as *finalized*, it might be finalized by + marking it as *finalized* and calling its + :c:member:`~PyTypeObject.tp_finalize` function. Python does + *not* finalize an object when the last reference to it is deleted; use + :c:func:`PyObject_CallFinalizerFromDealloc` to ensure that + :c:member:`~PyTypeObject.tp_finalize` is always called. + #. If the object is marked as finalized, + :c:member:`~PyTypeObject.tp_clear` might be called by the garbage collector + to clear references held by the object. It is *not* called when the + object's reference count reaches zero. + #. :c:member:`~PyTypeObject.tp_dealloc` is called to destroy the object. + To avoid code duplication, :c:member:`~PyTypeObject.tp_dealloc` typically + calls into :c:member:`~PyTypeObject.tp_clear` to free up the object's + references. + #. When :c:member:`~PyTypeObject.tp_dealloc` finishes object destruction, + it directly calls :c:member:`~PyTypeObject.tp_free` (usually set to + :c:func:`PyObject_Free` or :c:func:`PyObject_GC_Del` automatically as + appropriate for the type) to deallocate the memory. + + * The :c:member:`~PyTypeObject.tp_finalize` function is permitted to add a + reference to the object if desired. If it does, the object is + *resurrected*, preventing its pending destruction. (Only + :c:member:`!tp_finalize` is allowed to resurrect an object; + :c:member:`~PyTypeObject.tp_clear` and + :c:member:`~PyTypeObject.tp_dealloc` cannot without calling into + :c:member:`!tp_finalize`.) Resurrecting an object may + or may not cause the object's *finalized* mark to be removed. Currently, + Python does not remove the *finalized* mark from a resurrected object if + it supports garbage collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set) but does remove the mark if the object does not support + garbage collection; either or both of these behaviors may change in the + future. + * :c:member:`~PyTypeObject.tp_dealloc` can optionally call + :c:member:`~PyTypeObject.tp_finalize` via + :c:func:`PyObject_CallFinalizerFromDealloc` if it wishes to reuse that + code to help with object destruction. This is recommended because it + guarantees that :c:member:`!tp_finalize` is always called before + destruction. See the :c:member:`~PyTypeObject.tp_dealloc` documentation + for example code. + * If the object is a member of a :term:`cyclic isolate` and either + :c:member:`~PyTypeObject.tp_clear` fails to break the reference cycle or + the cyclic isolate is not detected (perhaps :func:`gc.disable` was called, + or the :c:macro:`Py_TPFLAGS_HAVE_GC` flag was erroneously omitted in one + of the involved types), the objects remain indefinitely uncollectable + (they "leak"). See :data:`gc.garbage`. + + If the object is marked as supporting garbage collection (the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`), the following events are also possible: + + * The garbage collector occasionally calls + :c:member:`~PyTypeObject.tp_traverse` to identify :term:`cyclic isolates + <cyclic isolate>`. + * When the garbage collector discovers a :term:`cyclic isolate`, it + finalizes one of the objects in the group by marking it as *finalized* and + calling its :c:member:`~PyTypeObject.tp_finalize` function, if it has one. + This repeats until the cyclic isolate doesn't exist or all of the objects + have been finalized. + * :c:member:`~PyTypeObject.tp_finalize` is permitted to resurrect the object + by adding a reference from outside the :term:`cyclic isolate`. The new + reference causes the group of objects to no longer form a cyclic isolate + (the reference cycle may still exist, but if it does the objects are no + longer isolated). + * When the garbage collector discovers a :term:`cyclic isolate` and all of + the objects in the group have already been marked as *finalized*, the + garbage collector clears one or more of the uncleared objects in the group + (possibly concurrently) by calling each's + :c:member:`~PyTypeObject.tp_clear` function. This repeats as long as the + cyclic isolate still exists and not all of the objects have been cleared. + + +Cyclic Isolate Destruction +-------------------------- + +Listed below are the stages of life of a hypothetical :term:`cyclic isolate` +that continues to exist after each member object is finalized or cleared. It +is a memory leak if a cyclic isolate progresses through all of these stages; it should +vanish once all objects are cleared, if not sooner. A cyclic isolate can +vanish either because the reference cycle is broken or because the objects are +no longer isolated due to finalizer resurrection (see +:c:member:`~PyTypeObject.tp_finalize`). + +0. **Reachable** (not yet a cyclic isolate): All objects are in their normal, + reachable state. A reference cycle could exist, but an external reference + means the objects are not yet isolated. +#. **Unreachable but consistent:** The final reference from outside the cyclic + group of objects has been removed, causing the objects to become isolated + (thus a cyclic isolate is born). None of the group's objects have been + finalized or cleared yet. The cyclic isolate remains at this stage until + some future run of the garbage collector (not necessarily the next run + because the next run might not scan every object). +#. **Mix of finalized and not finalized:** Objects in a cyclic isolate are + finalized one at a time, which means that there is a period of time when the + cyclic isolate is composed of a mix of finalized and non-finalized objects. + Finalization order is unspecified, so it can appear random. A finalized + object must behave in a sane manner when non-finalized objects interact with + it, and a non-finalized object must be able to tolerate the finalization of + an arbitrary subset of its referents. +#. **All finalized:** All objects in a cyclic isolate are finalized before any + of them are cleared. +#. **Mix of finalized and cleared:** The objects can be cleared serially or + concurrently (but with the :term:`GIL` held); either way, some will finish + before others. A finalized object must be able to tolerate the clearing of + a subset of its referents. :pep:`442` calls this stage "cyclic trash". +#. **Leaked:** If a cyclic isolate still exists after all objects in the group + have been finalized and cleared, then the objects remain indefinitely + uncollectable (see :data:`gc.garbage`). It is a bug if a cyclic isolate + reaches this stage---it means the :c:member:`~PyTypeObject.tp_clear` methods + of the participating objects have failed to break the reference cycle as + required. + +If :c:member:`~PyTypeObject.tp_clear` did not exist, then Python would have no +way to safely break a reference cycle. Simply destroying an object in a cyclic +isolate would result in a dangling pointer, triggering undefined behavior when +an object referencing the destroyed object is itself destroyed. The clearing +step makes object destruction a two-phase process: first +:c:member:`~PyTypeObject.tp_clear` is called to partially destroy the objects +enough to detangle them from each other, then +:c:member:`~PyTypeObject.tp_dealloc` is called to complete the destruction. + +Unlike clearing, finalization is not a phase of destruction. A finalized +object must still behave properly by continuing to fulfill its design +contracts. An object's finalizer is allowed to execute arbitrary Python code, +and is even allowed to prevent the impending destruction by adding a reference. +The finalizer is only related to destruction by call order---if it runs, it runs +before destruction, which starts with :c:member:`~PyTypeObject.tp_clear` (if +called) and concludes with :c:member:`~PyTypeObject.tp_dealloc`. + +The finalization step is not necessary to safely reclaim the objects in a +cyclic isolate, but its existence makes it easier to design types that behave +in a sane manner when objects are cleared. Clearing an object might +necessarily leave it in a broken, partially destroyed state---it might be +unsafe to call any of the cleared object's methods or access any of its +attributes. With finalization, only finalized objects can possibly interact +with cleared objects; non-finalized objects are guaranteed to interact with +only non-cleared (but potentially finalized) objects. + +To summarize the possible interactions: + +* A non-finalized object might have references to or from non-finalized and + finalized objects, but not to or from cleared objects. +* A finalized object might have references to or from non-finalized, finalized, + and cleared objects. +* A cleared object might have references to or from finalized and cleared + objects, but not to or from non-finalized objects. + +Without any reference cycles, an object can be simply destroyed once its last +reference is deleted; the finalization and clearing steps are not necessary to +safely reclaim unused objects. However, it can be useful to automatically call +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` +before destruction anyway because type design is simplified when all objects +always experience the same series of events regardless of whether they +participated in a cyclic isolate. Python currently only calls +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` as +needed to destroy a cyclic isolate; this may change in a future version. + + +Functions +--------- + +To allocate and free memory, see :ref:`allocating-objects`. + + +.. c:function:: void PyObject_CallFinalizer(PyObject *op) + + Finalizes the object as described in :c:member:`~PyTypeObject.tp_finalize`. + Call this function (or :c:func:`PyObject_CallFinalizerFromDealloc`) instead + of calling :c:member:`~PyTypeObject.tp_finalize` directly because this + function may deduplicate multiple calls to :c:member:`!tp_finalize`. + Currently, calls are only deduplicated if the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may + change in the future. + + +.. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) + + Same as :c:func:`PyObject_CallFinalizer` but meant to be called at the + beginning of the object's destructor (:c:member:`~PyTypeObject.tp_dealloc`). + There must not be any references to the object. If the object's finalizer + resurrects the object, this function returns -1; no further destruction + should happen. Otherwise, this function returns 0 and destruction can + continue normally. + + .. seealso:: + + :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index c15cecd41b89d1..758415a76e5cb4 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -38,9 +38,12 @@ List Objects .. note:: If *len* is greater than zero, the returned list object's items are - set to ``NULL``. Thus you cannot use abstract API functions such as - :c:func:`PySequence_SetItem` or expose the object to Python code before - setting all items to a real object with :c:func:`PyList_SetItem`. + set to ``NULL``. Thus you cannot use abstract API functions such as + :c:func:`PySequence_SetItem` or expose the object to Python code before + setting all items to a real object with :c:func:`PyList_SetItem` or + :c:func:`PyList_SET_ITEM()`. The following APIs are safe APIs before + the list is fully initialized: :c:func:`PyList_SetItem()` and :c:func:`PyList_SET_ITEM()`. + .. c:function:: Py_ssize_t PyList_Size(PyObject *list) @@ -56,13 +59,21 @@ List Objects Similar to :c:func:`PyList_Size`, but without error checking. -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) +.. c:function:: PyObject* PyList_GetItemRef(PyObject *list, Py_ssize_t index) Return the object at position *index* in the list pointed to by *list*. The position must be non-negative; indexing from the end of the list is not - supported. If *index* is out of bounds (<0 or >=len(list)), + supported. If *index* is out of bounds (:code:`<0 or >=len(list)`), return ``NULL`` and set an :exc:`IndexError` exception. + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) + + Like :c:func:`PyList_GetItemRef`, but returns a + :term:`borrowed reference` instead of a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) @@ -128,6 +139,30 @@ List Objects list is not supported. +.. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) + + Extend *list* with the contents of *iterable*. This is the same as + ``PyList_SetSlice(list, PY_SSIZE_T_MAX, PY_SSIZE_T_MAX, iterable)`` + and analogous to ``list.extend(iterable)`` or ``list += iterable``. + + Raise an exception and return ``-1`` if *list* is not a :class:`list` + object. Return 0 on success. + + .. versionadded:: 3.13 + + +.. c:function:: int PyList_Clear(PyObject *list) + + Remove all items from *list*. This is the same as + ``PyList_SetSlice(list, 0, PY_SSIZE_T_MAX, NULL)`` and analogous to + ``list.clear()`` or ``del list[:]``. + + Raise an exception and return ``-1`` if *list* is not a :class:`list` + object. Return 0 on success. + + .. versionadded:: 3.13 + + .. c:function:: int PyList_Sort(PyObject *list) Sort the items of *list* in place. Return ``0`` on success, ``-1`` on diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index f1354a34f2b2d5..25d9e62e387279 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -69,12 +69,32 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromInt32(int32_t value) + PyObject* PyLong_FromInt64(int64_t value) + + Return a new :c:type:`PyLongObject` object from a signed C + :c:expr:`int32_t` or :c:expr:`int64_t`, or ``NULL`` + with an exception set on failure. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, or ``NULL`` on failure. +.. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) + PyObject* PyLong_FromUInt64(uint64_t value) + + Return a new :c:type:`PyLongObject` object from an unsigned C + :c:expr:`uint32_t` or :c:expr:`uint64_t`, or ``NULL`` + with an exception set on failure. + + .. versionadded:: 3.14 + + .. c:function:: PyObject* PyLong_FromDouble(double v) Return a new :c:type:`PyLongObject` object from the integer part of *v*, or @@ -94,9 +114,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. ignored. If there are no digits or *str* is not NULL-terminated following the digits and trailing whitespace, :exc:`ValueError` will be raised. - .. seealso:: Python methods :meth:`int.to_bytes` and :meth:`int.from_bytes` - to convert a :c:type:`PyLongObject` to/from an array of bytes in base - ``256``. You can call those from C using :c:func:`PyObject_CallMethod`. + .. seealso:: :c:func:`PyLong_AsNativeBytes()` and + :c:func:`PyLong_FromNativeBytes()` functions can be used to convert + a :c:type:`PyLongObject` to/from an array of bytes in base ``256``. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) @@ -113,11 +133,36 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. -.. XXX alias PyLong_AS_LONG (for now) +.. c:function:: PyObject* PyLong_FromNativeBytes(const void* buffer, size_t n_bytes, int flags) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as a two's-complement signed number. + + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is a sign bit. Passing + ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` will produce the same result as calling + :c:func:`PyLong_FromUnsignedNativeBytes`. Other flags are ignored. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyLong_FromUnsignedNativeBytes(const void* buffer, size_t n_bytes, int flags) + + Create a Python integer from the value contained in the first *n_bytes* of + *buffer*, interpreted as an unsigned number. + + *flags* are as for :c:func:`PyLong_AsNativeBytes`. Passing ``-1`` will select + the native endian that CPython was compiled with and assume that the + most-significant bit is not a sign bit. Flags other than endian are ignored. + + .. versionadded:: 3.13 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: - single: LONG_MAX + single: LONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`long` representation of *obj*. If *obj* is not an @@ -135,6 +180,24 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionchanged:: 3.10 This function will no longer use :meth:`~object.__int__`. + .. c:namespace:: NULL + + .. c:function:: long PyLong_AS_LONG(PyObject *obj) + + A :term:`soft deprecated` alias. + Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, + it can fail with :exc:`OverflowError` or another exception. + + .. deprecated:: 3.14 + The function is soft deprecated. + +.. c:function:: int PyLong_AsInt(PyObject *obj) + + Similar to :c:func:`PyLong_AsLong`, but store the result in a C + :c:expr:`int` instead of a C :c:expr:`long`. + + .. versionadded:: 3.13 + .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) @@ -202,7 +265,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) .. index:: - single: PY_SSIZE_T_MAX + single: PY_SSIZE_T_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must @@ -217,7 +280,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) .. index:: - single: ULONG_MAX + single: ULONG_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:expr:`unsigned long` representation of *pylong*. *pylong* @@ -233,7 +296,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) .. index:: - single: SIZE_MAX + single: SIZE_MAX (C macro) single: OverflowError (built-in exception) Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be @@ -303,6 +366,43 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. This function will no longer use :meth:`~object.__int__`. +.. c:function:: int PyLong_AsInt32(PyObject *obj, int32_t *value) + int PyLong_AsInt64(PyObject *obj, int64_t *value) + + Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` + representation of *obj*. + + If the *obj* value is out of range, raise an :exc:`OverflowError`. + + Set *\*value* and return ``0`` on success. + Set an exception and return ``-1`` on error. + + *value* must not be ``NULL``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_AsUInt32(PyObject *obj, uint32_t *value) + int PyLong_AsUInt64(PyObject *obj, uint64_t *value) + + Set *\*value* to an unsigned C :c:expr:`uint32_t` or :c:expr:`uint64_t` + representation of *obj*. + + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + + * If *obj* is negative, raise a :exc:`ValueError`. + * If the *obj* value is out of range, raise an :exc:`OverflowError`. + + Set *\*value* and return ``0`` on success. + Set an exception and return ``-1`` on error. + + *value* must not be ``NULL``. + + .. versionadded:: 3.14 + + .. c:function:: double PyLong_AsDouble(PyObject *pylong) Return a C :c:expr:`double` representation of *pylong*. *pylong* must be @@ -324,6 +424,208 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. +.. c:function:: Py_ssize_t PyLong_AsNativeBytes(PyObject *pylong, void* buffer, Py_ssize_t n_bytes, int flags) + + Copy the Python integer value *pylong* to a native *buffer* of size + *n_bytes*. The *flags* can be set to ``-1`` to behave similarly to a C cast, + or to values documented below to control the behavior. + + Returns ``-1`` with an exception raised on error. This may happen if + *pylong* cannot be interpreted as an integer, or if *pylong* was negative + and the ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` flag was set. + + Otherwise, returns the number of bytes required to store the value. + If this is equal to or less than *n_bytes*, the entire value was copied. + All *n_bytes* of the buffer are written: large buffers are padded with + zeroes. + + If the returned value is greater than than *n_bytes*, the value was + truncated: as many of the lowest bits of the value as could fit are written, + and the higher bits are ignored. This matches the typical behavior + of a C-style downcast. + + .. note:: + + Overflow is not considered an error. If the returned value + is larger than *n_bytes*, most significant bits were discarded. + + ``0`` will never be returned. + + Values are always copied as two's-complement. + + Usage example:: + + int32_t value; + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1); + if (bytes < 0) { + // Failed. A Python exception was set with the reason. + return NULL; + } + else if (bytes <= (Py_ssize_t)sizeof(value)) { + // Success! + } + else { + // Overflow occurred, but 'value' contains the truncated + // lowest bits of pylong. + } + + Passing zero to *n_bytes* will return the size of a buffer that would + be large enough to hold the value. This may be larger than technically + necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be + ``NULL``. + + .. note:: + + Passing *n_bytes=0* to this function is not an accurate way to determine + the bit length of the value. + + To get at the entire Python value of an unknown size, the function can be + called twice: first to determine the buffer size, then to fill it:: + + // Ask how much space we need. + Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1); + if (expected < 0) { + // Failed. A Python exception was set with the reason. + return NULL; + } + assert(expected != 0); // Impossible per the API definition. + uint8_t *bignum = malloc(expected); + if (!bignum) { + PyErr_SetString(PyExc_MemoryError, "bignum malloc failed."); + return NULL; + } + // Safely get the entire value. + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1); + if (bytes < 0) { // Exception has been set. + free(bignum); + return NULL; + } + else if (bytes > expected) { // This should not be possible. + PyErr_SetString(PyExc_RuntimeError, + "Unexpected bignum truncation after a size check."); + free(bignum); + return NULL; + } + // The expected success given the above pre-check. + // ... use bignum ... + free(bignum); + + *flags* is either ``-1`` (``Py_ASNATIVEBYTES_DEFAULTS``) to select defaults + that behave most like a C cast, or a combination of the other flags in + the table below. + Note that ``-1`` cannot be combined with other flags. + + Currently, ``-1`` corresponds to + ``Py_ASNATIVEBYTES_NATIVE_ENDIAN | Py_ASNATIVEBYTES_UNSIGNED_BUFFER``. + + .. c:namespace:: NULL + + ============================================= ====== + Flag Value + ============================================= ====== + .. c:macro:: Py_ASNATIVEBYTES_DEFAULTS ``-1`` + .. c:macro:: Py_ASNATIVEBYTES_BIG_ENDIAN ``0`` + .. c:macro:: Py_ASNATIVEBYTES_LITTLE_ENDIAN ``1`` + .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN ``3`` + .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4`` + .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8`` + .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX ``16`` + ============================================= ====== + + Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian + flags. Passing ``2`` is reserved. + + By default, sufficient buffer will be requested to include a sign bit. + For example, when converting 128 with *n_bytes=1*, the function will return + 2 (or more) in order to store a zero sign bit. + + If ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` is specified, a zero sign bit + will be omitted from size calculations. This allows, for example, 128 to fit + in a single-byte buffer. If the destination buffer is later treated as + signed, a positive input value may become negative. + Note that the flag does not affect handling of negative values: for those, + space for a sign bit is always requested. + + Specifying ``Py_ASNATIVEBYTES_REJECT_NEGATIVE`` causes an exception to be set + if *pylong* is negative. Without this flag, negative values will be copied + provided there is enough space for at least one sign bit, regardless of + whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified. + + If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is + passed, its :meth:`~object.__index__` method will be called first. This may + result in Python code executing and other threads being allowed to run, which + could cause changes to other objects or values in use. When *flags* is + ``-1``, this option is not set, and non-integer values will raise + :exc:`TypeError`. + + .. note:: + + With the default *flags* (``-1``, or *UNSIGNED_BUFFER* without + *REJECT_NEGATIVE*), multiple Python integers can map to a single value + without overflow. For example, both ``255`` and ``-1`` fit a single-byte + buffer and set all its bits. + This matches typical C cast behavior. + + .. versionadded:: 3.13 + + +.. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) + + Get the sign of the integer object *obj*. + + On success, set *\*sign* to the integer sign (0, -1 or +1 for zero, negative or + positive integer, respectively) and return 0. + + On failure, return -1 with an exception set. This function always succeeds + if *obj* is a :c:type:`PyLongObject` or its subtype. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsPositive(PyObject *obj) + + Check if the integer object *obj* is positive (``obj > 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's positive and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsNegative(PyObject *obj) + + Check if the integer object *obj* is negative (``obj < 0``). + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's negative and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: int PyLong_IsZero(PyObject *obj) + + Check if the integer object *obj* is zero. + + If *obj* is an instance of :c:type:`PyLongObject` or its subtype, + return ``1`` when it's zero and ``0`` otherwise. Else set an + exception and return ``-1``. + + .. versionadded:: 3.14 + + +.. c:function:: PyObject* PyLong_GetInfo(void) + + On success, return a read only :term:`named tuple`, that holds + information about Python's internal representation of integers. + See :data:`sys.int_info` for description of individual fields. + + On failure, return ``NULL`` with an exception set. + + .. versionadded:: 3.1 + + .. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op) Return 1 if *op* is compact, 0 otherwise. @@ -332,13 +634,16 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. a “fast path” for small integers. For compact values use :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a :c:func:`PyLong_As* <PyLong_AsSize_t>` function or - :c:func:`calling <PyObject_CallMethod>` :meth:`int.to_bytes`. + :c:func:`PyLong_AsNativeBytes`. The speedup is expected to be negligible for most users. Exactly what values are considered compact is an implementation detail and is subject to change. + .. versionadded:: 3.12 + + .. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op) If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`, @@ -346,3 +651,179 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, the return value is undefined. + .. versionadded:: 3.12 + + +Export API +^^^^^^^^^^ + +.. versionadded:: 3.14 + +.. c:struct:: PyLongLayout + + Layout of an array of "digits" ("limbs" in the GMP terminology), used to + represent absolute value for arbitrary precision integers. + + Use :c:func:`PyLong_GetNativeLayout` to get the native layout of Python + :class:`int` objects, used internally for integers with "big enough" + absolute value. + + See also :data:`sys.int_info` which exposes similar information in Python. + + .. c:member:: uint8_t bits_per_digit + + Bits per digit. For example, a 15 bit digit means that bits 0-14 contain + meaningful information. + + .. c:member:: uint8_t digit_size + + Digit size in bytes. For example, a 15 bit digit will require at least 2 + bytes. + + .. c:member:: int8_t digits_order + + Digits order: + + - ``1`` for most significant digit first + - ``-1`` for least significant digit first + + .. c:member:: int8_t digit_endianness + + Digit endianness: + + - ``1`` for most significant byte first (big endian) + - ``-1`` for least significant byte first (little endian) + + +.. c:function:: const PyLongLayout* PyLong_GetNativeLayout(void) + + Get the native layout of Python :class:`int` objects. + + See the :c:struct:`PyLongLayout` structure. + + The function must not be called before Python initialization nor after + Python finalization. The returned layout is valid until Python is + finalized. The layout is the same for all Python sub-interpreters + in a process, and so it can be cached. + + +.. c:struct:: PyLongExport + + Export of a Python :class:`int` object. + + There are two cases: + + * If :c:member:`digits` is ``NULL``, only use the :c:member:`value` member. + * If :c:member:`digits` is not ``NULL``, use :c:member:`negative`, + :c:member:`ndigits` and :c:member:`digits` members. + + .. c:member:: int64_t value + + The native integer value of the exported :class:`int` object. + Only valid if :c:member:`digits` is ``NULL``. + + .. c:member:: uint8_t negative + + ``1`` if the number is negative, ``0`` otherwise. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: Py_ssize_t ndigits + + Number of digits in :c:member:`digits` array. + Only valid if :c:member:`digits` is not ``NULL``. + + .. c:member:: const void *digits + + Read-only array of unsigned digits. Can be ``NULL``. + + +.. c:function:: int PyLong_Export(PyObject *obj, PyLongExport *export_long) + + Export a Python :class:`int` object. + + *export_long* must point to a :c:struct:`PyLongExport` structure allocated + by the caller. It must not be ``NULL``. + + On success, fill in *\*export_long* and return ``0``. + On error, set an exception and return ``-1``. + + :c:func:`PyLong_FreeExport` must be called when the export is no longer + needed. + + .. impl-detail:: + This function always succeeds if *obj* is a Python :class:`int` object + or a subclass. + + +.. c:function:: void PyLong_FreeExport(PyLongExport *export_long) + + Release the export *export_long* created by :c:func:`PyLong_Export`. + + .. impl-detail:: + Calling :c:func:`PyLong_FreeExport` is optional if *export_long->digits* + is ``NULL``. + + +PyLongWriter API +^^^^^^^^^^^^^^^^ + +The :c:type:`PyLongWriter` API can be used to import an integer. + +.. versionadded:: 3.14 + +.. c:struct:: PyLongWriter + + A Python :class:`int` writer instance. + + The instance must be destroyed by :c:func:`PyLongWriter_Finish` or + :c:func:`PyLongWriter_Discard`. + + +.. c:function:: PyLongWriter* PyLongWriter_Create(int negative, Py_ssize_t ndigits, void **digits) + + Create a :c:type:`PyLongWriter`. + + On success, allocate *\*digits* and return a writer. + On error, set an exception and return ``NULL``. + + *negative* is ``1`` if the number is negative, or ``0`` otherwise. + + *ndigits* is the number of digits in the *digits* array. It must be + greater than 0. + + *digits* must not be NULL. + + After a successful call to this function, the caller should fill in the + array of digits *digits* and then call :c:func:`PyLongWriter_Finish` to get + a Python :class:`int`. + The layout of *digits* is described by :c:func:`PyLong_GetNativeLayout`. + + Digits must be in the range [``0``; ``(1 << bits_per_digit) - 1``] + (where the :c:struct:`~PyLongLayout.bits_per_digit` is the number of bits + per digit). + Any unused most significant digits must be set to ``0``. + + Alternately, call :c:func:`PyLongWriter_Discard` to destroy the writer + instance without creating an :class:`~int` object. + + +.. c:function:: PyObject* PyLongWriter_Finish(PyLongWriter *writer) + + Finish a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + On success, return a Python :class:`int` object. + On error, set an exception and return ``NULL``. + + The function takes care of normalizing the digits and converts the object + to a compact integer if needed. + + The writer instance and the *digits* array are invalid after the call. + + +.. c:function:: void PyLongWriter_Discard(PyLongWriter *writer) + + Discard a :c:type:`PyLongWriter` created by :c:func:`PyLongWriter_Create`. + + If *writer* is ``NULL``, no operation is performed. + + The writer instance and the *digits* array are invalid after the call. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 82011d9d36a524..1f55c0aa955c75 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -28,9 +28,9 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - Return element of *o* corresponding to the string *key* or ``NULL`` on failure. - This is the equivalent of the Python expression ``o[key]``. - See also :c:func:`PyObject_GetItem`. + This is the same as :c:func:`PyObject_GetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyMapping_GetOptionalItem(PyObject *obj, PyObject *key, PyObject **result) @@ -50,60 +50,78 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. c:function:: int PyMapping_GetOptionalItemString(PyObject *obj, const char *key, PyObject **result) - Variant of :c:func:`PyMapping_GetItemString` which doesn't raise - :exc:`KeyError` if the key is not found. - - If the key is found, return ``1`` and set *\*result* to a new - :term:`strong reference` to the corresponding value. - If the key is not found, return ``0`` and set *\*result* to ``NULL``; - the :exc:`KeyError` is silenced. - If an error other than :exc:`KeyError` is raised, return ``-1`` and - set *\*result* to ``NULL``. + This is the same as :c:func:`PyMapping_GetOptionalItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. versionadded:: 3.13 .. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) - Map the string *key* to the value *v* in object *o*. Returns ``-1`` on - failure. This is the equivalent of the Python statement ``o[key] = v``. - See also :c:func:`PyObject_SetItem`. This function *does not* steal a - reference to *v*. + This is the same as :c:func:`PyObject_SetItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) - Remove the mapping for the object *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. This is an alias of :c:func:`PyObject_DelItem`. .. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) - Remove the mapping for the string *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. -.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) +.. c:function:: int PyMapping_HasKeyWithError(PyObject *o, PyObject *key) Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This is equivalent to the Python expression ``key in o``. - This function always succeeds. + On failure, return ``-1``. - Note that exceptions which occur while calling the :meth:`~object.__getitem__` - method will get suppressed. - To get error reporting use :c:func:`PyObject_GetItem()` instead. + .. versionadded:: 3.13 -.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) +.. c:function:: int PyMapping_HasKeyStringWithError(PyObject *o, const char *key) + + This is the same as :c:func:`PyMapping_HasKeyWithError`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + +.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. This is equivalent to the Python expression ``key in o``. This function always succeeds. - Note that exceptions which occur while calling the :meth:`~object.__getitem__` - method and creating a temporary string object will get suppressed. - To get error reporting use :c:func:`PyMapping_GetItemString()` instead. + .. note:: + + Exceptions which occur when this calls :meth:`~object.__getitem__` + method are silently ignored. + For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, + :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. + + +.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) + + This is the same as :c:func:`PyMapping_HasKey`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. note:: + + Exceptions that occur when this calls :meth:`~object.__getitem__` + method or while creating the temporary :class:`str` + object are silently ignored. + For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, + :c:func:`PyMapping_GetOptionalItemString` or + :c:func:`PyMapping_GetItemString` instead. .. c:function:: PyObject* PyMapping_Keys(PyObject *o) diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 489f1580a414b2..61218a1bf6f171 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -13,11 +13,12 @@ binary mode. Numeric values are stored with the least significant byte first. -The module supports two versions of the data format: version 0 is the -historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating point numbers. -``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). +The module supports several versions of the data format; see +the :py:mod:`Python module documentation <marshal>` for details. +.. c:macro:: Py_MARSHAL_VERSION + + The current format version. See :py:data:`marshal.version`. .. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 1df8c2b911ca8f..61fa49f8681cce 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -41,10 +41,10 @@ buffers is performed on demand by the Python memory manager through the Python/C API functions listed in this document. .. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() + single: malloc (C function) + single: calloc (C function) + single: realloc (C function) + single: free (C function) To avoid memory corruption, extension writers should never try to operate on Python objects with the functions exported by the C library: :c:func:`malloc`, @@ -102,37 +102,45 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here <default-memory-allocators>`. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. +table can be found at :ref:`here <default-memory-allocators>`. +The APIs used to allocate and free a block of memory must be from the same domain. +For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. The three allocation domains are: * Raw domain: intended for allocating memory for general-purpose memory buffers where the allocation *must* go to the system allocator or where the - allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. + allocator can operate without an :term:`attached thread state`. The memory + is requested directly from the system. See :ref:`Raw Memory Interface <raw-memoryinterface>`. * "Mem" domain: intended for allocating memory for Python buffers and general-purpose memory buffers where the allocation must be performed with - the :term:`GIL` held. The memory is taken from the Python private heap. + an :term:`attached thread state`. The memory is taken from the Python private heap. + See :ref:`Memory Interface <memoryinterface>`. -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. +* Object domain: intended for allocating memory for Python objects. The + memory is taken from the Python private heap. See :ref:`Object allocators <objectinterface>`. + +.. note:: + + The :term:`free-threaded <free threading>` build requires that only Python objects are allocated using the "object" domain + and that all Python objects are allocated using that domain. This differs from the prior Python versions, + where this was only a best practice and not a hard requirement. + + For example, buffers (non-Python objects) should be allocated using :c:func:`PyMem_Malloc`, + :c:func:`PyMem_RawMalloc`, or :c:func:`malloc`, but not :c:func:`PyObject_Malloc`. + + See :ref:`Memory Allocation APIs <free-threaded-memory-allocation>`. -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. + +.. _raw-memoryinterface: Raw Memory Interface ==================== The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL <global interpreter lock>` does not -need to be held. +functions are thread-safe, so a :term:`thread state` does not +need to be :term:`attached <attached thread state>`. The :ref:`default raw memory allocator <default-memory-allocators>` uses the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` @@ -205,8 +213,7 @@ The :ref:`default memory allocator <default-memory-allocators>` uses the .. warning:: - The :term:`GIL <global interpreter lock>` must be held when using these - functions. + There must be an :term:`attached thread state` when using these functions. .. versionchanged:: 3.6 @@ -267,14 +274,14 @@ The following type-oriented macros are provided for convenience. Note that .. c:macro:: PyMem_New(TYPE, n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:expr:`TYPE*`. The memory will not have + memory. Returns a pointer cast to ``TYPE*``. The memory will not have been initialized in any way. .. c:macro:: PyMem_Resize(p, TYPE, n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:expr:`TYPE*`. On return, + sizeof(TYPE))`` bytes. Returns a pointer cast to ``TYPE*``. On return, *p* will be a pointer to the new memory area, or ``NULL`` in the event of failure. @@ -299,6 +306,8 @@ versions and is therefore deprecated in extension modules. * ``PyMem_DEL(ptr)`` +.. _objectinterface: + Object allocators ================= @@ -317,8 +326,7 @@ The :ref:`default object allocator <default-memory-allocators>` uses the .. warning:: - The :term:`GIL <global interpreter lock>` must be held when using these - functions. + There must be an :term:`attached thread state` when using these functions. .. c:function:: void* PyObject_Malloc(size_t n) @@ -368,6 +376,24 @@ The :ref:`default object allocator <default-memory-allocators>` uses the If *p* is ``NULL``, no operation is performed. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_GC_New` or + :c:macro:`PyObject_GC_NewVar`; use :c:func:`PyObject_GC_Del` instead. + + .. seealso:: + + * :c:func:`PyObject_GC_Del` is the equivalent of this function for memory + allocated by types that support garbage collection. + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:macro:`PyObject_New` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. _default-memory-allocators: @@ -391,6 +417,8 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator <pymalloc>`. +* ``mimalloc``: :ref:`mimalloc memory allocator <mimalloc>`. The pymalloc + allocator will be used if mimalloc support isn't available. * "+ debug": with :ref:`debug hooks on the Python memory allocators <pymem-debug-hooks>`. * "Debug build": :ref:`Python build in debug mode <debug-build>`. @@ -473,12 +501,12 @@ Customize Memory Allocators zero bytes. For the :c:macro:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL <global interpreter lock>` is not held when the - allocator is called. + thread-safe: a :term:`thread state` is not :term:`attached <attached thread state>` + when the allocator is called. For the remaining domains, the allocator must also be thread-safe: the allocator may be called in different interpreters that do not - share a ``GIL``. + share a :term:`GIL`. If the new allocator is not a hook (does not call the previous allocator), the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the @@ -491,18 +519,18 @@ Customize Memory Allocators :c:func:`PyMem_SetAllocator` does have the following contract: - * It can be called after :c:func:`Py_PreInitialize` and before - :c:func:`Py_InitializeFromConfig` to install a custom memory - allocator. There are no restrictions over the installed allocator - other than the ones imposed by the domain (for instance, the Raw - Domain allows the allocator to be called without the GIL held). See - :ref:`the section on allocator domains <allocator-domains>` for more - information. + * It can be called after :c:func:`Py_PreInitialize` and before + :c:func:`Py_InitializeFromConfig` to install a custom memory + allocator. There are no restrictions over the installed allocator + other than the ones imposed by the domain (for instance, the Raw + Domain allows the allocator to be called without an :term:`attached thread state`). + See :ref:`the section on allocator domains <allocator-domains>` for more + information. - * If called after Python has finish initializing (after - :c:func:`Py_InitializeFromConfig` has been called) the allocator - **must** wrap the existing allocator. Substituting the current - allocator for some other arbitrary one is **not supported**. + * If called after Python has finish initializing (after + :c:func:`Py_InitializeFromConfig` has been called) the allocator + **must** wrap the existing allocator. Substituting the current + allocator for some other arbitrary one is **not supported**. .. versionchanged:: 3.12 All allocators must be thread-safe. @@ -543,7 +571,7 @@ Runtime checks: called on a memory block allocated by :c:func:`PyMem_Malloc`. - Detect write before the start of the buffer (buffer underflow). - Detect write after the end of the buffer (buffer overflow). -- Check that the :term:`GIL <global interpreter lock>` is held when +- Check that there is an :term:`attached thread state` when allocator functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) and :c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) domains are called. @@ -608,8 +636,8 @@ PYMEM_CLEANBYTE (meaning uninitialized memory is getting used). The :c:func:`PyMem_SetupDebugHooks` function now also works on Python compiled in release mode. On error, the debug hooks now use :mod:`tracemalloc` to get the traceback where a memory block was allocated. - The debug hooks now also check if the GIL is held when functions of - :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are + The debug hooks now also check if there is an :term:`attached thread state` when + functions of :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are called. .. versionchanged:: 3.8 @@ -626,7 +654,8 @@ The pymalloc allocator Python has a *pymalloc* allocator optimized for small objects (smaller or equal to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and +with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit +platforms. It falls back to :c:func:`PyMem_RawMalloc` and :c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. *pymalloc* is the :ref:`default allocator <default-memory-allocators>` of the @@ -671,6 +700,16 @@ Customize pymalloc Arena Allocator Set the arena allocator. +.. _mimalloc: + +The mimalloc allocator +====================== + +.. versionadded:: 3.13 + +Python supports the mimalloc allocator when the underlying platform support is available. +mimalloc "is a general purpose allocator with excellent performance characteristics. +Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." tracemalloc C API ================= @@ -721,7 +760,7 @@ The same code using the type-oriented function set:: return PyErr_NoMemory(); /* ...Do some I/O operation involving buf... */ res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ + PyMem_Free(buf); /* allocated with PyMem_New */ return res; Note that in the two examples above, the buffer is always manipulated via @@ -737,11 +776,11 @@ allocators operating on different heaps. :: ... PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ + free(buf1); /* Fatal -- should be PyMem_Free() */ In addition to the functions aimed at handling raw memory blocks from the Python heap, objects in Python are allocated and released with :c:macro:`PyObject_New`, -:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Del`. +:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Free`. These will be explained in the next chapter on defining and implementing new object types in C. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index 2aa43318e7a455..f6038032805259 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -20,6 +20,17 @@ any other object. read/write, otherwise it may be either read-only or read/write at the discretion of the exporter. + +.. c:macro:: PyBUF_READ + + Flag to request a readonly buffer. + + +.. c:macro:: PyBUF_WRITE + + Flag to request a writable buffer. + + .. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) Create a memoryview object using *mem* as the underlying buffer. @@ -41,6 +52,8 @@ any other object. original memory. Otherwise, a copy is made and the memoryview points to a new bytes object. + *buffertype* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. + .. c:function:: int PyMemoryView_Check(PyObject *obj) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 187f8419d4ee4f..710135dca89eda 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -37,16 +37,19 @@ Module Objects single: __package__ (module attribute) single: __loader__ (module attribute) - Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. + Return a new module object with :attr:`module.__name__` set to *name*. + The module's :attr:`!__name__`, :attr:`~module.__doc__`, + :attr:`~module.__package__` and :attr:`~module.__loader__` attributes are + filled in (all but :attr:`!__name__` are set to ``None``). The caller is + responsible for setting a :attr:`~module.__file__` attribute. + + Return ``NULL`` with an exception set on error. .. versionadded:: 3.3 .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. + :attr:`~module.__package__` and :attr:`~module.__loader__` are now set to + ``None``. .. c:function:: PyObject* PyModule_New(const char *name) @@ -75,8 +78,9 @@ Module Objects single: __name__ (module attribute) single: SystemError (built-in exception) - Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. + Return *module*'s :attr:`~module.__name__` value. If the module does not + provide one, or if it is not a string, :exc:`SystemError` is raised and + ``NULL`` is returned. .. versionadded:: 3.3 @@ -106,8 +110,8 @@ Module Objects single: SystemError (built-in exception) Return the name of the file from which *module* was loaded using *module*'s - :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return + :attr:`~module.__file__` attribute. If this is not defined, or if it is not a + string, raise :exc:`SystemError` and return ``NULL``; otherwise return a reference to a Unicode object. .. versionadded:: 3.2 @@ -265,6 +269,8 @@ of the following two module creation functions: API version *module_api_version*. If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_Create` @@ -282,22 +288,40 @@ An alternate way to specify extensions is to request "multi-phase initialization Extension modules created this way behave more like Python modules: the initialization is split between the *creation phase*, when the module object is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods -of classes. +The distinction is similar to the :py:meth:`~object.__new__` and +:py:meth:`~object.__init__` methods of classes. Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). +singletons. +For example, if the :py:attr:`sys.modules` entry is removed and the module +is re-imported, a new module object is created, and typically populated with +fresh method and type objects. +The old module is subject to normal garbage collection. +This mirrors the behavior of pure-Python modules. + +Additional module instances may be created in +:ref:`sub-interpreters <sub-interpreter-support>` +or after after Python runtime reinitialization +(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). +In these cases, sharing Python objects between module instances would likely +cause crashes or undefined behavior. + +To avoid such issues, each instance of an extension module should +be *isolated*: changes to one instance should not implicitly affect the others, +and all state, including references to Python objects, should be specific to +a particular module instance. +See :ref:`isolating-extensions-howto` for more details and a practical guide. + +A simpler way to avoid these issues is +:ref:`raising an error on repeated initialization <isolating-extensions-optout>`. All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters <sub-interpreter-support>`. Making sure multiple modules -are independent is typically enough to achieve this. +:ref:`sub-interpreters <sub-interpreter-support>`, or otherwise explicitly +signal a lack of support. +This is usually achieved by isolation or blocking repeated initialization, +as above. +A module may also be limited to the main interpreter using +the :c:data:`Py_mod_multiple_interpreters` slot. To request multi-phase initialization, the initialization function (PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty @@ -338,6 +362,8 @@ The available slot types are: The *value* pointer of this slot must point to a function of the signature: .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) + :no-index-entry: + :no-contents-entry: The function receives a :py:class:`~importlib.machinery.ModuleSpec` instance, as defined in :PEP:`451`, and the module definition. @@ -372,6 +398,8 @@ The available slot types are: The signature of the function is: .. c:function:: int exec_module(PyObject* module) + :no-index-entry: + :no-contents-entry: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. @@ -380,6 +408,8 @@ The available slot types are: Specifies one of the following values: + .. c:namespace:: NULL + .. c:macro:: Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED The module does not support being imported in subinterpreters. @@ -403,10 +433,37 @@ The available slot types are: in one module definition. If ``Py_mod_multiple_interpreters`` is not specified, the import - machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED``. + machinery defaults to ``Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED``. .. versionadded:: 3.12 +.. c:macro:: Py_mod_gil + + Specifies one of the following values: + + .. c:namespace:: NULL + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :ref:`whatsnew313-free-threaded-cpython` for more detail. + + Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded:: 3.13 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -432,6 +489,8 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and If that version does not match the version of the running interpreter, a :exc:`RuntimeWarning` is emitted. + Return ``NULL`` with an exception set on error. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -482,9 +541,6 @@ state: On success, return ``0``. On error, raise an exception and return ``-1``. - Return ``NULL`` if *value* is ``NULL``. It must be called with an exception - raised in this case. - Example usage:: static int @@ -499,6 +555,10 @@ state: return res; } + To be convenient, the function accepts ``NULL`` *value* with an exception + set. In this case, return ``-1`` and just leave the raised exception + unchanged. + The example can also be written without checking explicitly if *obj* is ``NULL``:: @@ -514,6 +574,14 @@ state: Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in this case, since *obj* can be ``NULL``. + The number of different *name* strings passed to this function + should be kept small, usually by only using statically allocated strings + as *name*. + For names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. + .. versionadded:: 3.10 @@ -572,15 +640,23 @@ state: .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. + used from the module's initialization function. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls :c:func:`PyLong_FromLong` and + :c:func:`PyModule_AddObjectRef`; see their documentation for details. .. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) Add a string constant to *module* as *name*. This convenience function can be used from the module's initialization function. The string *value* must be - ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. + ``NULL``-terminated. + Return ``-1`` with an exception set on error, ``0`` on success. + + This is a convenience function that calls + :c:func:`PyUnicode_InternFromString` and :c:func:`PyModule_AddObjectRef`; + see their documentation for details. .. c:macro:: PyModule_AddIntMacro(module, macro) @@ -588,7 +664,7 @@ state: Add an int constant to *module*. The name and the value are taken from *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. c:macro:: PyModule_AddStringMacro(module, macro) @@ -601,10 +677,23 @@ state: The type object is finalized by calling internally :c:func:`PyType_Ready`. The name of the type object is taken from the last component of :c:member:`~PyTypeObject.tp_name` after dot. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.9 +.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) + + Indicate that *module* does or does not support running without the global + interpreter lock (GIL), using one of the values from + :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization + function. If this function is not called during module initialization, the + import machinery assumes the module does not support running without the + GIL. This function is only available in Python builds configured with + :option:`--disable-gil`. + Return ``-1`` with an exception set on error, ``0`` on success. + + .. versionadded:: 3.13 + Module lookup ^^^^^^^^^^^^^ @@ -638,17 +727,17 @@ since multiple such modules can be created from a single definition. mechanisms (either by calling it directly, or by referring to its implementation for details of the required state updates). - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.3 .. c:function:: int PyState_RemoveModule(PyModuleDef *def) Removes the module object created from *def* from the interpreter state. - Return 0 on success or -1 on failure. + Return ``-1`` with an exception set on error, ``0`` on success. - The caller must hold the GIL. + The caller must have an :term:`attached thread state`. .. versionadded:: 3.3 diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst new file mode 100644 index 00000000000000..7926148302af0b --- /dev/null +++ b/Doc/c-api/monitoring.rst @@ -0,0 +1,210 @@ +.. highlight:: c + +.. _c-api-monitoring: + +Monitoring C API +================ + +Added in version 3.13. + +An extension may need to interact with the event monitoring system. Subscribing +to events and registering callbacks can be done via the Python API exposed in +:mod:`sys.monitoring`. + +Generating Execution Events +=========================== + +The functions below make it possible for an extension to fire monitoring +events as it emulates the execution of Python code. Each of these functions +accepts a ``PyMonitoringState`` struct which contains concise information +about the activation state of events, as well as the event arguments, which +include a ``PyObject*`` representing the code object, the instruction offset +and sometimes additional, event-specific arguments (see :mod:`sys.monitoring` +for details about the signatures of the different event callbacks). +The ``codelike`` argument should be an instance of :class:`types.CodeType` +or of a type that emulates it. + +The VM disables tracing when firing an event, so there is no need for user +code to do that. + +Monitoring functions should not be called with an exception set, +except those listed below as working with the current exception. + +.. c:type:: PyMonitoringState + + Representation of the state of an event type. It is allocated by the user + while its contents are maintained by the monitoring API functions described below. + + +All of the functions below return 0 on success and -1 (with an exception set) on error. + +See :mod:`sys.monitoring` for descriptions of the events. + +.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_START`` event. + + +.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_RESUME`` event. + + +.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_YIELD`` event. + + +.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0) + + Fire a ``CALL`` event. + + +.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno) + + Fire a ``LINE`` event. + + +.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``JUMP`` event. + + +.. c:function:: int PyMonitoring_FireBranchLeftEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_LEFT`` event. + + +.. c:function:: int PyMonitoring_FireBranchRightEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH_RIGHT`` event. + + +.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) + + Fire a ``C_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_THROW`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``C_RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RERAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_UNWIND`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *value) + + Fire a ``STOP_ITERATION`` event. If ``value`` is an instance of :exc:`StopIteration`, it is used. Otherwise, + a new :exc:`StopIteration` instance is created with ``value`` as its argument. + + +Managing the Monitoring State +----------------------------- + +Monitoring states can be managed with the help of monitoring scopes. A scope +would typically correspond to a python function. + +.. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) + + Enter a monitored scope. ``event_types`` is an array of the event IDs for + events that may be fired from the scope. For example, the ID of a ``PY_START`` + event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal + to the base-2 logarithm of ``sys.monitoring.events.PY_START``. + ``state_array`` is an array with a monitoring state entry for each event in + ``event_types``, it is allocated by the user but populated by + :c:func:`!PyMonitoring_EnterScope` with information about the activation state of + the event. The size of ``event_types`` (and hence also of ``state_array``) + is given in ``length``. + + The ``version`` argument is a pointer to a value which should be allocated + by the user together with ``state_array`` and initialized to 0, + and then set only by :c:func:`!PyMonitoring_EnterScope` itself. It allows this + function to determine whether event states have changed since the previous call, + and to return quickly if they have not. + + The scopes referred to here are lexical scopes: a function, class or method. + :c:func:`!PyMonitoring_EnterScope` should be called whenever the lexical scope is + entered. Scopes can be reentered, reusing the same *state_array* and *version*, + in situations like when emulating a recursive Python function. When a code-like's + execution is paused, such as when emulating a generator, the scope needs to + be exited and re-entered. + + The macros for *event_types* are: + + .. c:namespace:: NULL + + .. The table is here to make the docs searchable, and to allow automatic + links to the identifiers. + + ================================================== ===================================== + Macro Event + ================================================== ===================================== + .. c:macro:: PY_MONITORING_EVENT_BRANCH_LEFT :monitoring-event:`BRANCH_LEFT` + .. c:macro:: PY_MONITORING_EVENT_BRANCH_RIGHT :monitoring-event:`BRANCH_RIGHT` + .. c:macro:: PY_MONITORING_EVENT_CALL :monitoring-event:`CALL` + .. c:macro:: PY_MONITORING_EVENT_C_RAISE :monitoring-event:`C_RAISE` + .. c:macro:: PY_MONITORING_EVENT_C_RETURN :monitoring-event:`C_RETURN` + .. c:macro:: PY_MONITORING_EVENT_EXCEPTION_HANDLED :monitoring-event:`EXCEPTION_HANDLED` + .. c:macro:: PY_MONITORING_EVENT_INSTRUCTION :monitoring-event:`INSTRUCTION` + .. c:macro:: PY_MONITORING_EVENT_JUMP :monitoring-event:`JUMP` + .. c:macro:: PY_MONITORING_EVENT_LINE :monitoring-event:`LINE` + .. c:macro:: PY_MONITORING_EVENT_PY_RESUME :monitoring-event:`PY_RESUME` + .. c:macro:: PY_MONITORING_EVENT_PY_RETURN :monitoring-event:`PY_RETURN` + .. c:macro:: PY_MONITORING_EVENT_PY_START :monitoring-event:`PY_START` + .. c:macro:: PY_MONITORING_EVENT_PY_THROW :monitoring-event:`PY_THROW` + .. c:macro:: PY_MONITORING_EVENT_PY_UNWIND :monitoring-event:`PY_UNWIND` + .. c:macro:: PY_MONITORING_EVENT_PY_YIELD :monitoring-event:`PY_YIELD` + .. c:macro:: PY_MONITORING_EVENT_RAISE :monitoring-event:`RAISE` + .. c:macro:: PY_MONITORING_EVENT_RERAISE :monitoring-event:`RERAISE` + .. c:macro:: PY_MONITORING_EVENT_STOP_ITERATION :monitoring-event:`STOP_ITERATION` + ================================================== ===================================== + +.. c:function:: int PyMonitoring_ExitScope(void) + + Exit the last scope that was entered with :c:func:`!PyMonitoring_EnterScope`. + + +.. c:function:: int PY_MONITORING_IS_INSTRUMENTED_EVENT(uint8_t ev) + + Return true if the event corresponding to the event ID *ev* is + a :ref:`local event <monitoring-event-local>`. + + .. versionadded:: 3.13 + + .. deprecated:: 3.14 + + This function is :term:`soft deprecated`. diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst index dd8bfb56104251..f1941fc4bc4e85 100644 --- a/Doc/c-api/none.rst +++ b/Doc/c-api/none.rst @@ -16,10 +16,10 @@ same reason. .. c:var:: PyObject* Py_None The Python ``None`` object, denoting lack of value. This object has no methods - and is `immortal <https://peps.python.org/pep-0683/>`_. + and is :term:`immortal`. -.. versionchanged:: 3.12 - :c:data:`Py_None` is immortal. + .. versionchanged:: 3.12 + :c:data:`Py_None` is :term:`immortal`. .. c:macro:: Py_RETURN_NONE diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst index 13d3c5af956905..ad8b5935258fa7 100644 --- a/Doc/c-api/number.rst +++ b/Doc/c-api/number.rst @@ -51,8 +51,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. This is the equivalent of the Python expression ``o1 / o2``. @@ -177,8 +177,8 @@ Number Protocol Return a reasonable approximation for the mathematical value of *o1* divided by *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when + floating-point numbers are approximate; it is not possible to represent all real + numbers in base two. This function can return a floating-point value when passed two integers. The operation is done *in-place* when *o1* supports it. This is the equivalent of the Python statement ``o1 /= o2``. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 6d9fc7f50604da..0fd159f1eb87f8 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -6,6 +6,56 @@ Object Protocol =============== +.. c:function:: PyObject* Py_GetConstant(unsigned int constant_id) + + Get a :term:`strong reference` to a constant. + + Set an exception and return ``NULL`` if *constant_id* is invalid. + + *constant_id* must be one of these constant identifiers: + + .. c:namespace:: NULL + + ======================================== ===== ========================= + Constant Identifier Value Returned object + ======================================== ===== ========================= + .. c:macro:: Py_CONSTANT_NONE ``0`` :py:data:`None` + .. c:macro:: Py_CONSTANT_FALSE ``1`` :py:data:`False` + .. c:macro:: Py_CONSTANT_TRUE ``2`` :py:data:`True` + .. c:macro:: Py_CONSTANT_ELLIPSIS ``3`` :py:data:`Ellipsis` + .. c:macro:: Py_CONSTANT_NOT_IMPLEMENTED ``4`` :py:data:`NotImplemented` + .. c:macro:: Py_CONSTANT_ZERO ``5`` ``0`` + .. c:macro:: Py_CONSTANT_ONE ``6`` ``1`` + .. c:macro:: Py_CONSTANT_EMPTY_STR ``7`` ``''`` + .. c:macro:: Py_CONSTANT_EMPTY_BYTES ``8`` ``b''`` + .. c:macro:: Py_CONSTANT_EMPTY_TUPLE ``9`` ``()`` + ======================================== ===== ========================= + + Numeric values are only given for projects which cannot use the constant + identifiers. + + + .. versionadded:: 3.13 + + .. impl-detail:: + + In CPython, all of these constants are :term:`immortal`. + + +.. c:function:: PyObject* Py_GetConstantBorrowed(unsigned int constant_id) + + Similar to :c:func:`Py_GetConstant`, but return a :term:`borrowed + reference`. + + This function is primarily intended for backwards compatibility: + using :c:func:`Py_GetConstant` is recommended for new code. + + The reference is borrowed from the interpreter, and is valid until the + interpreter finalization. + + .. versionadded:: 3.13 + + .. c:var:: PyObject* Py_NotImplemented The ``NotImplemented`` singleton, used to signal that an operation is @@ -16,7 +66,15 @@ Object Protocol Properly handle returning :c:data:`Py_NotImplemented` from within a C function (that is, create a new :term:`strong reference` - to NotImplemented and return it). + to :const:`NotImplemented` and return it). + + +.. c:macro:: Py_PRINT_RAW + + Flag to be used with multiple functions that print the object (like + :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). + If passed, these function would use the :func:`str` of the object + instead of the :func:`repr`. .. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) @@ -27,32 +85,51 @@ Object Protocol instead of the :func:`repr`. +.. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) + + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. + This is equivalent to the Python expression ``hasattr(o, attr_name)``. + On failure, return ``-1``. + + .. versionadded:: 3.13 + + +.. c:function:: int PyObject_HasAttrStringWithError(PyObject *o, const char *attr_name) + + This is the same as :c:func:`PyObject_HasAttrWithError`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: 3.13 + + .. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. + Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. + This function always succeeds. .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods are silently ignored. - For proper error handling, use :c:func:`PyObject_GetOptionalAttr` or - :c:func:`PyObject_GetAttr` instead. + :meth:`~object.__getattribute__` methods aren't propagated, + but instead given to :func:`sys.unraisablehook`. + For proper error handling, use :c:func:`PyObject_HasAttrWithError`, + :c:func:`PyObject_GetOptionalAttr` or :c:func:`PyObject_GetAttr` instead. .. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. + This is the same as :c:func:`PyObject_HasAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. note:: Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods or while creating the temporary :class:`str` - object are silently ignored. - For proper error handling, use :c:func:`PyObject_GetOptionalAttrString` + :meth:`~object.__getattribute__` methods or while creating the temporary + :class:`str` object are silently ignored. + For proper error handling, use :c:func:`PyObject_HasAttrStringWithError`, + :c:func:`PyObject_GetOptionalAttrString` or :c:func:`PyObject_GetAttrString` instead. @@ -68,9 +145,9 @@ Object Protocol .. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) - Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or ``NULL`` on failure. This is the equivalent of the Python - expression ``o.attr_name``. + This is the same as :c:func:`PyObject_GetAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. If the missing attribute should not be treated as a failure, you can use :c:func:`PyObject_GetOptionalAttrString` instead. @@ -93,15 +170,9 @@ Object Protocol .. c:function:: int PyObject_GetOptionalAttrString(PyObject *obj, const char *attr_name, PyObject **result); - Variant of :c:func:`PyObject_GetAttrString` which doesn't raise - :exc:`AttributeError` if the attribute is not found. - - If the attribute is found, return ``1`` and set *\*result* to a new - :term:`strong reference` to the attribute. - If the attribute is not found, return ``0`` and set *\*result* to ``NULL``; - the :exc:`AttributeError` is silenced. - If an error other than :exc:`AttributeError` is raised, return ``-1`` and - set *\*result* to ``NULL``. + This is the same as :c:func:`PyObject_GetOptionalAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. .. versionadded:: 3.13 @@ -129,14 +200,20 @@ Object Protocol .. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) - Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement - ``o.attr_name = v``. + This is the same as :c:func:`PyObject_SetAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. If *v* is ``NULL``, the attribute is deleted, but this feature is deprecated in favour of using :c:func:`PyObject_DelAttrString`. + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_SetAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object. .. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) @@ -158,8 +235,17 @@ Object Protocol .. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name) - Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. - This is the equivalent of the Python statement ``del o.attr_name``. + This is the same as :c:func:`PyObject_DelAttr`, but *attr_name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + The number of different attribute names passed to this function + should be kept small, usually by using a statically allocated string + as *attr_name*. + For attribute names that aren't known at compile time, prefer calling + :c:func:`PyUnicode_FromString` and :c:func:`PyObject_DelAttr` directly. + For more details, see :c:func:`PyUnicode_InternFromString`, which may be + used internally to create a key object for lookup. .. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) @@ -209,12 +295,8 @@ Object Protocol .. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, - ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the - Python expression ``o1 op o2``, where ``op`` is the operator corresponding to - *opid*. + like :c:func:`PyObject_RichCompare`, but returns ``-1`` on error, ``0`` if + the result is false, ``1`` otherwise. .. note:: If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` @@ -286,14 +368,14 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to + If *cls* has a :meth:`~type.__subclasscheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. + i.e. contained in :attr:`cls.__mro__ <type.__mro__>`. Normally only class objects, i.e. instances of :class:`type` or a derived class, are considered classes. However, objects can override this by having - a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). + a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). .. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) @@ -305,15 +387,15 @@ Object Protocol The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to + If *cls* has a :meth:`~type.__instancecheck__` method, it will be called to determine the subclass status as described in :pep:`3119`. Otherwise, *inst* is an instance of *cls* if its class is a subclass of *cls*. An instance *inst* can override what is considered its class by having a - :attr:`~instance.__class__` attribute. + :attr:`~object.__class__` attribute. An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple + classes are, by having a :attr:`~type.__bases__` attribute (which must be a tuple of base classes). @@ -411,6 +493,13 @@ Object Protocol on failure. This is equivalent to the Python statement ``del o[key]``. +.. c:function:: int PyObject_DelItemString(PyObject *o, const char *key) + + This is the same as :c:func:`PyObject_DelItem`, but *key* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. c:function:: PyObject* PyObject_Dir(PyObject *o) This is equivalent to the Python expression ``dir(o)``, returning a (possibly @@ -428,6 +517,12 @@ Object Protocol iterated. +.. c:function:: PyObject* PyObject_SelfIter(PyObject *obj) + + This is equivalent to the Python ``__iter__(self): return self`` method. + It is intended for :term:`iterator` types, to be used in the :c:member:`PyTypeObject.tp_iter` slot. + + .. c:function:: PyObject* PyObject_GetAIter(PyObject *o) This is the equivalent to the Python expression ``aiter(o)``. Takes an @@ -476,3 +571,187 @@ Object Protocol :c:macro:`Py_TPFLAGS_ITEMS_AT_END` set. .. versionadded:: 3.12 + +.. c:function:: int PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) + + Visit the managed dictionary of *obj*. + + This function must only be called in a traverse function of the type which + has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. + + .. versionadded:: 3.13 + +.. c:function:: void PyObject_ClearManagedDict(PyObject *obj) + + Clear the managed dictionary of *obj*. + + This function must only be called in a traverse function of the type which + has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_Object_EnableDeferredRefcount(PyObject *obj) + + Enable `deferred reference counting <https://peps.python.org/pep-0703/#deferred-reference-counting>`_ on *obj*, + if supported by the runtime. In the :term:`free-threaded <free threading>` build, + this allows the interpreter to avoid reference count adjustments to *obj*, + which may improve multi-threaded performance. The tradeoff is + that *obj* will only be deallocated by the tracing garbage collector. + + This function returns ``1`` if deferred reference counting is enabled on *obj* + (including when it was enabled before the call), + and ``0`` if deferred reference counting is not supported or if the hint was + ignored by the runtime. This function is thread-safe, and cannot fail. + + This function does nothing on builds with the :term:`GIL` enabled, which do + not support deferred reference counting. This also does nothing if *obj* is not + an object tracked by the garbage collector (see :func:`gc.is_tracked` and + :c:func:`PyObject_GC_IsTracked`). + + This function is intended to be used soon after *obj* is created, + by the code that creates it. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *obj) + + Check if *obj* is a unique temporary object. + Returns ``1`` if *obj* is known to be a unique temporary object, + and ``0`` otherwise. This function cannot fail, but the check is + conservative, and may return ``0`` in some cases even if *obj* is a unique + temporary object. + + If an object is a unique temporary, it is guaranteed that the current code + has the only reference to the object. For arguments to C functions, this + should be used instead of checking if the reference count is ``1``. Starting + with Python 3.14, the interpreter internally avoids some reference count + modifications when loading objects onto the operands stack by + :term:`borrowing <borrowed reference>` references when possible, which means + that a reference count of ``1`` by itself does not guarantee that a function + argument uniquely referenced. + + In the example below, ``my_func`` is called with a unique temporary object + as its argument:: + + my_func([1, 2, 3]) + + In the example below, ``my_func`` is **not** called with a unique temporary + object as its argument, even if its refcount is ``1``:: + + my_list = [1, 2, 3] + my_func(my_list) + + See also the function :c:func:`Py_REFCNT`. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_IsImmortal(PyObject *obj) + + This function returns non-zero if *obj* is :term:`immortal`, and zero + otherwise. This function cannot fail. + + .. note:: + + Objects that are immortal in one CPython version are not guaranteed to + be immortal in another. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_TryIncRef(PyObject *obj) + + Increments the reference count of *obj* if it is not zero. Returns ``1`` + if the object's reference count was successfully incremented. Otherwise, + this function returns ``0``. + + :c:func:`PyUnstable_EnableTryIncRef` must have been called + earlier on *obj* or this function may spuriously return ``0`` in the + :term:`free threading` build. + + This function is logically equivalent to the following C code, except that + it behaves atomically in the :term:`free threading` build:: + + if (Py_REFCNT(op) > 0) { + Py_INCREF(op); + return 1; + } + return 0; + + This is intended as a building block for managing weak references + without the overhead of a Python :ref:`weak reference object <weakrefobjects>`. + + Typically, correct use of this function requires support from *obj*'s + deallocator (:c:member:`~PyTypeObject.tp_dealloc`). + For example, the following sketch could be adapted to implement a + "weakmap" that works like a :py:class:`~weakref.WeakValueDictionary` + for a specific type: + + .. code-block:: c + + PyMutex mutex; + + PyObject * + add_entry(weakmap_key_type *key, PyObject *value) + { + PyUnstable_EnableTryIncRef(value); + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_add_entry(weakmap, key, value); + PyMutex_Unlock(&mutex); + Py_RETURN_NONE; + } + + PyObject * + get_value(weakmap_key_type *key) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + PyObject *result = weakmap_find(weakmap, key); + if (PyUnstable_TryIncRef(result)) { + // `result` is safe to use + PyMutex_Unlock(&mutex); + return result; + } + // if we get here, `result` is starting to be garbage-collected, + // but has not been removed from the weakmap yet + PyMutex_Unlock(&mutex); + return NULL; + } + + // tp_dealloc function for weakmap values + void + value_dealloc(PyObject *value) + { + weakmap_type weakmap = ...; + PyMutex_Lock(&mutex); + weakmap_remove_value(weakmap, value); + + ... + PyMutex_Unlock(&mutex); + } + + .. versionadded:: 3.14 + +.. c:function:: void PyUnstable_EnableTryIncRef(PyObject *obj) + + Enables subsequent uses of :c:func:`PyUnstable_TryIncRef` on *obj*. The + caller must hold a :term:`strong reference` to *obj* when calling this. + + .. versionadded:: 3.14 + +.. c:function:: int PyUnstable_Object_IsUniquelyReferenced(PyObject *op) + + Determine if *op* only has one reference. + + On GIL-enabled builds, this function is equivalent to + :c:expr:`Py_REFCNT(op) == 1`. + + On a :term:`free threaded <free threading>` build, this checks if *op*'s + :term:`reference count` is equal to one and additionally checks if *op* + is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not** + thread-safe on free threaded builds; prefer this function. + + The caller must hold an :term:`attached thread state`, despite the fact + that this function doesn't call into the Python interpreter. This function + cannot fail. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 8bd8c107c98bdf..83de4248039949 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -12,6 +12,7 @@ object types. .. toctree:: allocation.rst + lifecycle.rst structures.rst typeobj.rst gcsupport.rst diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 3d44d2eb6bf41d..77b5e3c0876bbb 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -16,7 +16,7 @@ kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt>`_ In Python, these helper APIs can be used by libraries and features that rely on generating machine code on the fly. -Note that holding the Global Interpreter Lock (GIL) is not required for these APIs. +Note that holding an :term:`attached thread state` is not required for these APIs. .. c:function:: int PyUnstable_PerfMapState_Init(void) diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 118af7a1a8cf90..b23f016f9b0a06 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -17,26 +17,36 @@ of Python objects. Note that the returned value may not actually reflect how many references to the object are actually held. For example, some - objects are "immortal" and have a very high refcount that does not + objects are :term:`immortal` and have a very high refcount that does not reflect the actual number of references. Consequently, do not rely on the returned value to be accurate, other than a value of 0 or 1. Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. - .. versionchanged:: 3.11 - The parameter type is no longer :c:expr:`const PyObject*`. + .. note:: + + On :term:`free threaded <free threading>` builds of Python, returning 1 + isn't sufficient to determine if it's safe to treat *o* as having no + access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced` + for that instead. + + See also the function :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary()`. .. versionchanged:: 3.10 :c:func:`Py_REFCNT()` is changed to the inline static function. + .. versionchanged:: 3.11 + The parameter type is no longer :c:expr:`const PyObject*`. + .. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) Set the object *o* reference counter to *refcnt*. - Note that this function has no effect on - `immortal <https://peps.python.org/pep-0683/>`_ - objects. + On :ref:`Python build with Free Threading <free-threading-build>`, if + *refcnt* is larger than ``UINT32_MAX``, the object is made :term:`immortal`. + + This function has no effect on :term:`immortal` objects. .. versionadded:: 3.9 @@ -49,6 +59,8 @@ of Python objects. Indicate taking a new :term:`strong reference` to object *o*, indicating it is in use and should not be destroyed. + This function has no effect on :term:`immortal` objects. + This function is usually used to convert a :term:`borrowed reference` to a :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be used to create a new :term:`strong reference`. @@ -59,7 +71,7 @@ of Python objects. ``NULL``, use :c:func:`Py_XINCREF`. Do not expect this function to actually modify *o* in any way. - For at least `some objects <https://peps.python.org/pep-0683/>`_, + For at least :pep:`some objects <0683>`, this function has no effect. .. versionchanged:: 3.12 @@ -113,6 +125,8 @@ of Python objects. Release a :term:`strong reference` to object *o*, indicating the reference is no longer used. + This function has no effect on :term:`immortal` objects. + Once the last :term:`strong reference` is released (i.e. the object's reference count reaches 0), the object's type's deallocation @@ -125,7 +139,7 @@ of Python objects. use :c:func:`Py_XDECREF`. Do not expect this function to actually modify *o* in any way. - For at least `some objects <https://peps.python.org/pep-0683/>`_, + For at least :pep:`some objects <683>`, this function has no effect. .. warning:: diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 4b1c4770848a30..54fd5a064aa2ac 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -7,30 +7,90 @@ Reflection .. c:function:: PyObject* PyEval_GetBuiltins(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameBuiltins` instead. + Return a dictionary of the builtins in the current execution frame, or the interpreter of the thread state if no frame is currently executing. .. c:function:: PyObject* PyEval_GetLocals(void) - Return a dictionary of the local variables in the current execution frame, + .. deprecated:: 3.13 + + Use either :c:func:`PyEval_GetFrameLocals` to obtain the same behaviour as calling + :func:`locals` in Python code, or else call :c:func:`PyFrame_GetLocals` on the result + of :c:func:`PyEval_GetFrame` to access the :attr:`~frame.f_locals` attribute of the + currently executing frame. + + Return a mapping providing access to the local variables in the current execution frame, or ``NULL`` if no frame is currently executing. + Refer to :func:`locals` for details of the mapping returned at different scopes. + + As this function returns a :term:`borrowed reference`, the dictionary returned for + :term:`optimized scopes <optimized scope>` is cached on the frame object and will remain + alive as long as the frame object does. Unlike :c:func:`PyEval_GetFrameLocals` and + :func:`locals`, subsequent calls to this function in the same frame will update the + contents of the cached dictionary to reflect changes in the state of the local variables + rather than returning a new snapshot. + + .. versionchanged:: 3.13 + As part of :pep:`667`, :c:func:`PyFrame_GetLocals`, :func:`locals`, and + :attr:`FrameType.f_locals <frame.f_locals>` no longer make use of the shared cache + dictionary. Refer to the :ref:`What's New entry <whatsnew313-locals-semantics>` for + additional details. + .. c:function:: PyObject* PyEval_GetGlobals(void) + .. deprecated:: 3.13 + + Use :c:func:`PyEval_GetFrameGlobals` instead. + Return a dictionary of the global variables in the current execution frame, or ``NULL`` if no frame is currently executing. .. c:function:: PyFrameObject* PyEval_GetFrame(void) - Return the current thread state's frame, which is ``NULL`` if no frame is + Return the :term:`attached thread state`'s frame, which is ``NULL`` if no frame is currently executing. See also :c:func:`PyThreadState_GetFrame`. +.. c:function:: PyObject* PyEval_GetFrameBuiltins(void) + + Return a dictionary of the builtins in the current execution frame, + or the interpreter of the thread state if no frame is currently executing. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameLocals(void) + + Return a dictionary of the local variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`locals` in Python code. + + To access :attr:`~frame.f_locals` on the current frame without making an independent + snapshot in :term:`optimized scopes <optimized scope>`, call :c:func:`PyFrame_GetLocals` + on the result of :c:func:`PyEval_GetFrame`. + + .. versionadded:: 3.13 + + +.. c:function:: PyObject* PyEval_GetFrameGlobals(void) + + Return a dictionary of the global variables in the current execution frame, + or ``NULL`` if no frame is currently executing. Equivalent to calling + :func:`globals` in Python code. + + .. versionadded:: 3.13 + + .. c:function:: const char* PyEval_GetFuncName(PyObject *func) Return the name of *func* if it is a function, class or instance object, else the diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index ce28839f5ba739..df5bf6b64a93a0 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -105,6 +105,15 @@ Sequence Protocol equivalent to the Python expression ``value in o``. +.. c:function:: int PySequence_In(PyObject *o, PyObject *value) + + Alias for :c:func:`PySequence_Contains`. + + .. deprecated:: 3.14 + The function is :term:`soft deprecated` and should no longer be used to + write new code. + + .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) Return the first index *i* for which ``o[i] == value``. On error, return diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 1e8a09509032f5..cba823aa027bd6 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -147,7 +147,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. @@ -163,4 +163,6 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. .. c:function:: int PySet_Clear(PyObject *set) - Empty an existing set of all elements. + Empty an existing set of all elements. Return ``0`` on + success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of + :class:`set` or its subtype. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 9e880c6b7f25ad..c6d761fe7fd1c9 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -23,7 +23,9 @@ Slice Objects Return a new slice object with the given values. The *start*, *stop*, and *step* parameters are used as the values of the slice object attributes of the same names. Any of the values may be ``NULL``, in which case the - ``None`` will be used for the corresponding attribute. Return ``NULL`` if + ``None`` will be used for the corresponding attribute. + + Return ``NULL`` with an exception set if the new object could not be allocated. @@ -52,7 +54,7 @@ Slice Objects of bounds indices are clipped in a manner consistent with the handling of normal slices. - Returns ``0`` on success and ``-1`` on error with exception set. + Return ``0`` on success and ``-1`` on error with an exception set. .. note:: This function is considered not safe for resizable sequences. @@ -95,7 +97,7 @@ Slice Objects ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``. - Return ``-1`` on error, ``0`` on success. + Return ``-1`` with an exception set on error, ``0`` on success. .. versionadded:: 3.6.1 @@ -116,11 +118,16 @@ Ellipsis Object ^^^^^^^^^^^^^^^ +.. c:var:: PyTypeObject PyEllipsis_Type + + The type of Python :const:`Ellipsis` object. Same as :class:`types.EllipsisType` + in the Python layer. + + .. c:var:: PyObject *Py_Ellipsis The Python ``Ellipsis`` object. This object has no methods. Like - :c:data:`Py_None`, it is an `immortal <https://peps.python.org/pep-0683/>`_. - singleton object. + :c:data:`Py_None`, it is an :term:`immortal` singleton object. .. versionchanged:: 3.12 :c:data:`Py_Ellipsis` is immortal. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index c66b296d304adc..9b65e0b8d23d93 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -16,9 +16,9 @@ CPython's Application Binary Interface (ABI) is forward- and backwards-compatible across a minor release (if these are compiled the same way; see :ref:`stable-abi-platform` below). So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, -but will need to be compiled separately for 3.9.x and 3.10.x. +but will need to be compiled separately for 3.9.x and 3.11.x. -There are two tiers of C API with different stability exepectations: +There are two tiers of C API with different stability expectations: - :ref:`Unstable API <unstable-c-api>`, may change in minor versions without a deprecation period. It is marked by the ``PyUnstable`` prefix in names. @@ -51,6 +51,7 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-application-binary-interface: Stable Application Binary Interface =================================== @@ -66,7 +67,7 @@ Limited C API Python 3.2 introduced the *Limited API*, a subset of Python's C API. Extensions that only use the Limited API can be -compiled once and work with multiple versions of Python. +compiled once and be loaded on multiple versions of Python. Contents of the Limited API are :ref:`listed below <limited-api-list>`. .. c:macro:: Py_LIMITED_API @@ -76,7 +77,7 @@ Contents of the Limited API are :ref:`listed below <limited-api-list>`. Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` corresponding to the lowest Python version your extension supports. - The extension will work without recompilation with all Python 3 releases + The extension will be ABI-compatible with all Python 3 releases from the specified one onward, and can use Limited API introduced up to that version. @@ -94,7 +95,15 @@ Stable ABI ---------- To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. +remain ABI-compatible across Python 3.x versions. + +.. note:: + + The Stable ABI prevents ABI issues, like linker errors due to missing + symbols or data corruption due to changes in structure layouts or function + signatures. + However, other changes in Python can change the *behavior* of extensions. + See Python's Backwards Compatibility Policy (:pep:`387`) for details. The Stable ABI contains symbols exposed in the :ref:`Limited API <limited-c-api>`, but also other ones – for example, functions necessary to diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 747cfa62294c21..987bc167c68d6c 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -63,6 +63,11 @@ under :ref:`reference counting <countingrefs>`. See documentation of :c:type:`PyVarObject` above. +.. c:var:: PyTypeObject PyBaseObject_Type + + The base class of all other objects, the same as :class:`object` in Python. + + .. c:function:: int Py_Is(PyObject *x, PyObject *y) Test if the *x* object is the *y* object, the same as ``x is y`` in Python. @@ -187,26 +192,26 @@ Implementing functions and methods PyObject *kwargs); -.. c:type:: _PyCFunctionFast +.. c:type:: PyCFunctionFast Type of the functions used to implement Python callables in C with signature :c:macro:`METH_FASTCALL`. The function signature is:: - PyObject *_PyCFunctionFast(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs); + PyObject *PyCFunctionFast(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs); -.. c:type:: _PyCFunctionFastWithKeywords +.. c:type:: PyCFunctionFastWithKeywords Type of the functions used to implement Python callables in C with signature :ref:`METH_FASTCALL | METH_KEYWORDS <METH_FASTCALL-METH_KEYWORDS>`. The function signature is:: - PyObject *_PyCFunctionFastWithKeywords(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames); + PyObject *PyCFunctionFastWithKeywords(PyObject *self, + PyObject *const *args, + Py_ssize_t nargs, + PyObject *kwnames); .. c:type:: PyCMethod @@ -290,7 +295,7 @@ There are these calling conventions: .. c:macro:: METH_FASTCALL Fast calling convention supporting only positional arguments. - The methods have the type :c:type:`_PyCFunctionFast`. + The methods have the type :c:type:`PyCFunctionFast`. The first parameter is *self*, the second parameter is a C array of :c:expr:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). @@ -306,7 +311,7 @@ There are these calling conventions: :c:expr:`METH_FASTCALL | METH_KEYWORDS` Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments, - with methods of type :c:type:`_PyCFunctionFastWithKeywords`. + with methods of type :c:type:`PyCFunctionFastWithKeywords`. Keyword arguments are passed the same way as in the :ref:`vectorcall protocol <vectorcall>`: there is an additional fourth :c:expr:`PyObject*` parameter @@ -399,6 +404,40 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. +.. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) + + Turn *ml* into a Python :term:`callable` object. + The caller must ensure that *ml* outlives the :term:`callable`. + Typically, *ml* is defined as a static variable. + + The *self* parameter will be passed as the *self* argument + to the C function in ``ml->ml_meth`` when invoked. + *self* can be ``NULL``. + + The :term:`callable` object's ``__module__`` attribute + can be set from the given *module* argument. + *module* should be a Python string, + which will be used as name of the module the function is defined in. + If unavailable, it can be set to :const:`None` or ``NULL``. + + .. seealso:: :attr:`function.__module__` + + The *cls* parameter will be passed as the *defining_class* + argument to the C function. + Must be set if :c:macro:`METH_METHOD` is set on ``ml->ml_flags``. + + .. versionadded:: 3.9 + + +.. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) + + Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. + + +.. c:function:: PyObject * PyCFunction_New(PyMethodDef *ml, PyObject *self) + + Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. + Accessing attributes of extension types --------------------------------------- @@ -406,7 +445,11 @@ Accessing attributes of extension types .. c:type:: PyMemberDef Structure which describes an attribute of a type which corresponds to a C - struct member. Its fields are, in order: + struct member. + When defining a class, put a NULL-terminated array of these + structures in the :c:member:`~PyTypeObject.tp_members` slot. + + Its fields are, in order: .. c:member:: const char* name @@ -415,15 +458,15 @@ Accessing attributes of extension types The string should be static, no copy is made of it. - .. c:member:: Py_ssize_t offset - - The offset in bytes that the member is located on the type’s object struct. - .. c:member:: int type The type of the member in the C struct. See :ref:`PyMemberDef-types` for the possible values. + .. c:member:: Py_ssize_t offset + + The offset in bytes that the member is located on the type’s object struct. + .. c:member:: int flags Zero or more of the :ref:`PyMemberDef-flags`, combined using bitwise OR. @@ -447,7 +490,8 @@ Accessing attributes of extension types ``PyMemberDef`` may contain a definition for the special member ``"__vectorcalloffset__"``, corresponding to :c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects. - These must be defined with ``Py_T_PYSSIZET`` and ``Py_READONLY``, for example:: + This member must be defined with ``Py_T_PYSSIZET``, and either + ``Py_READONLY`` or ``Py_READONLY | Py_RELATIVE_OFFSET``. For example:: static PyMemberDef spam_type_members[] = { {"__vectorcalloffset__", Py_T_PYSSIZET, @@ -468,6 +512,12 @@ Accessing attributes of extension types ``PyMemberDef`` is always available. Previously, it required including ``"structmember.h"``. + .. versionchanged:: 3.14 + + :c:macro:`Py_RELATIVE_OFFSET` is now allowed for + ``"__vectorcalloffset__"``, ``"__dictoffset__"`` and + ``"__weaklistoffset__"``. + .. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m) Get an attribute belonging to the object at address *obj_addr*. The @@ -513,19 +563,19 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: from ``PyObject``. Can only be used as part of :c:member:`Py_tp_members <PyTypeObject.tp_members>` - :c:type:`slot <PyTypeSlot>` when creating a class using negative + :c:type:`slot <PyType_Slot>` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - This flag is only used in :c:type:`PyTypeSlot`. + This flag is only used in :c:type:`PyType_Slot`. When setting :c:member:`~PyTypeObject.tp_members` during class creation, Python clears it and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: - single: READ_RESTRICTED - single: WRITE_RESTRICTED - single: RESTRICTED + single: READ_RESTRICTED (C macro) + single: WRITE_RESTRICTED (C macro) + single: RESTRICTED (C macro) .. versionchanged:: 3.10 @@ -536,7 +586,7 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: :c:macro:`Py_AUDIT_READ`; :c:macro:`!WRITE_RESTRICTED` does nothing. .. index:: - single: READONLY + single: READONLY (C macro) .. versionchanged:: 3.12 @@ -588,7 +638,7 @@ Macro name C type Python type (*): Zero-terminated, UTF8-encoded C string. With :c:macro:`!Py_T_STRING` the C representation is a pointer; - with :c:macro:`!Py_T_STRING_INLINE` the string is stored directly + with :c:macro:`!Py_T_STRING_INPLACE` the string is stored directly in the structure. (**): String of length 1. Only ASCII is accepted. @@ -599,24 +649,24 @@ Macro name C type Python type Reading a ``NULL`` pointer raises :py:exc:`AttributeError`. .. index:: - single: T_BYTE - single: T_SHORT - single: T_INT - single: T_LONG - single: T_LONGLONG - single: T_UBYTE - single: T_USHORT - single: T_UINT - single: T_ULONG - single: T_ULONGULONG - single: T_PYSSIZET - single: T_FLOAT - single: T_DOUBLE - single: T_BOOL - single: T_CHAR - single: T_STRING - single: T_STRING_INPLACE - single: T_OBJECT_EX + single: T_BYTE (C macro) + single: T_SHORT (C macro) + single: T_INT (C macro) + single: T_LONG (C macro) + single: T_LONGLONG (C macro) + single: T_UBYTE (C macro) + single: T_USHORT (C macro) + single: T_UINT (C macro) + single: T_ULONG (C macro) + single: T_ULONGULONG (C macro) + single: T_PYSSIZET (C macro) + single: T_FLOAT (C macro) + single: T_DOUBLE (C macro) + single: T_BOOL (C macro) + single: T_CHAR (C macro) + single: T_STRING (C macro) + single: T_STRING_INPLACE (C macro) + single: T_OBJECT_EX (C macro) single: structmember.h .. versionadded:: 3.12 @@ -655,7 +705,8 @@ Defining Getters and Setters .. c:member:: setter set - Optional C function to set or delete the attribute, if omitted the attribute is readonly. + Optional C function to set or delete the attribute. + If ``NULL``, the attribute is read-only. .. c:member:: const char* doc @@ -663,20 +714,20 @@ Defining Getters and Setters .. c:member:: void* closure - Optional function pointer, providing additional data for getter and setter. + Optional user data pointer, providing additional data for getter and setter. - The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``):: +.. c:type:: PyObject *(*getter)(PyObject *, void *) - typedef PyObject *(*getter)(PyObject *, void *); + The ``get`` function takes one :c:expr:`PyObject*` parameter (the + instance) and a user data pointer (the associated ``closure``): It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``):: +.. c:type:: int (*setter)(PyObject *, PyObject *, void *) - typedef int (*setter)(PyObject *, PyObject *, void *); + ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and + the value to be set) and a user data pointer (the associated ``closure``): In case the attribute should be deleted the second parameter is ``NULL``. Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index aed625c5f6cdae..b34936dd55e94c 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -5,6 +5,7 @@ Operating System Utilities ========================== + .. c:function:: PyObject* PyOS_FSPath(PyObject *path) Return the file system representation for *path*. If the object is a @@ -97,27 +98,30 @@ Operating System Utilities .. c:function:: int PyOS_CheckStack() + .. index:: single: USE_STACKCHECK (C macro) + Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently + check, but is only available when :c:macro:`!USE_STACKCHECK` is defined (currently on certain versions of Windows using the Microsoft Visual C++ compiler). - :c:macro:`USE_STACKCHECK` will be defined automatically; you should never + :c:macro:`!USE_STACKCHECK` will be defined automatically; you should never change the definition in your own code. +.. c:type:: void (*PyOS_sighandler_t)(int) + + .. c:function:: PyOS_sighandler_t PyOS_getsig(int i) Return the current signal handler for signal *i*. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions - directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void - (\*)(int)`. + directly! .. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) Set the signal handler for signal *i* to be *h*; return the old signal handler. This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do - not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef - alias for :c:expr:`void (\*)(int)`. + not call those functions directly! .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) @@ -212,6 +216,38 @@ Operating System Utilities The function now uses the UTF-8 encoding on Windows if :c:member:`PyPreConfig.legacy_windows_fs_encoding` is zero. +.. c:function:: FILE* Py_fopen(PyObject *path, const char *mode) + + Similar to :c:func:`!fopen`, but *path* is a Python object and + an exception is set on error. + + *path* must be a :class:`str` object, a :class:`bytes` object, + or a :term:`path-like object`. + + On success, return the new file pointer. + On error, set an exception and return ``NULL``. + + The file must be closed by :c:func:`Py_fclose` rather than calling directly + :c:func:`!fclose`. + + The file descriptor is created non-inheritable (:pep:`446`). + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.14 + + +.. c:function:: int Py_fclose(FILE *file) + + Close a file that was opened by :c:func:`Py_fopen`. + + On success, return ``0``. + On error, return ``EOF`` and ``errno`` is set to indicate the error. + In either case, any further access (including another call to + :c:func:`Py_fclose`) to the stream results in undefined behavior. + + .. versionadded:: 3.14 + .. _systemfunctions: @@ -222,10 +258,57 @@ These are utility functions that make functionality from the :mod:`sys` module accessible to C code. They all work with the current interpreter thread's :mod:`sys` module's dict, which is contained in the internal thread state structure. +.. c:function:: PyObject *PySys_GetAttr(PyObject *name) + + Get the attribute *name* of the :mod:`sys` module. + Return a :term:`strong reference`. + Raise :exc:`RuntimeError` and return ``NULL`` if it does not exist or + if the :mod:`sys` module cannot be found. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttr` instead. + + .. versionadded:: next + +.. c:function:: PyObject *PySys_GetAttrString(const char *name) + + This is the same as :c:func:`PySys_GetAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + If the non-existing object should not be treated as a failure, you can use + :c:func:`PySys_GetOptionalAttrString` instead. + + .. versionadded:: next + +.. c:function:: int PySys_GetOptionalAttr(PyObject *name, PyObject **result) + + Variant of :c:func:`PySys_GetAttr` which doesn't raise + exception if the object does not exist. + + * Set *\*result* to a new :term:`strong reference` to the object and + return ``1`` if the object exists. + * Set *\*result* to ``NULL`` and return ``0`` without setting an exception + if the object does not exist. + * Set an exception, set *\*result* to ``NULL``, and return ``-1``, + if an error occurred. + + .. versionadded:: next + +.. c:function:: int PySys_GetOptionalAttrString(const char *name, PyObject **result) + + This is the same as :c:func:`PySys_GetOptionalAttr`, but *name* is + specified as a :c:expr:`const char*` UTF-8 encoded bytes string, + rather than a :c:expr:`PyObject*`. + + .. versionadded:: next + .. c:function:: PyObject *PySys_GetObject(const char *name) - Return the object *name* from the :mod:`sys` module or ``NULL`` if it does - not exist, without setting an exception. + Similar to :c:func:`PySys_GetAttrString`, but return a :term:`borrowed + reference` and return ``NULL`` *without* setting exception on failure. + + Preserves exception that was set before the call. .. c:function:: int PySys_SetObject(const char *name, PyObject *v) @@ -291,19 +374,24 @@ accessible to C code. They all work with the current interpreter thread's Raise an auditing event with any active hooks. Return zero for success and non-zero with an exception set on failure. + The *event* string argument must not be *NULL*. + If any hooks have been added, *format* and other arguments will be used to construct a tuple to pass. Apart from ``N``, the same format characters as used in :c:func:`Py_BuildValue` are available. If the built value is not - a tuple, it will be added into a single-element tuple. (The ``N`` format - option consumes a reference, but since there is no way to know whether - arguments to this function will be consumed, using it may cause reference - leaks.) + a tuple, it will be added into a single-element tuple. + + The ``N`` format option must not be used. It consumes a reference, but since + there is no way to know whether arguments to this function will be consumed, + using it may cause reference leaks. Note that ``#`` format characters should always be treated as :c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. :func:`sys.audit` performs the same function from Python code. + See also :c:func:`PySys_AuditTuple`. + .. versionadded:: 3.8 .. versionchanged:: 3.8.2 @@ -312,6 +400,14 @@ accessible to C code. They all work with the current interpreter thread's unavoidable deprecation warning was raised. +.. c:function:: int PySys_AuditTuple(const char *event, PyObject *args) + + Similar to :c:func:`PySys_Audit`, but pass arguments as a Python object. + *args* must be a :class:`tuple`. To pass no arguments, *args* can be *NULL*. + + .. versionadded:: 3.13 + + .. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) Append the callable *hook* to the list of active auditing hooks. @@ -329,10 +425,8 @@ accessible to C code. They all work with the current interpreter thread's silently abort the operation by raising an error subclassed from :class:`Exception` (other errors will not be silenced). - The hook function is of type :c:expr:`int (*)(const char *event, PyObject - *args, void *userData)`, where *args* is guaranteed to be a - :c:type:`PyTupleObject`. The hook function is always called with the GIL - held by the Python interpreter that raised the event. + The hook function is always called with an :term:`attached thread state` by + the Python interpreter that raised the event. See :pep:`578` for a detailed description of auditing. Functions in the runtime and standard library that raise events are listed in the @@ -341,12 +435,21 @@ accessible to C code. They all work with the current interpreter thread's .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook - If the interpreter is initialized, this function raises a auditing event + If the interpreter is initialized, this function raises an auditing event ``sys.addaudithook`` with no arguments. If any existing hooks raise an exception derived from :class:`Exception`, the new hook will not be added and the exception is cleared. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. + .. c:namespace:: NULL + .. c:type:: int (*Py_AuditHookFunction) (const char *event, PyObject *args, void *userData) + + The type of the hook function. + *event* is the C string event argument passed to :c:func:`PySys_Audit` or + :c:func:`PySys_AuditTuple`. + *args* is guaranteed to be a :c:type:`PyTupleObject`. + *userData* is the argument passed to PySys_AddAuditHook(). + .. versionadded:: 3.8 @@ -358,7 +461,7 @@ Process Control .. c:function:: void Py_FatalError(const char *message) - .. index:: single: abort() + .. index:: single: abort (C function) Print a fatal error message and kill the process. No cleanup is performed. This function should only be invoked when a condition is detected that would @@ -378,8 +481,8 @@ Process Control .. c:function:: void Py_Exit(int status) .. index:: - single: Py_FinalizeEx() - single: exit() + single: Py_FinalizeEx (C function) + single: exit (C function) Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` @@ -392,7 +495,7 @@ Process Control .. c:function:: int Py_AtExit(void (*func) ()) .. index:: - single: Py_FinalizeEx() + single: Py_FinalizeEx (C function) single: cleanup functions Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup @@ -402,3 +505,7 @@ Process Control function registered last is called first. Each cleanup function will be called at most once. Since Python's internal finalization will have completed before the cleanup function, no Python APIs should be called by *func*. + + .. seealso:: + + :c:func:`PyUnstable_AtExit` for passing a ``void *data`` argument. diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst new file mode 100644 index 00000000000000..31d6adbc225439 --- /dev/null +++ b/Doc/c-api/time.rst @@ -0,0 +1,114 @@ +.. highlight:: c + +.. _c-api-time: + +PyTime C API +============ + +.. versionadded:: 3.13 + +The clock C API provides access to system clocks. +It is similar to the Python :mod:`time` module. + +For C API related to the :mod:`datetime` module, see :ref:`datetimeobjects`. + + +Types +----- + +.. c:type:: PyTime_t + + A timestamp or duration in nanoseconds, represented as a signed 64-bit + integer. + + The reference point for timestamps depends on the clock used. For example, + :c:func:`PyTime_Time` returns timestamps relative to the UNIX epoch. + + The supported range is around [-292.3 years; +292.3 years]. + Using the Unix epoch (January 1st, 1970) as reference, the supported date + range is around [1677-09-21; 2262-04-11]. + The exact limits are exposed as constants: + +.. c:var:: PyTime_t PyTime_MIN + + Minimum value of :c:type:`PyTime_t`. + +.. c:var:: PyTime_t PyTime_MAX + + Maximum value of :c:type:`PyTime_t`. + + +Clock Functions +--------------- + +The following functions take a pointer to a :c:expr:`PyTime_t` that they +set to the value of a particular clock. +Details of each clock are given in the documentation of the corresponding +Python function. + +The functions return ``0`` on success, or ``-1`` (with an exception set) +on failure. + +On integer overflow, they set the :c:data:`PyExc_OverflowError` exception and +set ``*result`` to the value clamped to the ``[PyTime_MIN; PyTime_MAX]`` +range. +(On current systems, integer overflows are likely caused by misconfigured +system time.) + +As any other C API (unless otherwise specified), the functions must be called +with an :term:`attached thread state`. + +.. c:function:: int PyTime_Monotonic(PyTime_t *result) + + Read the monotonic clock. + See :func:`time.monotonic` for important details on this clock. + +.. c:function:: int PyTime_PerfCounter(PyTime_t *result) + + Read the performance counter. + See :func:`time.perf_counter` for important details on this clock. + +.. c:function:: int PyTime_Time(PyTime_t *result) + + Read the “wall clock” time. + See :func:`time.time` for details important on this clock. + + +Raw Clock Functions +------------------- + +Similar to clock functions, but don't set an exception on error and don't +require the caller to have an :term:`attached thread state`. + +On success, the functions return ``0``. + +On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting +an exception. To get the cause of the error, :term:`attach <attached thread state>` a :term:`thread state`, +and call the regular (non-``Raw``) function. Note that the regular function may succeed after +the ``Raw`` one failed. + +.. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Monotonic`, + but don't set an exception on error and don't require an :term:`attached thread state`. + +.. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_PerfCounter`, + but don't set an exception on error and don't require an :term:`attached thread state`. + +.. c:function:: int PyTime_TimeRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Time`, + but don't set an exception on error and don't require an :term:`attached thread state`. + + +Conversion functions +-------------------- + +.. c:function:: double PyTime_AsSecondsDouble(PyTime_t t) + + Convert a timestamp to a number of seconds as a C :c:expr:`double`. + + The function cannot fail, but note that :c:expr:`double` has limited + accuracy for large values. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b3710560ebe7ac..815afddad19df1 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -33,12 +33,14 @@ Tuple Objects .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - Return a new tuple object of size *len*, or ``NULL`` on failure. + Return a new tuple object of size *len*, + or ``NULL`` with an exception set on failure. .. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values + Return a new tuple object of size *n*, + or ``NULL`` with an exception set on failure. The tuple values are initialized to the subsequent *n* C arguments pointing to Python objects. ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. @@ -46,12 +48,12 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. + On error, return ``-1`` and with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - Return the size of the tuple *p*, which must be non-``NULL`` and point to a tuple; - no error checking is performed. + Like :c:func:`PyTuple_Size`, but without error checking. .. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) @@ -59,6 +61,12 @@ Tuple Objects Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. + The returned reference is borrowed from the tuple *p* + (that is: it is only valid as long as you hold a reference to *p*). + To get a :term:`strong reference`, use + :c:func:`Py_NewRef(PyTuple_GetItem(...)) <Py_NewRef>` + or :c:func:`PySequence_GetItem`. + .. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) @@ -68,8 +76,10 @@ Tuple Objects .. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) Return the slice of the tuple pointed to by *p* between *low* and *high*, - or ``NULL`` on failure. This is the equivalent of the Python expression - ``p[low:high]``. Indexing from the end of the tuple is not supported. + or ``NULL`` with an exception set on failure. + + This is the equivalent of the Python expression ``p[low:high]``. + Indexing from the end of the tuple is not supported. .. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -99,6 +109,12 @@ Tuple Objects is being replaced; any reference in the tuple at position *pos* will be leaked. + .. warning:: + + This macro should *only* be used on tuples that are newly created. + Using this macro on a tuple that is already in use (or in other words, has + a refcount > 1) could lead to undefined behavior. + .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) @@ -129,6 +145,8 @@ type. Create a new struct sequence type from the data in *desc*, described below. Instances of the resulting type can be created with :c:func:`PyStructSequence_New`. + Return ``NULL`` with an exception set on failure. + .. c:function:: void PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) @@ -137,8 +155,8 @@ type. .. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) - The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on - failure. + Like :c:func:`PyStructSequence_InitType`, but returns ``0`` on success + and ``-1`` with an exception set on failure. .. versionadded:: 3.4 @@ -149,7 +167,8 @@ type. .. c:member:: const char *name - Name of the struct sequence type. + Fully qualified name of the type; null-terminated UTF-8 encoded. + The name must contain the module name. .. c:member:: const char *doc @@ -195,6 +214,8 @@ type. Creates an instance of *type*, which must have been created with :c:func:`PyStructSequence_NewType`. + Return ``NULL`` with an exception set on failure. + .. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 0f58326f6c06b7..2176b8e492f306 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -53,7 +53,8 @@ Type Objects .. c:function:: PyObject* PyType_GetDict(PyTypeObject* type) Return the type object's internal namespace, which is otherwise only - exposed via a read-only proxy (``cls.__dict__``). This is a + exposed via a read-only proxy (:attr:`cls.__dict__ <type.__dict__>`). + This is a replacement for accessing :c:member:`~PyTypeObject.tp_dict` directly. The returned dictionary must be treated as read-only. @@ -81,6 +82,9 @@ Type Objects error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + In free-threaded builds, :c:func:`PyType_AddWatcher` is not thread-safe, + so it must be called at start up (before spawning the first thread). + .. versionadded:: 3.12 @@ -140,21 +144,36 @@ Type Objects Return true if *a* is a subtype of *b*. This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call + :meth:`~type.__subclasscheck__` is not called on *b*. Call :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` would do. .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to ``NULL``. + Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type + object. Uses Python's default memory allocation mechanism to allocate memory + for a new instance, zeros the memory, then initializes the memory as if by + calling :c:func:`PyObject_Init` or :c:func:`PyObject_InitVar`. + + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + For types that support garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set), this function behaves like + :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar` (except the + memory is guaranteed to be zeroed before initialization), and should be + paired with :c:func:`PyObject_GC_Del` in :c:member:`~PyTypeObject.tp_free`. + Otherwise, it behaves like :c:macro:`PyObject_New` or + :c:macro:`PyObject_NewVar` (except the memory is guaranteed to be zeroed + before initialization) and should be paired with :c:func:`PyObject_Free` in + :c:member:`~PyTypeObject.tp_free`. .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. + Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type + object. Creates a new instance using the type's + :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. .. c:function:: int PyType_Ready(PyTypeObject *type) @@ -174,17 +193,33 @@ Type Objects .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) - Return the type's name. Equivalent to getting the type's ``__name__`` attribute. + Return the type's name. Equivalent to getting the type's + :attr:`~type.__name__` attribute. .. versionadded:: 3.11 .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the - type's ``__qualname__`` attribute. + type's :attr:`~type.__qualname__` attribute. .. versionadded:: 3.11 +.. c:function:: PyObject* PyType_GetFullyQualifiedName(PyTypeObject *type) + + Return the type's fully qualified name. Equivalent to + ``f"{type.__module__}.{type.__qualname__}"``, or :attr:`type.__qualname__` + if :attr:`type.__module__` is not a string or is equal to ``"builtins"``. + + .. versionadded:: 3.13 + +.. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) + + Return the type's module name. Equivalent to getting the + :attr:`type.__module__` attribute. + + .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -249,6 +284,24 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) + + Find the first superclass in *type*'s :term:`method resolution order` whose + :c:macro:`Py_tp_token` token is equal to the given one. + + * If found, set *\*result* to a new :term:`strong reference` + to it and return ``1``. + * If not found, set *\*result* to ``NULL`` and return ``0``. + * On error, set *\*result* to ``NULL`` and return ``-1`` with an + exception set. + + The *result* argument may be ``NULL``, in which case *\*result* is not set. + Use this if you need only the return value. + + The *token* argument may not be ``NULL``. + + .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -276,10 +329,6 @@ The following functions and structs are used to create Metaclasses that override :c:member:`~PyTypeObject.tp_new` are not supported, except if ``tp_new`` is ``NULL``. - (For backwards compatibility, other ``PyType_From*`` functions allow - such metaclasses. They ignore ``tp_new``, which may result in incomplete - initialization. This is deprecated and in Python 3.14+ such metaclasses will - not be supported.) The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. @@ -330,8 +379,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) @@ -347,8 +400,12 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) @@ -363,8 +420,26 @@ The following functions and structs are used to create The :c:member:`~PyTypeObject.tp_new` of the metaclass is *ignored*. which may result in incomplete initialization. Creating classes whose metaclass overrides - :c:member:`~PyTypeObject.tp_new` is deprecated and in Python 3.14+ it - will be no longer allowed. + :c:member:`~PyTypeObject.tp_new` is deprecated. + + .. versionchanged:: 3.14 + + Creating classes whose metaclass overrides + :c:member:`~PyTypeObject.tp_new` is no longer allowed. + +.. c:function:: int PyType_Freeze(PyTypeObject *type) + + Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. + + All base classes of *type* must be immutable. + + On success, return ``0``. + On error, set an exception and return ``-1``. + + The type must not be used before it's made immutable. For example, type + instances must not be created before the type is made immutable. + + .. versionadded:: 3.14 .. raw:: html @@ -395,6 +470,9 @@ The following functions and structs are used to create class need *in addition* to the superclass. Use :c:func:`PyObject_GetTypeData` to get a pointer to subclass-specific memory reserved this way. + For negative :c:member:`!basicsize`, Python will insert padding when + needed to meet :c:member:`~PyTypeObject.tp_basicsize`'s alignment + requirements. .. versionchanged:: 3.12 @@ -461,39 +539,100 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - The following fields cannot be set at all using :c:type:`PyType_Spec` and - :c:type:`PyType_Slot`: + An additional slot is supported that does not correspond to a + :c:type:`!PyTypeObject` struct field: + + * :c:data:`Py_tp_token` + + The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - * :c:member:`~PyTypeObject.tp_dict` - * :c:member:`~PyTypeObject.tp_mro` - * :c:member:`~PyTypeObject.tp_cache` - * :c:member:`~PyTypeObject.tp_subclasses` - * :c:member:`~PyTypeObject.tp_weaklist` - * :c:member:`~PyTypeObject.tp_vectorcall` * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead) + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead) + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (see :ref:`PyMemberDef <pymemberdef-offsets>`) + (use ``"__vectorcalloffset__"`` in + :ref:`PyMemberDef <pymemberdef-offsets>`) + + If it is not possible to switch to a ``MANAGED`` flag (for example, + for vectorcall or to support Python older than 3.12), specify the + offset in :c:member:`Py_tp_members <PyTypeObject.tp_members>`. + See :ref:`PyMemberDef documentation <pymemberdef-offsets>` + for details. + + The following internal fields cannot be set at all when creating a heap + type: + + * :c:member:`~PyTypeObject.tp_dict`, + :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, + :c:member:`~PyTypeObject.tp_subclasses`, and + :c:member:`~PyTypeObject.tp_weaklist`. Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be problematic on some platforms. To avoid issues, use the *bases* argument of :c:func:`PyType_FromSpecWithBases` instead. - .. versionchanged:: 3.9 + .. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + .. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API <limited-c-api>`. - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API <limited-c-api>`. + .. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now set + using ``Py_tp_vectorcall``. See the field's documentation + for details. .. c:member:: void *pfunc The desired value of the slot. In most cases, this is a pointer to a function. - Slots other than ``Py_tp_doc`` may not be ``NULL``. + *pfunc* values may not be ``NULL``, except for the following slots: + + * ``Py_tp_doc`` + * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` + rather than ``NULL``) + +.. c:macro:: Py_tp_token + + A :c:member:`~PyType_Slot.slot` that records a static memory layout ID + for a class. + + If the :c:type:`PyType_Spec` of the class is statically + allocated, the token can be set to the spec using the special value + :c:data:`Py_TP_USE_SPEC`: + + .. code-block:: c + + static PyType_Slot foo_slots[] = { + {Py_tp_token, Py_TP_USE_SPEC}, + + It can also be set to an arbitrary pointer, but you must ensure that: + + * The pointer outlives the class, so it's not reused for something else + while the class exists. + * It "belongs" to the extension module where the class lives, so it will not + clash with other extensions. + + Use :c:func:`PyType_GetBaseByToken` to check if a class's superclass has + a given token -- that is, check whether the memory layout is compatible. + + To get the token for a given class (without considering superclasses), + use :c:func:`PyType_GetSlot` with ``Py_tp_token``. + + .. versionadded:: 3.14 + + .. c:namespace:: NULL + + .. c:macro:: Py_TP_USE_SPEC + + Used as a value with :c:data:`Py_tp_token` to set the token to the + class's :c:type:`PyType_Spec`. + Expands to ``NULL``. + + .. versionadded:: 3.14 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 221a05b1922404..91046c0e6f18ae 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -2,8 +2,8 @@ .. _type-structs: -Type Objects -============ +Type Object Structures +====================== Perhaps one of the most important structures of the Python object system is the structure that defines a new type: the :c:type:`PyTypeObject` structure. Type @@ -79,7 +79,7 @@ Quick Reference | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | | | | __delattr__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | :ref:`sub-slots` | | | | % | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -147,7 +147,7 @@ Quick Reference +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_watched`] | char | | | | | | + | [:c:member:`~PyTypeObject.tp_watched`] | unsigned char | | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ .. [#slots] @@ -325,9 +325,10 @@ sub-slots +---------------------------------------------------------+-----------------------------------+---------------+ | | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | + | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | __buffer__ | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | + | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | __release_\ | + | | | buffer\__ | +---------------------------------------------------------+-----------------------------------+---------------+ .. _slot-typedefs-table: @@ -343,19 +344,19 @@ slot typedefs | | :c:type:`PyTypeObject` * | | | | :c:type:`Py_ssize_t` | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`destructor` | void * | void | +| :c:type:`destructor` | :c:type:`PyObject` * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`freefunc` | void * | void | +-----------------------------+-----------------------------+----------------------+ | :c:type:`traverseproc` | .. line-block:: | int | | | | | -| | void * | | +| | :c:type:`PyObject` * | | | | :c:type:`visitproc` | | | | void * | | +-----------------------------+-----------------------------+----------------------+ | :c:type:`newfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | -| | :c:type:`PyObject` * | | +| | :c:type:`PyTypeObject` * | | | | :c:type:`PyObject` * | | | | :c:type:`PyObject` * | | +-----------------------------+-----------------------------+----------------------+ @@ -426,7 +427,7 @@ slot typedefs | | :c:type:`PyObject` * | | | | :c:type:`Py_buffer` * | | +-----------------------------+-----------------------------+----------------------+ -| :c:type:`inquiry` | void * | int | +| :c:type:`inquiry` | :c:type:`PyObject` * | int | +-----------------------------+-----------------------------+----------------------+ | :c:type:`unaryfunc` | .. line-block:: | :c:type:`PyObject` * | | | | | @@ -473,7 +474,7 @@ PyTypeObject Definition ----------------------- The structure definition for :c:type:`PyTypeObject` can be found in -:file:`Include/object.h`. For convenience of reference, this repeats the +:file:`Include/cpython/object.h`. For convenience of reference, this repeats the definition found there: .. XXX Drop this? @@ -528,28 +529,6 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is inherited by subtypes. -.. c:member:: PyObject* PyObject._ob_next - PyObject* PyObject._ob_prev - - These fields are only present when the macro ``Py_TRACE_REFS`` is defined - (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). - - Their initialization to ``NULL`` is taken care of by the - ``PyObject_HEAD_INIT`` macro. For :ref:`statically allocated objects - <static-types>`, these fields always remain ``NULL``. For :ref:`dynamically - allocated objects <heap-types>`, these two fields are used to link the - object into a doubly linked list of *all* live objects on the heap. - - This could be used for various debugging purposes; currently the only uses - are the :func:`sys.getobjects` function and to print the objects that are - still alive at the end of a run when the environment variable - :envvar:`PYTHONDUMPREFS` is set. - - **Inheritance:** - - These fields are not inherited by subtypes. - - PyVarObject Slots ----------------- @@ -559,6 +538,9 @@ PyVarObject Slots initialized to zero. For :ref:`dynamically allocated type objects <heap-types>`, this field has a special internal meaning. + This field should be accessed using the :c:func:`Py_SIZE()` and + :c:func:`Py_SET_SIZE()` macros. + **Inheritance:** This field is not inherited by subtypes. @@ -589,12 +571,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) For :ref:`statically allocated type objects <static-types>`, the *tp_name* field should contain a dot. - Everything before the last dot is made accessible as the :attr:`__module__` + Everything before the last dot is made accessible as the :attr:`~type.__module__` attribute, and everything after the last dot is made accessible as the - :attr:`~definition.__name__` attribute. + :attr:`~type.__name__` attribute. If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the - :attr:`~definition.__name__` attribute, and the :attr:`__module__` attribute is undefined + :attr:`~type.__name__` attribute, and the :attr:`~type.__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. Additionally, it will not be listed in module documentations created with pydoc. @@ -609,106 +591,208 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: Py_ssize_t PyTypeObject.tp_basicsize - Py_ssize_t PyTypeObject.tp_itemsize + Py_ssize_t PyTypeObject.tp_itemsize These fields allow calculating the size in bytes of instances of the type. There are two kinds of types: types with fixed-length instances have a zero - :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero - :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. + :c:member:`!tp_itemsize` field, types with variable-length instances have a non-zero + :c:member:`!tp_itemsize` field. For a type with fixed-length instances, all + instances have the same size, given in :c:member:`!tp_basicsize`. + (Exceptions to this rule can be made using + :c:func:`PyUnstable_Object_GC_NewWithExtraData`.) For a type with variable-length instances, the instances must have an - :c:member:`~PyVarObject.ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N - times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of - N is typically stored in the instance's :c:member:`~PyVarObject.ob_size` field. There are - exceptions: for example, ints use a negative :c:member:`~PyVarObject.ob_size` to indicate a - negative number, and N is ``abs(ob_size)`` there. Also, the presence of an - :c:member:`~PyVarObject.ob_size` field in the instance layout doesn't mean that the instance - structure is variable-length (for example, the structure for the list type has - fixed-length instances, yet those instances have a meaningful :c:member:`~PyVarObject.ob_size` - field). - - The basic size includes the fields in the instance declared by the macro - :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to - declare the instance struct) and this in turn includes the :c:member:`~PyObject._ob_prev` and - :c:member:`~PyObject._ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the - ``sizeof`` operator on the struct used to declare the instance layout. - The basic size does not include the GC header size. + :c:member:`~PyVarObject.ob_size` field, and the instance size is + :c:member:`!tp_basicsize` plus N times :c:member:`!tp_itemsize`, + where N is the "length" of the object. + + Functions like :c:func:`PyObject_NewVar` will take the value of N as an + argument, and store in the instance's :c:member:`~PyVarObject.ob_size` field. + Note that the :c:member:`~PyVarObject.ob_size` field may later be used for + other purposes. For example, :py:type:`int` instances use the bits of + :c:member:`~PyVarObject.ob_size` in an implementation-defined + way; the underlying storage and its size should be accessed using + :c:func:`PyLong_Export`. - A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: - suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is - ``sizeof(double)``. It is the programmer's responsibility that - :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the - alignment requirement for ``double``). + .. note:: + + The :c:member:`~PyVarObject.ob_size` field should be accessed using + the :c:func:`Py_SIZE()` and :c:func:`Py_SET_SIZE()` macros. - For any type with variable-length instances, this field must not be ``NULL``. + Also, the presence of an :c:member:`~PyVarObject.ob_size` field in the + instance layout doesn't mean that the instance structure is variable-length. + For example, the :py:type:`list` type has fixed-length instances, yet those + instances have a :c:member:`~PyVarObject.ob_size` field. + (As with :py:type:`int`, avoid reading lists' :c:member:`!ob_size` directly. + Call :c:func:`PyList_Size` instead.) + + The :c:member:`!tp_basicsize` includes size needed for data of the type's + :c:member:`~PyTypeObject.tp_base`, plus any extra data needed + by each instance. + + The correct way to set :c:member:`!tp_basicsize` is to use the + ``sizeof`` operator on the struct used to declare the instance layout. + This struct must include the struct used to declare the base type. + In other words, :c:member:`!tp_basicsize` must be greater than or equal + to the base's :c:member:`!tp_basicsize`. + + Since every type is a subtype of :py:type:`object`, this struct must + include :c:type:`PyObject` or :c:type:`PyVarObject` (depending on + whether :c:member:`~PyVarObject.ob_size` should be included). These are + usually defined by the macro :c:macro:`PyObject_HEAD` or + :c:macro:`PyObject_VAR_HEAD`, respectively. + + The basic size does not include the GC header size, as that header is not + part of :c:macro:`PyObject_HEAD`. + + For cases where struct used to declare the base type is unknown, + see :c:member:`PyType_Spec.basicsize` and :c:func:`PyType_FromMetaclass`. + + Notes about alignment: + + - :c:member:`!tp_basicsize` must be a multiple of ``_Alignof(PyObject)``. + When using ``sizeof`` on a ``struct`` that includes + :c:macro:`PyObject_HEAD`, as recommended, the compiler ensures this. + When not using a C ``struct``, or when using compiler + extensions like ``__attribute__((packed))``, it is up to you. + - If the variable items require a particular alignment, + :c:member:`!tp_basicsize` and :c:member:`!tp_itemsize` must each be a + multiple of that alignment. + For example, if a type's variable part stores a ``double``, it is + your responsibility that both fields are a multiple of + ``_Alignof(double)``. **Inheritance:** - These fields are inherited separately by subtypes. If the base type has a - non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set + These fields are inherited separately by subtypes. + (That is, if the field is set to zero, :c:func:`PyType_Ready` will copy + the value from the base type, indicating that the instances do not + need additional storage.) + + If the base type has a non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this depends on the implementation of the base type). .. c:member:: destructor PyTypeObject.tp_dealloc - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). The function signature is:: + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and call the type's - :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Del` if the instance was allocated - using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. - - If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` - flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` + The destructor function should remove all references which the instance owns + (e.g., call :c:func:`Py_CLEAR`), free all memory buffers owned by the + instance, and call the type's :c:member:`~PyTypeObject.tp_free` function to + free the object itself. + + No guarantees are made about when an object is destroyed, except: + + * Python will destroy an object immediately or some time after the final + reference to the object is deleted, unless its finalizer + (:c:member:`~PyTypeObject.tp_finalize`) subsequently resurrects the + object. + * An object will not be destroyed while it is being automatically finalized + (:c:member:`~PyTypeObject.tp_finalize`) or automatically cleared + (:c:member:`~PyTypeObject.tp_clear`). + + CPython currently destroys an object immediately from :c:func:`Py_DECREF` + when the new reference count is zero, but this may change in a future + version. + + It is recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is always + finalized before destruction. + + If the type supports garbage collection (the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set), the destructor should call :c:func:`PyObject_GC_UnTrack` before clearing any member fields. - .. code-block:: c + It is permissible to call :c:member:`~PyTypeObject.tp_clear` from + :c:member:`!tp_dealloc` to reduce code duplication and to guarantee that the + object is always cleared before destruction. Beware that + :c:member:`!tp_clear` might have already been called. + + If the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should release the owned reference to its type object (via + :c:func:`Py_DECREF`) after calling the type deallocator. See the example + code below.:: - static void foo_dealloc(foo_object *self) { + static void + foo_dealloc(PyObject *op) + { + foo_object *self = (foo_object *) op; PyObject_GC_UnTrack(self); Py_CLEAR(self->ref); - Py_TYPE(self)->tp_free((PyObject *)self); - } + Py_TYPE(self)->tp_free(self); + } + + :c:member:`!tp_dealloc` must leave the exception status unchanged. If it + needs to call something that might raise an exception, the exception state + must be backed up first and restored later (after logging any exceptions + with :c:func:`PyErr_WriteUnraisable`). + + Example:: + + static void + foo_dealloc(PyObject *self) + { + PyObject *exc = PyErr_GetRaisedException(); + + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // self was resurrected. + goto done; + } + + PyTypeObject *tp = Py_TYPE(self); + + if (tp->tp_flags & Py_TPFLAGS_HAVE_GC) { + PyObject_GC_UnTrack(self); + } - Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should release the owned reference to its type object - (via :c:func:`Py_DECREF`) after - calling the type deallocator. In order to avoid dangling pointers, the - recommended way to achieve this is: + // Optional, but convenient to avoid code duplication. + if (tp->tp_clear && tp->tp_clear(self) < 0) { + PyErr_WriteUnraisable(self); + } - .. code-block:: c + // Any additional destruction goes here. - static void foo_dealloc(foo_object *self) { - PyTypeObject *tp = Py_TYPE(self); - // free references and buffers here - tp->tp_free(self); - Py_DECREF(tp); - } + tp->tp_free(self); + self = NULL; // In case PyErr_WriteUnraisable() is called below. + + if (tp->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_CLEAR(tp); + } + + done: + // Optional, if something was called that might have raised an + // exception. + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + PyErr_SetRaisedException(exc); + } + + :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + with an :term:`attached thread state`. However, if the object being + destroyed in turn destroys objects from some other C library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. **Inheritance:** This field is inherited by subtypes. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset @@ -905,6 +989,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. + **Default:** + + :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericHash`. + .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -1028,6 +1116,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` values. + .. XXX are most flag bits *really* inherited individually? **Default:** @@ -1052,7 +1141,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) the type, and the type object is INCREF'ed when a new instance is created, and DECREF'ed when an instance is destroyed (this does not apply to instances of subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). + DECREF'ed). Heap types should also :ref:`support garbage collection <supporting-cycle-detection>` + as they can form a reference cycle with their own module object. **Inheritance:** @@ -1093,11 +1183,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_GC This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:macro:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. + is set, memory for new instances (see :c:member:`~PyTypeObject.tp_alloc`) + must be allocated using :c:macro:`PyObject_GC_New` or + :c:func:`PyType_GenericAlloc` and deallocated (see + :c:member:`~PyTypeObject.tp_free`) using :c:func:`PyObject_GC_Del`. More + information in section :ref:`supporting-cycle-detection`. **Inheritance:** @@ -1148,11 +1238,14 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a ``__dict__`` + This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. + The type traverse function must call :c:func:`PyObject_VisitManagedDict` + and its clear function must call :c:func:`PyObject_ClearManagedDict`. + .. versionadded:: 3.12 **Inheritance:** @@ -1342,15 +1435,15 @@ and :c:data:`PyType_Type` effectively act as defaults.) To indicate that a class has changed call :c:func:`PyType_Modified` .. warning:: - This flag is present in header files, but is an internal feature and should - not be used. It will be removed in a future version of CPython + This flag is present in header files, but is not be used. + It will be removed in a future version of CPython .. c:member:: const char* PyTypeObject.tp_doc An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`__doc__` attribute on the type and - instances of the type. + type object. This is exposed as the :attr:`~type.__doc__` attribute on the + type and instances of the type. **Inheritance:** @@ -1374,8 +1467,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) :mod:`!_thread` extension module:: static int - local_traverse(localobject *self, visitproc visit, void *arg) + local_traverse(PyObject *op, visitproc visit, void *arg) { + localobject *self = (localobject *) op; Py_VISIT(self->args); Py_VISIT(self->kw); Py_VISIT(self->dict); @@ -1390,6 +1484,23 @@ and :c:data:`PyType_Type` effectively act as defaults.) debugging aid you may want to visit it anyway just so the :mod:`gc` module's :func:`~gc.get_referents` function will include it. + Heap types (:c:macro:`Py_TPFLAGS_HEAPTYPE`) must visit their type with:: + + Py_VISIT(Py_TYPE(self)); + + It is only needed since Python 3.9. To support Python 3.8 and older, this + line must be conditional:: + + #if PY_VERSION_HEX >= 0x03090000 + Py_VISIT(Py_TYPE(self)); + #endif + + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_VisitManagedDict` like this:: + + PyObject_VisitManagedDict((PyObject*)self, visit, arg); + .. warning:: When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members that the instance *owns* (by having :term:`strong references @@ -1403,7 +1514,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) are allowed to be removed even if the instance is still alive). Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`local_traverse` to have these specific names; don't name them just + :c:func:`!local_traverse` to have these specific names; don't name them just anything. Instances of :ref:`heap-allocated types <heap-types>` hold a reference to @@ -1413,6 +1524,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) heap-allocated superclass). If they do not, the type object may not be garbage-collected. + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + .. versionchanged:: 3.9 Heap-allocated types are expected to visit ``Py_TYPE(self)`` in @@ -1432,28 +1548,110 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear - An optional pointer to a clear function for the garbage collector. This is only - used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. + The purpose of this function is to break reference cycles that are causing a + :term:`cyclic isolate` so that the objects can be safely destroyed. A + cleared object is a partially destroyed object; the object is not obligated + to satisfy design invariants held during normal use. + + :c:member:`!tp_clear` does not need to delete references to objects that + can't participate in reference cycles, such as Python strings or Python + integers. However, it may be convenient to clear all references, and write + the type's :c:member:`~PyTypeObject.tp_dealloc` function to invoke + :c:member:`!tp_clear` to avoid code duplication. (Beware that + :c:member:`!tp_clear` might have already been called. Prefer calling + idempotent functions like :c:func:`Py_CLEAR`.) + + Any non-trivial cleanup should be performed in + :c:member:`~PyTypeObject.tp_finalize` instead of :c:member:`!tp_clear`. + + .. note:: + + If :c:member:`!tp_clear` fails to break a reference cycle then the + objects in the :term:`cyclic isolate` may remain indefinitely + uncollectable ("leak"). See :data:`gc.garbage`. + + .. note:: + + Referents (direct and indirect) might have already been cleared; they are + not guaranteed to be in a consistent state. + + .. note:: + + The :c:member:`~PyTypeObject.tp_clear` function can be called from any + thread. + + .. note:: + + An object is not guaranteed to be automatically cleared before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. + + This function differs from the destructor + (:c:member:`~PyTypeObject.tp_dealloc`) in the following ways: + + * The purpose of clearing an object is to remove references to other objects + that might participate in a reference cycle. The purpose of the + destructor, on the other hand, is a superset: it must release *all* + resources it owns, including references to objects that cannot participate + in a reference cycle (e.g., integers) as well as the object's own memory + (by calling :c:member:`~PyTypeObject.tp_free`). + * When :c:member:`!tp_clear` is called, other objects might still hold + references to the object being cleared. Because of this, + :c:member:`!tp_clear` must not deallocate the object's own memory + (:c:member:`~PyTypeObject.tp_free`). The destructor, on the other hand, + is only called when no (strong) references exist, and as such, must + safely destroy the object itself by deallocating it. + * :c:member:`!tp_clear` might never be automatically called. An object's + destructor, on the other hand, will be automatically called some time + after the object becomes unreachable (i.e., either there are no references + to the object or the object is a member of a :term:`cyclic isolate`). + + No guarantees are made about when, if, or how often Python automatically + clears an object, except: + + * Python will not automatically clear an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically clear an object if it has not been + automatically finalized (see :c:member:`~PyTypeObject.tp_finalize`). (If + the finalizer resurrected the object, the object may or may not be + automatically finalized again before it is cleared.) + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically clear it if any member of the cyclic isolate has not yet + been automatically finalized (:c:member:`~PyTypeObject.tp_finalize`). + * Python will not destroy an object until after any automatic calls to its + :c:member:`!tp_clear` function have returned. This ensures that the act + of breaking a reference cycle does not invalidate the ``self`` pointer + while :c:member:`!tp_clear` is still executing. + * Python will not automatically call :c:member:`!tp_clear` multiple times + concurrently. + + CPython currently only automatically clears objects as needed to break + reference cycles in a :term:`cyclic isolate`, but future versions might + clear objects regularly before their destruction. + + Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the + system must combine to break all reference cycles. This is subtle, and if + in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For + example, the tuple type does not implement a + :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove + that no reference cycle can be composed entirely of tuples. Therefore the + :c:member:`~PyTypeObject.tp_clear` functions of other types are responsible + for breaking any cycle containing a tuple. This isn't immediately obvious, + and there's rarely a good reason to avoid implementing + :c:member:`~PyTypeObject.tp_clear`. Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those members to ``NULL``, as in the following example:: static int - local_clear(localobject *self) + local_clear(PyObject *op) { + localobject *self = (localobject *) op; Py_CLEAR(self->key); Py_CLEAR(self->args); Py_CLEAR(self->kw); @@ -1473,17 +1671,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. - Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called - before an instance is deallocated. For example, when reference counting - is enough to determine that an object is no longer used, the cyclic garbage - collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is - called directly. + If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the + :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:func:`PyObject_ClearManagedDict` like this:: - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. + PyObject_ClearManagedDict((PyObject*)self); More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. @@ -1497,6 +1689,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: richcmpfunc PyTypeObject.tp_richcompare @@ -1583,7 +1779,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) weak references to the type object itself. It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and - :c:member:`~PyTypeObject.tp_weaklist`. + :c:member:`~PyTypeObject.tp_weaklistoffset`. **Inheritance:** @@ -1595,7 +1791,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Default:** If the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_weaklistoffset` will be set to a negative value, to indicate that it is unsafe to use this field. @@ -1697,7 +1893,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) to a pointer, are valid C99 address constants. However, the unary '&' operator applied to a non-static variable - like :c:func:`PyBaseObject_Type` is not required to produce an address + like :c:data:`PyBaseObject_Type` is not required to produce an address constant. Compilers may support this (gcc does), MSVC does not. Both compilers are strictly standard conforming in this particular behavior. @@ -1728,7 +1924,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) treated as read-only. Some types may not store their dictionary in this slot. - Use :c:func:`PyType_GetDict` to retreive the dictionary for an arbitrary + Use :c:func:`PyType_GetDict` to retrieve the dictionary for an arbitrary type. .. versionchanged:: 3.12 @@ -1807,7 +2003,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) dictionary, so it is may be more efficient to call :c:func:`PyObject_GetAttr` when accessing an attribute on the object. - It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` bit and + It is an error to set both the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit and :c:member:`~PyTypeObject.tp_dictoffset`. **Inheritance:** @@ -1823,7 +2019,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) field is ``NULL`` then no :attr:`~object.__dict__` gets created for instances. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_dict` field, then + :c:member:`~PyTypeObject.tp_flags` field, then :c:member:`~PyTypeObject.tp_dictoffset` will be set to ``-1``, to indicate that it is unsafe to use this field. @@ -1873,18 +2069,17 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement). + Static subtypes inherit this slot, which will be + :c:func:`PyType_GenericAlloc` if inherited from :class:`object`. + + :ref:`Heap subtypes <heap-types>` do not inherit this slot. **Default:** - For dynamic subtypes, this field is always set to - :c:func:`PyType_GenericAlloc`, to force a standard heap - allocation strategy. + For heap subtypes, this field is always set to + :c:func:`PyType_GenericAlloc`. - For static subtypes, :c:data:`PyBaseObject_Type` uses - :c:func:`PyType_GenericAlloc`. That is the recommended value - for all statically defined types. + For static subtypes, this slot is inherited (see above). .. c:member:: newfunc PyTypeObject.tp_new @@ -1932,20 +2127,27 @@ and :c:data:`PyType_Type` effectively act as defaults.) void tp_free(void *self); - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. + This function must free the memory allocated by + :c:member:`~PyTypeObject.tp_alloc`. **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement) + Static subtypes inherit this slot, which will be :c:func:`PyObject_Free` if + inherited from :class:`object`. Exception: If the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and it would inherit + :c:func:`PyObject_Free`, then this slot is not inherited but instead defaults + to :c:func:`PyObject_GC_Del`. + + :ref:`Heap subtypes <heap-types>` do not inherit this slot. **Default:** - In dynamic subtypes, this field is set to a deallocator suitable to - match :c:func:`PyType_GenericAlloc` and the value of the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. + For :ref:`heap subtypes <heap-types>`, this slot defaults to a deallocator suitable to match + :c:func:`PyType_GenericAlloc` and the value of the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Del`. + For static subtypes, this slot is inherited (see above). .. c:member:: inquiry PyTypeObject.tp_is_gc @@ -2027,7 +2229,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) A collection of subclasses. Internal use only. May be an invalid pointer. To get a list of subclasses, call the Python method - :py:meth:`~class.__subclasses__`. + :py:meth:`~type.__subclasses__`. .. versionchanged:: 3.12 @@ -2072,45 +2274,141 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize - An optional pointer to an instance finalization function. Its signature is:: + An optional pointer to an instance finalization function. This is the C + implementation of the :meth:`~object.__del__` special method. Its signature + is:: void tp_finalize(PyObject *self); - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. + The primary purpose of finalization is to perform any non-trivial cleanup + that must be performed before the object is destroyed, while the object and + any other objects it directly or indirectly references are still in a + consistent state. The finalizer is allowed to execute + arbitrary Python code. + + Before Python automatically finalizes an object, some of the object's direct + or indirect referents might have themselves been automatically finalized. + However, none of the referents will have been automatically cleared + (:c:member:`~PyTypeObject.tp_clear`) yet. - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: + Other non-finalized objects might still be using a finalized object, so the + finalizer must leave the object in a sane state (e.g., invariants are still + met). + + .. note:: + + After Python automatically finalizes an object, Python might start + automatically clearing (:c:member:`~PyTypeObject.tp_clear`) the object + and its referents (direct and indirect). Cleared objects are not + guaranteed to be in a consistent state; a finalized object must be able + to tolerate cleared referents. + + .. note:: + + An object is not guaranteed to be automatically finalized before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. It is + recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is + always finalized before destruction. + + .. note:: + + The :c:member:`~PyTypeObject.tp_finalize` function can be called from any + thread, although the :term:`GIL` will be held. + + .. note:: + + The :c:member:`!tp_finalize` function can be called during shutdown, + after some global variables have been deleted. See the documentation of + the :meth:`~object.__del__` method for details. + + When Python finalizes an object, it behaves like the following algorithm: + + #. Python might mark the object as *finalized*. Currently, Python always + marks objects whose type supports garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and never marks other types of + objects; this might change in a future version. + #. If the object is not marked as *finalized* and its + :c:member:`!tp_finalize` finalizer function is non-``NULL``, the + finalizer function is called. + #. If the finalizer function was called and the finalizer made the object + reachable (i.e., there is a reference to the object and it is not a + member of a :term:`cyclic isolate`), then the finalizer is said to have + *resurrected* the object. It is unspecified whether the finalizer can + also resurrect the object by adding a new reference to the object that + does not make it reachable, i.e., the object is (still) a member of a + cyclic isolate. + #. If the finalizer resurrected the object, the object's pending destruction + is canceled and the object's *finalized* mark might be removed if + present. Currently, Python never removes the *finalized* mark; this + might change in a future version. + + *Automatic finalization* refers to any finalization performed by Python + except via calls to :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. No guarantees are made about + when, if, or how often an object is automatically finalized, except: + + * Python will not automatically finalize an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically finalize an object if finalizing it would + not mark the object as *finalized*. Currently, this applies to objects + whose type does not support garbage collection, i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is not set. Such objects can still be + manually finalized by calling :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. + * Python will not automatically finalize any two members of a :term:`cyclic + isolate` concurrently. + * Python will not automatically finalize an object after it has + automatically cleared (:c:member:`~PyTypeObject.tp_clear`) the object. + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically finalize it after automatically clearing (see + :c:member:`~PyTypeObject.tp_clear`) any other member. + * Python will automatically finalize every member of a :term:`cyclic + isolate` before it automatically clears (see + :c:member:`~PyTypeObject.tp_clear`) any of them. + * If Python is going to automatically clear an object + (:c:member:`~PyTypeObject.tp_clear`), it will automatically finalize the + object first. + + Python currently only automatically finalizes objects that are members of a + :term:`cyclic isolate`, but future versions might finalize objects regularly + before their destruction. + + To manually finalize an object, do not call this function directly; call + :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc` instead. + + :c:member:`~PyTypeObject.tp_finalize` should leave the current exception + status unchanged. The recommended way to write a non-trivial finalizer is + to back up the exception at the beginning by calling + :c:func:`PyErr_GetRaisedException` and restore the exception at the end by + calling :c:func:`PyErr_SetRaisedException`. If an exception is encountered + in the middle of the finalizer, log and clear it with + :c:func:`PyErr_WriteUnraisable` or :c:func:`PyErr_FormatUnraisable`. For + example:: static void - local_finalize(PyObject *self) + foo_finalize(PyObject *self) { - PyObject *error_type, *error_value, *error_traceback; + // Save the current exception, if any. + PyObject *exc = PyErr_GetRaisedException(); - /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); + // ... - /* ... */ + if (do_something_that_might_raise() != success_indicator) { + PyErr_WriteUnraisable(self); + goto done; + } - /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); + done: + // Restore the saved exception. This silently discards any exception + // raised above, so be sure to call PyErr_WriteUnraisable first if + // necessary. + PyErr_SetRaisedException(exc); } - Also, note that, in a garbage collected Python, - :c:member:`~PyTypeObject.tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the object - becomes part of a refcount cycle, that cycle might be collected by a garbage - collection on any thread). This is not a problem for Python API calls, since - the thread on which tp_dealloc is called will own the Global Interpreter Lock - (GIL). However, if the object being destroyed in turn destroys objects from some - other C or C++ library, care should be taken to ensure that destroying those - objects on the thread which called tp_dealloc will not violate any assumptions - of the library. - **Inheritance:** This field is inherited by subtypes. @@ -2123,16 +2421,51 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be used. This is no longer required. - .. seealso:: "Safe object finalization" (:pep:`442`) + .. seealso:: + + * :pep:`442`: "Safe object finalization" + * :ref:`life-cycle` for details about how this slot relates to other + slots. + * :c:func:`PyObject_CallFinalizer` + * :c:func:`PyObject_CallFinalizerFromDealloc` .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall - Vectorcall function to use for calls of this type object. - In other words, it is used to implement - :ref:`vectorcall <vectorcall>` for ``type.__call__``. - If ``tp_vectorcall`` is ``NULL``, the default call implementation - using :meth:`~object.__new__` and :meth:`~object.__init__` is used. + A :ref:`vectorcall function <vectorcall>` to use for calls of this type + object (rather than instances). + In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, + which typically returns a new instance of *type*. + + As with any vectorcall function, if ``tp_vectorcall`` is ``NULL``, + the *tp_call* protocol (``Py_TYPE(type)->tp_call``) is used instead. + + .. note:: + + The :ref:`vectorcall protocol <vectorcall>` requires that the vectorcall + function has the same behavior as the corresponding ``tp_call``. + This means that ``type->tp_vectorcall`` must match the behavior of + ``Py_TYPE(type)->tp_call``. + + Specifically, if *type* uses the default metaclass, + ``type->tp_vectorcall`` must behave the same as + :c:expr:`PyType_Type->tp_call`, which: + + - calls ``type->tp_new``, + + - if the result is a subclass of *type*, calls ``type->tp_init`` + on the result of ``tp_new``, and + + - returns the result of ``tp_new``. + + Typically, ``tp_vectorcall`` is overridden to optimize this process + for specific :c:member:`~PyTypeObject.tp_new` and + :c:member:`~PyTypeObject.tp_init`. + When doing this for user-subclassable types, note that both can be + overridden (using :py:func:`~object.__new__` and + :py:func:`~object.__init__`, respectively). + + **Inheritance:** @@ -2141,7 +2474,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) -.. c:member:: char PyTypeObject.tp_watched +.. c:member:: unsigned char PyTypeObject.tp_watched Internal. Do not use. @@ -2190,7 +2523,7 @@ This is done by filling a :c:type:`PyType_Spec` structure and calling .. _number-structs: Number Object Structures -======================== +------------------------ .. sectionauthor:: Amaury Forgeot d'Arc @@ -2304,7 +2637,7 @@ Number Object Structures .. _mapping-structs: Mapping Object Structures -========================= +------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2341,7 +2674,7 @@ Mapping Object Structures .. _sequence-structs: Sequence Object Structures -========================== +-------------------------- .. sectionauthor:: Amaury Forgeot d'Arc @@ -2421,7 +2754,7 @@ Sequence Object Structures .. _buffer-structs: Buffer Object Structures -======================== +------------------------ .. sectionauthor:: Greg J. Stein <greg@lyra.org> .. sectionauthor:: Benjamin Peterson @@ -2443,7 +2776,7 @@ Buffer Object Structures Except for point (3), an implementation of this function MUST take these steps: - (1) Check if the request can be met. If not, raise :c:data:`PyExc_BufferError`, + (1) Check if the request can be met. If not, raise :exc:`BufferError`, set :c:expr:`view->obj` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. @@ -2516,7 +2849,7 @@ Buffer Object Structures Async Object Structures -======================= +----------------------- .. sectionauthor:: Yury Selivanov <yselivanov@sprymix.com> @@ -2584,7 +2917,7 @@ Async Object Structures .. _slot-typedefs: Slot Type typedefs -================== +------------------ .. c:type:: PyObject *(*allocfunc)(PyTypeObject *cls, Py_ssize_t nitems) @@ -2607,7 +2940,7 @@ Slot Type typedefs See :c:member:`~PyTypeObject.tp_free`. -.. c:type:: PyObject *(*newfunc)(PyObject *, PyObject *, PyObject *) +.. c:type:: PyObject *(*newfunc)(PyTypeObject *, PyObject *, PyObject *) See :c:member:`~PyTypeObject.tp_new`. @@ -2693,7 +3026,7 @@ Slot Type typedefs .. _typedef-examples: Examples -======== +-------- The following are simple examples of Python type definitions. They include common usage you may encounter. Some demonstrate tricky corner diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index cf965fa1676524..45f50ba5f97d26 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -31,6 +31,18 @@ Unicode Type These are the basic Unicode object types used for the Unicode implementation in Python: +.. c:var:: PyTypeObject PyUnicode_Type + + This instance of :c:type:`PyTypeObject` represents the Python Unicode type. + It is exposed to Python code as :py:class:`str`. + + +.. c:var:: PyTypeObject PyUnicodeIter_Type + + This instance of :c:type:`PyTypeObject` represents the Python Unicode + iterator type. It is used to iterate over Unicode string objects. + + .. c:type:: Py_UCS4 Py_UCS2 Py_UCS1 @@ -42,19 +54,6 @@ Python: .. versionadded:: 3.3 -.. c:type:: Py_UNICODE - - This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type - depending on the platform. - - .. versionchanged:: 3.3 - In previous versions, this was a 16-bit type or a 32-bit type depending on - whether you selected a "narrow" or "wide" Unicode version of Python at - build time. - - .. deprecated-removed:: 3.13 3.15 - - .. c:type:: PyASCIIObject PyCompactUnicodeObject PyUnicodeObject @@ -66,48 +65,32 @@ Python: .. versionadded:: 3.3 -.. c:var:: PyTypeObject PyUnicode_Type - - This instance of :c:type:`PyTypeObject` represents the Python Unicode type. It - is exposed to Python code as ``str``. - - The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: -.. c:function:: int PyUnicode_Check(PyObject *o) +.. c:function:: int PyUnicode_Check(PyObject *obj) - Return true if the object *o* is a Unicode object or an instance of a Unicode + Return true if the object *obj* is a Unicode object or an instance of a Unicode subtype. This function always succeeds. -.. c:function:: int PyUnicode_CheckExact(PyObject *o) +.. c:function:: int PyUnicode_CheckExact(PyObject *obj) - Return true if the object *o* is a Unicode object, but not an instance of a + Return true if the object *obj* is a Unicode object, but not an instance of a subtype. This function always succeeds. -.. c:function:: int PyUnicode_READY(PyObject *o) - - Returns ``0``. This API is kept only for backward compatibility. - - .. versionadded:: 3.3 - - .. deprecated:: 3.10 - This API does nothing since Python 3.12. - - -.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) +.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *unicode) - Return the length of the Unicode string, in code points. *o* has to be a + Return the length of the Unicode string, in code points. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o) - Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o) - Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o) +.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *unicode) + Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *unicode) + Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *unicode) Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4 integer types for direct character access. No checks are performed if the @@ -129,18 +112,18 @@ access to internal read-only data of Unicode objects: ``PyUnicode_WCHAR_KIND`` has been removed. -.. c:function:: int PyUnicode_KIND(PyObject *o) +.. c:function:: int PyUnicode_KIND(PyObject *unicode) Return one of the PyUnicode kind constants (see above) that indicate how many - bytes per character this Unicode object uses to store its data. *o* has to + bytes per character this Unicode object uses to store its data. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 -.. c:function:: void* PyUnicode_DATA(PyObject *o) +.. c:function:: void* PyUnicode_DATA(PyObject *unicode) - Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode + Return a void pointer to the raw Unicode buffer. *unicode* has to be a Unicode object in the "canonical" representation (not checked). .. versionadded:: 3.3 @@ -149,12 +132,16 @@ access to internal read-only data of Unicode objects: .. c:function:: void PyUnicode_WRITE(int kind, void *data, \ Py_ssize_t index, Py_UCS4 value) - Write into a canonical representation *data* (as obtained with - :c:func:`PyUnicode_DATA`). This function performs no sanity checks, and is - intended for usage in loops. The caller should cache the *kind* value and - *data* pointer as obtained from other calls. *index* is the index in - the string (starts at 0) and *value* is the new code point value which should - be written to that location. + Write the code point *value* to the given zero-based *index* in a string. + + The *kind* value and *data* pointer must have been obtained from a + string using :c:func:`PyUnicode_KIND` and :c:func:`PyUnicode_DATA` + respectively. You must hold a reference to that string while calling + :c:func:`!PyUnicode_WRITE`. All requirements of + :c:func:`PyUnicode_WriteChar` also apply. + + The function performs no checks for any of its requirements, + and is intended for usage in loops. .. versionadded:: 3.3 @@ -168,25 +155,25 @@ access to internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index) +.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *unicode, Py_ssize_t index) - Read a character from a Unicode object *o*, which must be in the "canonical" + Read a character from a Unicode object *unicode*, which must be in the "canonical" representation. This is less efficient than :c:func:`PyUnicode_READ` if you do multiple consecutive reads. .. versionadded:: 3.3 -.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *unicode) Return the maximum code point that is suitable for creating another string - based on *o*, which must be in the "canonical" representation. This is + based on *unicode*, which must be in the "canonical" representation. This is always an approximation but more efficient than iterating over the string. .. versionadded:: 3.3 -.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) +.. c:function:: int PyUnicode_IsIdentifier(PyObject *unicode) Return ``1`` if the string is a valid identifier according to the language definition, section :ref:`identifiers`. Return ``0`` otherwise. @@ -196,6 +183,14 @@ access to internal read-only data of Unicode objects: is not ready. +.. c:function:: unsigned int PyUnicode_IS_ASCII(PyObject *unicode) + + Return true if the string only contains ASCII characters. + Equivalent to :py:meth:`str.isascii`. + + .. versionadded:: 3.2 + + Unicode Character Properties """""""""""""""""""""""""""" @@ -256,13 +251,8 @@ the Python configuration. .. c:function:: int Py_UNICODE_ISPRINTABLE(Py_UCS4 ch) - Return ``1`` or ``0`` depending on whether *ch* is a printable character. - Nonprintable characters are those characters defined in the Unicode character - database as "Other" or "Separator", excepting the ASCII space (0x20) which is - considered printable. (Note that printable characters in this context are - those which should not be escaped when :func:`repr` is invoked on a string. - It has no bearing on the handling of strings written to :data:`sys.stdout` or - :data:`sys.stderr`.) + Return ``1`` or ``0`` depending on whether *ch* is a printable character, + in the sense of :meth:`str.isprintable`. These APIs can be used for fast direct character conversions: @@ -317,7 +307,7 @@ These APIs can be used to work with surrogates: .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) - Join two surrogate characters and return a single :c:type:`Py_UCS4` value. + Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must be in the range [0xDC00; 0xDFFF]. @@ -335,8 +325,28 @@ APIs: to be placed in the string. As an approximation, it can be rounded up to the nearest value in the sequence 127, 255, 65535, 1114111. - This is the recommended way to allocate a new Unicode object. Objects - created using this function are not resizable. + On error, set an exception and return ``NULL``. + + After creation, the string can be filled by :c:func:`PyUnicode_WriteChar`, + :c:func:`PyUnicode_CopyCharacters`, :c:func:`PyUnicode_Fill`, + :c:func:`PyUnicode_WRITE` or similar. + Since strings are supposed to be immutable, take care to not “use” the + result while it is being modified. In particular, before it's filled + with its final contents, a string: + + - must not be hashed, + - must not be :c:func:`converted to UTF-8 <PyUnicode_AsUTF8AndSize>`, + or another non-"canonical" representation, + - must not have its reference count changed, + - must not be shared with code that might do one of the above. + + This list is not exhaustive. Avoiding these uses is your responsibility; + Python does not always check these requirements. + + To avoid accidentally exposing a partially-written string object, prefer + using the :c:type:`PyUnicodeWriter` API, or one of the ``PyUnicode_From*`` + functions below. + .. versionadded:: 3.3 @@ -358,9 +368,9 @@ APIs: .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *str, Py_ssize_t size) - Create a Unicode object from the char buffer *u*. The bytes will be + Create a Unicode object from the char buffer *str*. The bytes will be interpreted as being UTF-8 encoded. The buffer is copied into the new object. The return value might be a shared object, i.e. modification of the data is @@ -369,16 +379,16 @@ APIs: This function raises :exc:`SystemError` when: * *size* < 0, - * *u* is ``NULL`` and *size* > 0 + * *str* is ``NULL`` and *size* > 0 .. versionchanged:: 3.12 - *u* == ``NULL`` with *size* > 0 is not allowed anymore. + *str* == ``NULL`` with *size* > 0 is not allowed anymore. -.. c:function:: PyObject *PyUnicode_FromString(const char *u) +.. c:function:: PyObject *PyUnicode_FromString(const char *str) Create a Unicode object from a UTF-8 encoded null-terminated char buffer - *u*. + *str*. .. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) @@ -518,6 +528,26 @@ APIs: - :c:expr:`PyObject*` - The result of calling :c:func:`PyObject_Repr`. + * - ``T`` + - :c:expr:`PyObject*` + - Get the fully qualified name of an object type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#T`` + - :c:expr:`PyObject*` + - Similar to ``T`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + + * - ``N`` + - :c:expr:`PyTypeObject*` + - Get the fully qualified name of a type; + call :c:func:`PyType_GetFullyQualifiedName`. + + * - ``#N`` + - :c:expr:`PyTypeObject*` + - Similar to ``N`` format, but use a colon (``:``) as separator between + the module name and the qualified name. + .. note:: The width formatter unit is number of characters rather than bytes. The precision formatter unit is number of bytes or :c:type:`wchar_t` @@ -553,6 +583,9 @@ APIs: In previous versions it caused all the rest of the format string to be copied as-is to the result string, and any extra arguments discarded. + .. versionchanged:: 3.13 + Support for ``%T``, ``%#T``, ``%N`` and ``%#N`` formats added. + .. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) @@ -569,6 +602,14 @@ APIs: Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. +.. c:function:: PyObject* PyUnicode_FromOrdinal(int ordinal) + + Create a Unicode Object from the given Unicode code point *ordinal*. + + The ordinal must be in ``range(0x110000)``. A :exc:`ValueError` is + raised in the case it is not. + + .. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \ const char *encoding, const char *errors) @@ -587,10 +628,49 @@ APIs: decref'ing the returned objects. +.. c:function:: void PyUnicode_Append(PyObject **p_left, PyObject *right) + + Append the string *right* to the end of *p_left*. + *p_left* must point to a :term:`strong reference` to a Unicode object; + :c:func:`!PyUnicode_Append` releases ("steals") this reference. + + On error, set *\*p_left* to ``NULL`` and set an exception. + + On success, set *\*p_left* to a new strong reference to the result. + + +.. c:function:: void PyUnicode_AppendAndDel(PyObject **p_left, PyObject *right) + + The function is similar to :c:func:`PyUnicode_Append`, with the only + difference being that it decrements the reference count of *right* by one. + + +.. c:function:: PyObject* PyUnicode_BuildEncodingMap(PyObject* string) + + Return a mapping suitable for decoding a custom single-byte encoding. + Given a Unicode string *string* of up to 256 characters representing an encoding + table, returns either a compact internal mapping object or a dictionary + mapping character ordinals to byte values. Raises a :exc:`TypeError` and + return ``NULL`` on invalid input. + + .. versionadded:: 3.2 + + +.. c:function:: const char* PyUnicode_GetDefaultEncoding(void) + + Return the name of the default string encoding, ``"utf-8"``. + See :func:`sys.getdefaultencoding`. + + The returned string does not need to be freed, and is valid + until interpreter shutdown. + + .. c:function:: Py_ssize_t PyUnicode_GetLength(PyObject *unicode) Return the length of the Unicode object, in code points. + On error, set an exception and return ``-1``. + .. versionadded:: 3.3 @@ -605,9 +685,27 @@ APIs: possible. Returns ``-1`` and sets an exception on error, otherwise returns the number of copied characters. + The string must not have been “used” yet. + See :c:func:`PyUnicode_New` for details. + .. versionadded:: 3.3 +.. c:function:: int PyUnicode_Resize(PyObject **unicode, Py_ssize_t length); + + Resize a Unicode object *\*unicode* to the new *length* in code points. + + Try to resize the string in place (which is usually faster than allocating + a new string and copying characters), or create a new string. + + *\*unicode* is modified to point to the new (resized) object and ``0`` is + returned on success. Otherwise, ``-1`` is returned and an exception is set, + and *\*unicode* is left untouched. + + The function doesn't check string content, the result may not be a + string in canonical representation. + + .. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t length, Py_UCS4 fill_char) @@ -617,6 +715,9 @@ APIs: Fail if *fill_char* is bigger than the string maximum character, or if the string has more than 1 reference. + The string must not have been “used” yet. + See :c:func:`PyUnicode_New` for details. + Return the number of written character, or return ``-1`` and raise an exception on error. @@ -626,13 +727,16 @@ APIs: .. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \ Py_UCS4 character) - Write a character to a string. The string must have been created through - :c:func:`PyUnicode_New`. Since Unicode strings are supposed to be immutable, - the string must not be shared, or have been hashed yet. + Write a *character* to the string *unicode* at the zero-based *index*. + Return ``0`` on success, ``-1`` on error with an exception set. This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object can be modified safely (i.e. that it - its reference count is one). + not out of bounds, and that the object's reference count is one). + See :c:func:`PyUnicode_WRITE` for a version that skips these checks, + making them your responsibility. + + The string must not have been “used” yet. + See :c:func:`PyUnicode_New` for details. .. versionadded:: 3.3 @@ -643,32 +747,35 @@ APIs: Unicode object and the index is not out of bounds, in contrast to :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. + Return character on success, ``-1`` on error with an exception set. + .. versionadded:: 3.3 -.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \ +.. c:function:: PyObject* PyUnicode_Substring(PyObject *unicode, Py_ssize_t start, \ Py_ssize_t end) - Return a substring of *str*, from character index *start* (included) to + Return a substring of *unicode*, from character index *start* (included) to character index *end* (excluded). Negative indices are not supported. + On error, set an exception and return ``NULL``. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \ +.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *unicode, Py_UCS4 *buffer, \ Py_ssize_t buflen, int copy_null) - Copy the string *u* into a UCS4 buffer, including a null character, if + Copy the string *unicode* into a UCS4 buffer, including a null character, if *copy_null* is set. Returns ``NULL`` and sets an exception on error (in particular, a :exc:`SystemError` if *buflen* is smaller than the length of - *u*). *buffer* is returned on success. + *unicode*). *buffer* is returned on success. .. versionadded:: 3.3 -.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) +.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *unicode) - Copy the string *u* into a new UCS4 buffer that is allocated using + Copy the string *unicode* into a new UCS4 buffer that is allocated using :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a :exc:`MemoryError` set. The returned buffer always has an extra null code point appended. @@ -683,7 +790,7 @@ The current locale encoding can be used to decode text from the operating system. .. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ + Py_ssize_t length, \ const char *errors) Decode a string from UTF-8 on Android and VxWorks, or from the current @@ -754,16 +861,25 @@ Functions encoding to and decoding from the :term:`filesystem encoding and error handler` (:pep:`383` and :pep:`529`). To encode file names to :class:`bytes` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSConverter` as the +converter should be used, passing :c:func:`!PyUnicode_FSConverter` as the conversion function: .. c:function:: int PyUnicode_FSConverter(PyObject* obj, void* result) - ParseTuple converter: encode :class:`str` objects -- obtained directly or + :ref:`PyArg_Parse\* converter <arg-parsing>`: encode :class:`str` objects -- obtained directly or through the :class:`os.PathLike` interface -- to :class:`bytes` using :c:func:`PyUnicode_EncodeFSDefault`; :class:`bytes` objects are output as-is. - *result* must be a :c:expr:`PyBytesObject*` which must be released when it is - no longer used. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyBytesObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`bytes object <bytesobjects>` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null bytes are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, the function releases a strong reference + stored in the variable referred by *result* and returns ``1``. .. versionadded:: 3.1 @@ -771,16 +887,26 @@ conversion function: Accepts a :term:`path-like object`. To decode file names to :class:`str` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSDecoder` as the +converter should be used, passing :c:func:`!PyUnicode_FSDecoder` as the conversion function: .. c:function:: int PyUnicode_FSDecoder(PyObject* obj, void* result) - ParseTuple converter: decode :class:`bytes` objects -- obtained either + :ref:`PyArg_Parse\* converter <arg-parsing>`: decode :class:`bytes` objects -- obtained either directly or indirectly through the :class:`os.PathLike` interface -- to :class:`str` using :c:func:`PyUnicode_DecodeFSDefaultAndSize`; :class:`str` - objects are output as-is. *result* must be a :c:expr:`PyUnicodeObject*` which - must be released when it is no longer used. + objects are output as-is. + *result* must be an address of a C variable of type :c:expr:`PyObject*` + (or :c:expr:`PyUnicodeObject*`). + On success, set the variable to a new :term:`strong reference` to + a :ref:`Unicode object <unicodeobjects>` which must be released + when it is no longer used and return a non-zero value + (:c:macro:`Py_CLEANUP_SUPPORTED`). + Embedded null characters are not allowed in the result. + On failure, return ``0`` with an exception set. + + If *obj* is ``NULL``, release the strong reference + to the object referred to by *result* and return ``1``. .. versionadded:: 3.2 @@ -788,7 +914,7 @@ conversion function: Accepts a :term:`path-like object`. -.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *str, Py_ssize_t size) Decode a string from the :term:`filesystem encoding and error handler`. @@ -804,7 +930,7 @@ conversion function: handler>` is now used. -.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) +.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *str) Decode a null-terminated string from the :term:`filesystem encoding and error handler`. @@ -841,20 +967,25 @@ wchar_t Support :c:type:`wchar_t` support for platforms which support it: -.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) +.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *wstr, Py_ssize_t size) - Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. + Create a Unicode object from the :c:type:`wchar_t` buffer *wstr* of the given *size*. Passing ``-1`` as the *size* indicates that the function must itself compute the length, - using wcslen. + using :c:func:`!wcslen`. Return ``NULL`` on failure. -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) +.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *wstr, Py_ssize_t size) - Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most + Copy the Unicode object contents into the :c:type:`wchar_t` buffer *wstr*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing null termination character). Return the number of :c:type:`wchar_t` characters - copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` + copied or ``-1`` in case of an error. + + When *wstr* is ``NULL``, instead return the *size* that would be required + to store all of *unicode* including a terminating null. + + Note that the resulting :c:expr:`wchar_t*` string may or may not be null-terminated. It is the responsibility of the caller to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is required by the application. Also, note that the :c:expr:`wchar_t*` string @@ -912,13 +1043,24 @@ generic ones are documented for simplicity. Generic Codecs """""""""""""" +The following macro is provided: + + +.. c:macro:: Py_UNICODE_REPLACEMENT_CHARACTER + + The Unicode code point ``U+FFFD`` (replacement character). + + This Unicode character is used as the replacement character during + decoding if the *errors* argument is set to "replace". + + These are the generic codec APIs: -.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_Decode(const char *str, Py_ssize_t size, \ const char *encoding, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s*. + Create a Unicode object by decoding *size* bytes of the encoded string *str*. *encoding* and *errors* have the same meaning as the parameters of the same name in the :func:`str` built-in function. The codec to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by @@ -941,13 +1083,13 @@ UTF-8 Codecs These are the UTF-8 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If @@ -962,6 +1104,9 @@ These are the UTF-8 codec APIs: object. Error handling is "strict". Return ``NULL`` if an exception was raised by the codec. + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). + .. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) @@ -971,8 +1116,11 @@ These are the UTF-8 codec APIs: returned buffer always has an extra null byte appended (not included in *size*), regardless of whether there are any other null code points. - In the case of an error, ``NULL`` is returned with an exception set and no - *size* is stored. + On error, set an exception, set *size* to ``-1`` (if it's not NULL) and + return ``NULL``. + + The function fails if the string contains surrogate code points + (``U+D800`` - ``U+DFFF``). This caches the UTF-8 representation of the string in the Unicode object, and subsequent calls will return a pointer to the same buffer. The caller is not @@ -992,6 +1140,15 @@ These are the UTF-8 codec APIs: As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. + .. warning:: + + This function does not have any special behavior for + `null characters <https://en.wikipedia.org/wiki/Null_character>`_ embedded within + *unicode*. As a result, strings containing null characters will remain in the returned + string, which some C functions might interpret as the end of the string, leading to + truncation. If truncation is an issue, it is recommended to use :c:func:`PyUnicode_AsUTF8AndSize` + instead. + .. versionadded:: 3.3 .. versionchanged:: 3.7 @@ -1004,7 +1161,7 @@ UTF-32 Codecs These are the UTF-32 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-32 encoded buffer string and return the @@ -1031,7 +1188,7 @@ These are the UTF-32 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If @@ -1054,7 +1211,7 @@ UTF-16 Codecs These are the UTF-16 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder) Decode *size* bytes from a UTF-16 encoded buffer string and return the @@ -1082,7 +1239,7 @@ These are the UTF-16 codec APIs: Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *str, Py_ssize_t size, \ const char *errors, int *byteorder, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If @@ -1105,13 +1262,13 @@ UTF-7 Codecs These are the UTF-7 codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If @@ -1126,11 +1283,11 @@ Unicode-Escape Codecs These are the "Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return ``NULL`` if an exception was raised by the codec. + string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) @@ -1146,11 +1303,11 @@ Raw-Unicode-Escape Codecs These are the "Raw Unicode Escape" codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \ +.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *str, \ Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return ``NULL`` if an exception was raised by the codec. + encoded string *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) @@ -1167,10 +1324,10 @@ These are the Latin-1 codec APIs: Latin-1 corresponds to the first 256 Unicode ordinals and only these are accepted by the codecs during encoding. -.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) @@ -1187,10 +1344,10 @@ These are the ASCII codec APIs. Only 7-bit ASCII data is accepted. All other codes generate errors. -.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *str, Py_ssize_t size, const char *errors) Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. + *str*. Return ``NULL`` if an exception was raised by the codec. .. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) @@ -1211,10 +1368,10 @@ decode characters. The mapping objects provided must support the These are the mapping codec APIs: -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *str, Py_ssize_t length, \ PyObject *mapping, const char *errors) - Create a Unicode object by decoding *size* bytes of the encoded string *s* + Create a Unicode object by decoding *size* bytes of the encoded string *str* using the given *mapping* object. Return ``NULL`` if an exception was raised by the codec. @@ -1241,7 +1398,7 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, PyObject *table, const char *errors) Translate a string by applying a character mapping table to it and return the resulting Unicode object. Return ``NULL`` if an exception was raised by the @@ -1266,13 +1423,13 @@ use the Win32 MBCS converters to implement the conversions. Note that MBCS (or DBCS) is a class of encodings, not just one. The target encoding is defined by the user settings on the machine running the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) +.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *str, Py_ssize_t size, const char *errors) - Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. + Create a Unicode object by decoding *size* bytes of the MBCS encoded string *str*. Return ``NULL`` if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ +.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *str, Py_ssize_t size, \ const char *errors, Py_ssize_t *consumed) If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If @@ -1281,6 +1438,13 @@ the user settings on the machine running the codec. in *consumed*. +.. c:function:: PyObject* PyUnicode_DecodeCodePageStateful(int code_page, const char *str, \ + Py_ssize_t size, const char *errors, Py_ssize_t *consumed) + + Similar to :c:func:`PyUnicode_DecodeMBCSStateful`, except uses the code page + specified by *code_page*. + + .. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode) Encode a Unicode object using MBCS and return the result as Python bytes @@ -1292,15 +1456,11 @@ the user settings on the machine running the codec. Encode the Unicode object using the specified code page and return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:macro:`CP_ACP` code page to get the MBCS encoder. + :c:macro:`!CP_ACP` code page to get the MBCS encoder. .. versionadded:: 3.3 -Methods & Slots -""""""""""""""" - - .. _unicodemethodsandslots: Methods and Slot Functions @@ -1318,49 +1478,90 @@ They all return ``NULL`` or ``-1`` if an exception occurs. Concat two strings giving a new Unicode string. -.. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) +.. c:function:: PyObject* PyUnicode_Split(PyObject *unicode, PyObject *sep, Py_ssize_t maxsplit) Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting will be done at all whitespace substrings. Otherwise, splits occur at the given separator. At most *maxsplit* splits will be done. If negative, no limit is set. Separators are not included in the resulting list. + On error, return ``NULL`` with an exception set. -.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *s, int keepend) + Equivalent to :py:meth:`str.split`. + + +.. c:function:: PyObject* PyUnicode_RSplit(PyObject *unicode, PyObject *sep, Py_ssize_t maxsplit) + + Similar to :c:func:`PyUnicode_Split`, but splitting will be done beginning + at the end of the string. + + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.rsplit`. + + +.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *unicode, int keepends) Split a Unicode string at line breaks, returning a list of Unicode strings. - CRLF is considered to be one line break. If *keepend* is ``0``, the line break + CRLF is considered to be one line break. If *keepends* is ``0``, the Line break characters are not included in the resulting strings. +.. c:function:: PyObject* PyUnicode_Partition(PyObject *unicode, PyObject *sep) + + Split a Unicode string at the first occurrence of *sep*, and return + a 3-tuple containing the part before the separator, the separator itself, + and the part after the separator. If the separator is not found, + return a 3-tuple containing the string itself, followed by two empty strings. + + *sep* must not be empty. + + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.partition`. + + +.. c:function:: PyObject* PyUnicode_RPartition(PyObject *unicode, PyObject *sep) + + Similar to :c:func:`PyUnicode_Partition`, but split a Unicode string at the + last occurrence of *sep*. If the separator is not found, return a 3-tuple + containing two empty strings, followed by the string itself. + + *sep* must not be empty. + + On error, return ``NULL`` with an exception set. + + Equivalent to :py:meth:`str.rpartition`. + + .. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) Join a sequence of strings using the given *separator* and return the resulting Unicode string. -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return ``1`` if *substr* matches ``str[start:end]`` at the given tail end + Return ``1`` if *substr* matches ``unicode[start:end]`` at the given tail end (*direction* == ``-1`` means to do a prefix match, *direction* == ``1`` a suffix match), ``0`` otherwise. Return ``-1`` if an error occurred. -.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of *substr* in ``str[start:end]`` using the given + Return the first position of *substr* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` indicates that an error occurred and an exception has been set. -.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \ +.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *unicode, Py_UCS4 ch, \ Py_ssize_t start, Py_ssize_t end, int direction) - Return the first position of the character *ch* in ``str[start:end]`` using + Return the first position of the character *ch* in ``unicode[start:end]`` using the given *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a backward search). The return value is the index of the first match; a value of ``-1`` indicates that no match was found, and ``-2`` @@ -1369,20 +1570,20 @@ They all return ``NULL`` or ``-1`` if an exception occurs. .. versionadded:: 3.3 .. versionchanged:: 3.7 - *start* and *end* are now adjusted to behave like ``str[start:end]``. + *start* and *end* are now adjusted to behave like ``unicode[start:end]``. -.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \ +.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *unicode, PyObject *substr, \ Py_ssize_t start, Py_ssize_t end) Return the number of non-overlapping occurrences of *substr* in - ``str[start:end]``. Return ``-1`` if an error occurred. + ``unicode[start:end]``. Return ``-1`` if an error occurred. -.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \ +.. c:function:: PyObject* PyUnicode_Replace(PyObject *unicode, PyObject *substr, \ PyObject *replstr, Py_ssize_t maxcount) - Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and + Replace at most *maxcount* occurrences of *substr* in *unicode* with *replstr* and return the resulting Unicode object. *maxcount* == ``-1`` means replace all occurrences. @@ -1395,10 +1596,58 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function returns ``-1`` upon failure, so one should call :c:func:`PyErr_Occurred` to check for errors. + .. seealso:: + + The :c:func:`PyUnicode_Equal` function. + + +.. c:function:: int PyUnicode_Equal(PyObject *a, PyObject *b) -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) + Test if two strings are equal: - Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less + * Return ``1`` if *a* is equal to *b*. + * Return ``0`` if *a* is not equal to *b*. + * Set a :exc:`TypeError` exception and return ``-1`` if *a* or *b* is not a + :class:`str` object. + + The function always succeeds if *a* and *b* are :class:`str` objects. + + The function works for :class:`str` subclasses, but does not honor custom + ``__eq__()`` method. + + .. seealso:: + + The :c:func:`PyUnicode_Compare` function. + + .. versionadded:: 3.14 + + +.. c:function:: int PyUnicode_EqualToUTF8AndSize(PyObject *unicode, const char *string, Py_ssize_t size) + + Compare a Unicode object with a char buffer which is interpreted as + being UTF-8 or ASCII encoded and return true (``1``) if they are equal, + or false (``0``) otherwise. + If the Unicode object contains surrogate code points + (``U+D800`` - ``U+DFFF``) or the C string is not valid UTF-8, + false (``0``) is returned. + + This function does not raise exceptions. + + .. versionadded:: 3.13 + + +.. c:function:: int PyUnicode_EqualToUTF8(PyObject *unicode, const char *string) + + Similar to :c:func:`PyUnicode_EqualToUTF8AndSize`, but compute *string* + length using :c:func:`!strlen`. + If the Unicode object contains null characters, false (``0``) is returned. + + .. versionadded:: 3.13 + + +.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *unicode, const char *string) + + Compare a Unicode object, *unicode*, with *string* and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, respectively. It is best to pass only ASCII-encoded strings, but the function interprets the input string as ISO-8859-1 if it contains non-ASCII characters. @@ -1406,7 +1655,7 @@ They all return ``NULL`` or ``-1`` if an exception occurs. This function does not raise exceptions. -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) +.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) Rich compare two Unicode strings and return one of the following: @@ -1424,31 +1673,260 @@ They all return ``NULL`` or ``-1`` if an exception occurs. ``format % args``. -.. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) +.. c:function:: int PyUnicode_Contains(PyObject *unicode, PyObject *substr) - Check whether *element* is contained in *container* and return true or false + Check whether *substr* is contained in *unicode* and return true or false accordingly. - *element* has to coerce to a one element Unicode string. ``-1`` is returned + *substr* has to coerce to a one element Unicode string. ``-1`` is returned if there was an error. -.. c:function:: void PyUnicode_InternInPlace(PyObject **string) +.. c:function:: void PyUnicode_InternInPlace(PyObject **p_unicode) - Intern the argument *\*string* in place. The argument must be the address of a + Intern the argument :c:expr:`*p_unicode` in place. The argument must be the address of a pointer variable pointing to a Python Unicode string object. If there is an - existing interned string that is the same as *\*string*, it sets *\*string* to + existing interned string that is the same as :c:expr:`*p_unicode`, it sets :c:expr:`*p_unicode` to it (releasing the reference to the old string object and creating a new :term:`strong reference` to the interned string object), otherwise it leaves - *\*string* alone and interns it (creating a new :term:`strong reference`). + :c:expr:`*p_unicode` alone and interns it. + (Clarification: even though there is a lot of talk about references, think - of this function as reference-neutral; you own the object after the call - if and only if you owned it before the call.) + of this function as reference-neutral. You must own the object you pass in; + after the call you no longer own the passed-in reference, but you newly own + the result.) + + This function never raises an exception. + On error, it leaves its argument unchanged without interning it. + + Instances of subclasses of :py:class:`str` may not be interned, that is, + :c:expr:`PyUnicode_CheckExact(*p_unicode)` must be true. If it is not, + then -- as with any other error -- the argument is left unchanged. + + Note that interned strings are not “immortal”. + You must keep a reference to the result to benefit from interning. -.. c:function:: PyObject* PyUnicode_InternFromString(const char *v) +.. c:function:: PyObject* PyUnicode_InternFromString(const char *str) A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string - object that has been interned, or a new ("owned") reference to an earlier - interned string object with the same value. + :c:func:`PyUnicode_InternInPlace`, meant for statically allocated strings. + + Return a new ("owned") reference to either a new Unicode string object + that has been interned, or an earlier interned string object with the + same value. + + Python may keep a reference to the result, or make it :term:`immortal`, + preventing it from being garbage-collected promptly. + For interning an unbounded number of different strings, such as ones coming + from user input, prefer calling :c:func:`PyUnicode_FromString` and + :c:func:`PyUnicode_InternInPlace` directly. + + +.. c:function:: unsigned int PyUnicode_CHECK_INTERNED(PyObject *str) + + Return a non-zero value if *str* is interned, zero if not. + The *str* argument must be a string; this is not checked. + This function always succeeds. + + .. impl-detail:: + + A non-zero return value may carry additional information + about *how* the string is interned. + The meaning of such non-zero values, as well as each specific string's + intern-related details, may change between CPython versions. + + +PyUnicodeWriter +^^^^^^^^^^^^^^^ + +The :c:type:`PyUnicodeWriter` API can be used to create a Python :class:`str` +object. + +.. versionadded:: 3.14 + +.. c:type:: PyUnicodeWriter + + A Unicode writer instance. + + The instance must be destroyed by :c:func:`PyUnicodeWriter_Finish` on + success, or :c:func:`PyUnicodeWriter_Discard` on error. + +.. c:function:: PyUnicodeWriter* PyUnicodeWriter_Create(Py_ssize_t length) + + Create a Unicode writer instance. + + *length* must be greater than or equal to ``0``. + + If *length* is greater than ``0``, preallocate an internal buffer of + *length* characters. + + Set an exception and return ``NULL`` on error. + +.. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) + + Return the final Python :class:`str` object and destroy the writer instance. + + Set an exception and return ``NULL`` on error. + + The writer instance is invalid after this call. + +.. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) + + Discard the internal Unicode buffer and destroy the writer instance. + + If *writer* is ``NULL``, no operation is performed. + + The writer instance is invalid after this call. + +.. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) + + Write the single Unicode character *ch* into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Decode the string *str* from UTF-8 in strict mode and write the output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. + +.. c:function:: int PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Write the ASCII string *str* into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *str* must only contain ASCII characters. The behavior is undefined if + *str* contains non-ASCII characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + .. versionadded:: next + +.. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) + + Write the wide string *str* into *writer*. + + *size* is a number of wide characters. If *size* is equal to ``-1``, call + ``wcslen(str)`` to get the string length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteUCS4(PyUnicodeWriter *writer, Py_UCS4 *str, Py_ssize_t size) + + Writer the UCS4 string *str* into *writer*. + + *size* is a number of UCS4 characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Str` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) + + Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) + + Write the substring ``str[start:end]`` into *writer*. + + *str* must be Python :class:`str` object. *start* must be greater than or + equal to 0, and less than or equal to *end*. *end* must be less than or + equal to *str* length. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_Format(PyUnicodeWriter *writer, const char *format, ...) + + Similar to :c:func:`PyUnicode_FromFormat`, but write the output directly into *writer*. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + +.. c:function:: int PyUnicodeWriter_DecodeUTF8Stateful(PyUnicodeWriter *writer, const char *string, Py_ssize_t length, const char *errors, Py_ssize_t *consumed) + + Decode the string *str* from UTF-8 with *errors* error handler and write the + output into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *errors* is an :ref:`error handler <error-handlers>` name, such as + ``"replace"``. If *errors* is ``NULL``, use the strict error handler. + + If *consumed* is not ``NULL``, set *\*consumed* to the number of decoded + bytes on success. + If *consumed* is ``NULL``, treat trailing incomplete UTF-8 byte sequences + as an error. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + + See also :c:func:`PyUnicodeWriter_WriteUTF8`. + +Deprecated API +^^^^^^^^^^^^^^ + +The following API is deprecated. + +.. c:type:: Py_UNICODE + + This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type + depending on the platform. + Please use :c:type:`wchar_t` directly instead. + + .. versionchanged:: 3.3 + In previous versions, this was a 16-bit type or a 32-bit type depending on + whether you selected a "narrow" or "wide" Unicode version of Python at + build time. + + .. deprecated-removed:: 3.13 3.15 + + +.. c:function:: int PyUnicode_READY(PyObject *unicode) + + Do nothing and return ``0``. + This API is kept only for backward compatibility, but there are no plans + to remove it. + + .. versionadded:: 3.3 + + .. deprecated:: 3.10 + This API does nothing since Python 3.12. + Previously, this needed to be called for each string created using + the old API (:c:func:`!PyUnicode_FromUnicode` or similar). + + +.. c:function:: unsigned int PyUnicode_IS_READY(PyObject *unicode) + + Do nothing and return ``1``. + This API is kept only for backward compatibility, but there are no plans + to remove it. + + .. versionadded:: 3.3 + + .. deprecated:: 3.14 + This API does nothing since Python 3.12. + Previously, this could be called to check if + :c:func:`PyUnicode_READY` is necessary. diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst index ccbf14e1850f68..9d0abf440f791d 100644 --- a/Doc/c-api/utilities.rst +++ b/Doc/c-api/utilities.rst @@ -17,6 +17,8 @@ and parsing function arguments and constructing Python values from C values. marshal.rst arg.rst conversion.rst + hash.rst reflection.rst codec.rst + time.rst perfmaps.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 324518c035096b..1ef4181d52eb10 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -25,30 +25,6 @@ are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter. This is made available for - programs which embed Python. The *argc* and *argv* parameters should be - prepared exactly as those which are passed to a C program's :c:func:`main` - function (converted to wchar_t according to the user's locale). It is - important to note that the argument list may be modified (but the contents of - the strings pointed to by the argument list are not). The return value will - be ``0`` if the interpreter exits normally (i.e., without an exception), - ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter - list does not represent a valid Python command line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - :c:member:`PyConfig.inspect` is zero. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. - - .. versionadded:: 3.8 - - .. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving @@ -322,7 +298,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_eval_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for isolated expressions; for use with :c:func:`Py_CompileString`. @@ -330,7 +306,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_file_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for sequences of statements as read from a file or other source; for use with :c:func:`Py_CompileString`. This is @@ -339,7 +315,7 @@ the same library that the Python runtime is using. .. c:var:: int Py_single_input - .. index:: single: Py_CompileString() + .. index:: single: Py_CompileString (C function) The start symbol from the Python grammar for a single statement; for use with :c:func:`Py_CompileString`. This is the symbol used for the interactive @@ -372,8 +348,20 @@ the same library that the Python runtime is using. .. versionchanged:: 3.8 Added *cf_feature_version* field. + The available compiler flags are accessible as macros: + + .. c:namespace:: NULL + + .. c:macro:: PyCF_ALLOW_TOP_LEVEL_AWAIT + PyCF_ONLY_AST + PyCF_OPTIMIZED_AST + PyCF_TYPE_COMMENTS + + See :ref:`compiler flags <ast-compiler-flags>` in documentation of the + :py:mod:`!ast` Python module, which exports these constants under + the same names. -.. c:var:: int CO_FUTURE_DIVISION + .. c:var:: int CO_FUTURE_DIVISION - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + This bit can be set in *flags* to cause division operator ``/`` to be + interpreted as "true division" according to :pep:`238`. diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 038f54a9751fd1..c3c6cf413dcef5 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -35,7 +35,7 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or + weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -47,7 +47,7 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, + is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -88,6 +88,15 @@ as much as it can. Use :c:func:`PyWeakref_GetRef` instead. +.. c:function:: int PyWeakref_IsDead(PyObject *ref) + + Test if the weak reference *ref* is dead. Returns 1 if the reference is + dead, 0 if it is alive, and -1 with an error set if *ref* is not a weak + reference object. + + .. versionadded:: 3.14 + + .. c:function:: void PyObject_ClearWeakRefs(PyObject *object) This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler @@ -96,3 +105,19 @@ as much as it can. This iterates through the weak references for *object* and calls callbacks for those references which have one. It returns when all callbacks have been attempted. + + +.. c:function:: void PyUnstable_Object_ClearWeakRefsNoCallbacks(PyObject *object) + + Clears the weakrefs for *object* without calling the callbacks. + + This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler + for types with finalizers (i.e., :meth:`~object.__del__`). The handler for + those objects first calls :c:func:`PyObject_ClearWeakRefs` to clear weakrefs + and call their callbacks, then the finalizer, and finally this function to + clear any weakrefs that may have been created by the finalizer. + + In most circumstances, it's more appropriate to use + :c:func:`PyObject_ClearWeakRefs` to clear weakrefs instead of this function. + + .. versionadded:: 3.13 diff --git a/Doc/conf.py b/Doc/conf.py index 19e05e1aa8fe19..7fadad66cb3238 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -6,32 +6,52 @@ # The contents of this file are pickled, so don't put values in the namespace # that aren't pickleable (module imports are okay, they're removed automatically). -import sys, os, time +import os +import sys +from importlib import import_module +from importlib.util import find_spec + +# Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) +# Python specific content from Doc/Tools/extensions/pyspecific.py +from pyspecific import SOURCE_URI + # General configuration # --------------------- +# Our custom Sphinx extensions are found in Doc/Tools/extensions/ extensions = [ - 'asdl_highlight', + 'audit_events', + 'availability', 'c_annotations', - 'escape4chm', + 'changes', 'glossary_search', - 'peg_highlight', + 'grammar_snippet', + 'implementation_detail', + 'issue_role', + 'lexers', + 'misc_news', + 'pydoc_topics', 'pyspecific', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', ] -# Skip if downstream redistributors haven't installed it -try: - import sphinxext.opengraph -except ImportError: - pass -else: - extensions.append('sphinxext.opengraph') - +# Skip if downstream redistributors haven't installed them +_OPTIONAL_EXTENSIONS = ( + 'notfound.extension', + 'sphinxext.opengraph', +) +for optional_ext in _OPTIONAL_EXTENSIONS: + try: + if find_spec(optional_ext) is not None: + extensions.append(optional_ext) + except (ImportError, ValueError): + pass +del _OPTIONAL_EXTENSIONS doctest_global_setup = ''' try: @@ -39,7 +59,7 @@ except ImportError: _tkinter = None # Treat warnings as errors, done here to prevent warnings in Sphinx code from -# causing spurious test failures. +# causing spurious CPython test failures. import warnings warnings.simplefilter('error') del warnings @@ -49,27 +69,45 @@ # General substitutions. project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') +copyright = "2001 Python Software Foundation" # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. -import patchlevel -version, release = patchlevel.get_version_info() - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: +# See Doc/tools/extensions/patchlevel.py +version, release = import_module('patchlevel').get_version_info() + +rst_epilog = f""" +.. |python_version_literal| replace:: ``Python {version}`` +.. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` + +.. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ +""" + +# There are two options for replacing |today|. Either, you set today to some +# non-false value and use it. today = '' -# Else, today_fmt is used as the format for a strftime call. +# Or else, today_fmt is used as the format for a strftime call. today_fmt = '%B %d, %Y' # By default, highlight as Python 3. highlight_language = 'python3' # Minimum version of sphinx required -needs_sphinx = '3.2' +# Keep this version in sync with ``Doc/requirements.txt``. +needs_sphinx = '8.2.0' + +# Create table of contents entries for domain objects (e.g. functions, classes, +# attributes, etc.). Default is True. +toc_object_entries = False # Ignore any .rst files in the includes/ directory; -# they're embedded in pages but not rendered individually. +# they're embedded in pages but not rendered as individual pages. # Ignore any .rst files in the venv/ directory. exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst'] venvdir = os.getenv('VENVDIR') @@ -79,25 +117,40 @@ nitpick_ignore = [ # Standard C functions ('c:func', 'calloc'), + ('c:func', 'ctime'), ('c:func', 'dlopen'), ('c:func', 'exec'), ('c:func', 'fcntl'), + ('c:func', 'flock'), ('c:func', 'fork'), ('c:func', 'free'), + ('c:func', 'gettimeofday'), ('c:func', 'gmtime'), + ('c:func', 'grantpt'), + ('c:func', 'ioctl'), + ('c:func', 'localeconv'), ('c:func', 'localtime'), ('c:func', 'main'), ('c:func', 'malloc'), + ('c:func', 'mktime'), + ('c:func', 'posix_openpt'), ('c:func', 'printf'), + ('c:func', 'ptsname'), + ('c:func', 'ptsname_r'), ('c:func', 'realloc'), ('c:func', 'snprintf'), ('c:func', 'sprintf'), ('c:func', 'stat'), + ('c:func', 'strftime'), ('c:func', 'system'), ('c:func', 'time'), + ('c:func', 'unlockpt'), ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int8_t'), + ('c:type', 'int16_t'), + ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), ('c:type', 'off_t'), @@ -106,6 +159,9 @@ ('c:type', 'size_t'), ('c:type', 'ssize_t'), ('c:type', 'time_t'), + ('c:type', 'uint8_t'), + ('c:type', 'uint16_t'), + ('c:type', 'uint32_t'), ('c:type', 'uint64_t'), ('c:type', 'uintmax_t'), ('c:type', 'uintptr_t'), @@ -113,11 +169,14 @@ ('c:type', 'wchar_t'), ('c:type', '__int64'), ('c:type', 'unsigned __int64'), + ('c:type', 'double'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), ('c:struct', 'stat'), ('c:struct', 'statvfs'), + ('c:struct', 'timeval'), + ('c:struct', 'timespec'), # Standard C macros ('c:macro', 'LLONG_MAX'), ('c:macro', 'LLONG_MIN'), @@ -145,6 +204,7 @@ ('envvar', 'LC_TIME'), ('envvar', 'LINES'), ('envvar', 'LOGNAME'), + ('envvar', 'MANPAGER'), ('envvar', 'PAGER'), ('envvar', 'PATH'), ('envvar', 'PATHEXT'), @@ -157,47 +217,143 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), + # Deprecated function that was never documented: + ('py:func', 'getargspec'), + ('py:func', 'inspect.getargspec'), + # Undocumented modules that users shouldn't have to worry about + # (implementation details of `os.path`): + ('py:mod', 'ntpath'), + ('py:mod', 'posixpath'), +] + +# Temporary undocumented names. +# In future this list must be empty. +nitpick_ignore += [ + # C API: Standard Python exception classes + ('c:data', 'PyExc_ArithmeticError'), + ('c:data', 'PyExc_AssertionError'), + ('c:data', 'PyExc_AttributeError'), + ('c:data', 'PyExc_BaseException'), + ('c:data', 'PyExc_BlockingIOError'), + ('c:data', 'PyExc_BrokenPipeError'), + ('c:data', 'PyExc_BufferError'), + ('c:data', 'PyExc_ChildProcessError'), + ('c:data', 'PyExc_ConnectionAbortedError'), + ('c:data', 'PyExc_ConnectionError'), + ('c:data', 'PyExc_ConnectionRefusedError'), + ('c:data', 'PyExc_ConnectionResetError'), + ('c:data', 'PyExc_EOFError'), + ('c:data', 'PyExc_Exception'), + ('c:data', 'PyExc_FileExistsError'), + ('c:data', 'PyExc_FileNotFoundError'), + ('c:data', 'PyExc_FloatingPointError'), + ('c:data', 'PyExc_GeneratorExit'), + ('c:data', 'PyExc_ImportError'), + ('c:data', 'PyExc_IndentationError'), + ('c:data', 'PyExc_IndexError'), + ('c:data', 'PyExc_InterruptedError'), + ('c:data', 'PyExc_IsADirectoryError'), + ('c:data', 'PyExc_KeyboardInterrupt'), + ('c:data', 'PyExc_KeyError'), + ('c:data', 'PyExc_LookupError'), + ('c:data', 'PyExc_MemoryError'), + ('c:data', 'PyExc_ModuleNotFoundError'), + ('c:data', 'PyExc_NameError'), + ('c:data', 'PyExc_NotADirectoryError'), + ('c:data', 'PyExc_NotImplementedError'), + ('c:data', 'PyExc_OSError'), + ('c:data', 'PyExc_OverflowError'), + ('c:data', 'PyExc_PermissionError'), + ('c:data', 'PyExc_ProcessLookupError'), + ('c:data', 'PyExc_PythonFinalizationError'), + ('c:data', 'PyExc_RecursionError'), + ('c:data', 'PyExc_ReferenceError'), + ('c:data', 'PyExc_RuntimeError'), + ('c:data', 'PyExc_StopAsyncIteration'), + ('c:data', 'PyExc_StopIteration'), + ('c:data', 'PyExc_SyntaxError'), + ('c:data', 'PyExc_SystemError'), + ('c:data', 'PyExc_SystemExit'), + ('c:data', 'PyExc_TabError'), + ('c:data', 'PyExc_TimeoutError'), + ('c:data', 'PyExc_TypeError'), + ('c:data', 'PyExc_UnboundLocalError'), + ('c:data', 'PyExc_UnicodeDecodeError'), + ('c:data', 'PyExc_UnicodeEncodeError'), + ('c:data', 'PyExc_UnicodeError'), + ('c:data', 'PyExc_UnicodeTranslateError'), + ('c:data', 'PyExc_ValueError'), + ('c:data', 'PyExc_ZeroDivisionError'), + # C API: Standard Python warning classes + ('c:data', 'PyExc_BytesWarning'), + ('c:data', 'PyExc_DeprecationWarning'), + ('c:data', 'PyExc_FutureWarning'), + ('c:data', 'PyExc_ImportWarning'), + ('c:data', 'PyExc_PendingDeprecationWarning'), + ('c:data', 'PyExc_ResourceWarning'), + ('c:data', 'PyExc_RuntimeWarning'), + ('c:data', 'PyExc_SyntaxWarning'), + ('c:data', 'PyExc_UnicodeWarning'), + ('c:data', 'PyExc_UserWarning'), + ('c:data', 'PyExc_Warning'), + # Undocumented public C macros + ('c:macro', 'Py_BUILD_ASSERT'), + ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), + # Attributes/methods/etc. that definitely should be documented better, + # but are deferred for now: + ('py:attr', '__annotations__'), + ('py:meth', '__missing__'), + ('py:attr', '__wrapped__'), + ('py:meth', 'index'), # list.index, tuple.index, etc. ] -# gh-106948: Copy standard C types declared in the "c:type" domain to the -# "c:identifier" domain, since "c:function" markup looks for types in the -# "c:identifier" domain. Use list() to not iterate on items which are being -# added +# gh-106948: Copy standard C types declared in the "c:type" domain and C +# structures declared in the "c:struct" domain to the "c:identifier" domain, +# since "c:function" markup looks for types in the "c:identifier" domain. Use +# list() to not iterate on items which are being added for role, name in list(nitpick_ignore): - if role == 'c:type': + if role in ('c:type', 'c:struct'): nitpick_ignore.append(('c:identifier', name)) del role, name # Disable Docutils smartquotes for several translations smartquotes_excludes = { - 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], + 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], + 'builders': ['man', 'text'], } -# Avoid a warning with Sphinx >= 2.0 -master_doc = 'contents' +# Avoid a warning with Sphinx >= 4.0 +root_doc = 'contents' # Allow translation of index directives gettext_additional_targets = [ 'index', + 'literal-block', ] # Options for HTML output # ----------------------- -# Use our custom theme. +# Use our custom theme: https://github.com/python/python-docs-theme html_theme = 'python_docs_theme' +# Location of overrides for theme templates and static files html_theme_path = ['tools'] html_theme_options = { 'collapsiblesidebar': True, 'issues_url': '/bugs.html', 'license_url': '/license.html', - 'root_include_title': False # We use the version switcher instead. + 'root_include_title': False, # We use the version switcher instead. } +if os.getenv("READTHEDOCS"): + html_theme_options["hosted_on"] = ( + '<a href="https://about.readthedocs.com/">Read the Docs</a>' + ) + # Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 # https://github.com/python/cpython/issues/91207 if any('htmlhelp' in arg for arg in sys.argv): @@ -206,22 +362,25 @@ print("It may be removed in the future\n") # Short title used e.g. for <title> HTML tags. -html_short_title = '%s Documentation' % release +html_short_title = f'{release} Documentation' # Deployment preview information # (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) -repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") +is_deployment_preview = os.getenv("READTHEDOCS_VERSION_TYPE") == "external" +repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL", "") +repository_url = repository_url.removesuffix(".git") html_context = { - "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", - "repository_url": repository_url.removesuffix(".git") if repository_url else None, - "pr_id": os.getenv("READTHEDOCS_VERSION") + "is_deployment_preview": is_deployment_preview, + "repository_url": repository_url or None, + "pr_id": os.getenv("READTHEDOCS_VERSION"), + "enable_analytics": os.getenv("PYTHON_DOCS_ENABLE_ANALYTICS"), } -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' +# This 'Last updated on:' timestamp is inserted at the bottom of every page. +html_last_updated_fmt = '%b %d, %Y (%H:%M UTC)' +html_last_updated_use_utc = True -# Path to find HTML templates. +# Path to find HTML templates to override theme templates_path = ['tools/templates'] # Custom sidebar templates, filenames relative to this file. @@ -250,18 +409,17 @@ # Split the index html_split_index = True +# Split pot files one per reST file +gettext_compact = False # Options for LaTeX output # ------------------------ latex_engine = 'xelatex' -# Get LaTeX to handle Unicode correctly latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' + # For the LaTeX preamble. + 'preamble': r''' \authoraddress{ \sphinxstrong{Python Software Foundation}\\ Email: \sphinxemail{docs@python.org} @@ -269,44 +427,81 @@ \let\Verbatim=\OriginalVerbatim \let\endVerbatim=\endOriginalVerbatim \setcounter{tocdepth}{2} -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' +''', + # The paper size ('letterpaper' or 'a4paper'). + 'papersize': 'a4paper', + # The font size ('10pt', '11pt' or '12pt'). + 'pointsize': '10pt', +} # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). _stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', - 'The Python/C API', _stdauthor, 'manual'), - ('distributing/index', 'distributing.tex', - 'Distributing Python Modules', _stdauthor, 'manual'), - ('extending/index', 'extending.tex', - 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', - 'Installing Python Modules', _stdauthor, 'manual'), - ('library/index', 'library.tex', - 'The Python Library Reference', _stdauthor, 'manual'), - ('reference/index', 'reference.tex', - 'The Python Language Reference', _stdauthor, 'manual'), - ('tutorial/index', 'tutorial.tex', - 'Python Tutorial', _stdauthor, 'manual'), - ('using/index', 'using.tex', - 'Python Setup and Usage', _stdauthor, 'manual'), - ('faq/index', 'faq.tex', - 'Python Frequently Asked Questions', _stdauthor, 'manual'), - ('whatsnew/' + version, 'whatsnew.tex', - 'What\'s New in Python', 'A. M. Kuchling', 'howto'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ( + 'extending/index', + 'extending.tex', + 'Extending and Embedding Python', + _stdauthor, + 'manual', + ), + ( + 'installing/index', + 'installing.tex', + 'Installing Python Modules', + _stdauthor, + 'manual', + ), + ( + 'library/index', + 'library.tex', + 'The Python Library Reference', + _stdauthor, + 'manual', + ), + ( + 'reference/index', + 'reference.tex', + 'The Python Language Reference', + _stdauthor, + 'manual', + ), + ( + 'tutorial/index', + 'tutorial.tex', + 'Python Tutorial', + _stdauthor, + 'manual', + ), + ( + 'using/index', + 'using.tex', + 'Python Setup and Usage', + _stdauthor, + 'manual', + ), + ( + 'faq/index', + 'faq.tex', + 'Python Frequently Asked Questions', + _stdauthor, + 'manual', + ), + ( + 'whatsnew/' + version, + 'whatsnew.tex', + 'What\'s New in Python', + 'A. M. Kuchling', + 'howto', + ), ] # Collect all HOWTOs individually -latex_documents.extend(('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', - '', _stdauthor, 'howto') - for fn in os.listdir('howto') - if fn.endswith('.rst') and fn != 'index.rst') +latex_documents.extend( + ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + for fn in os.listdir('howto') + if fn.endswith('.rst') and fn != 'index.rst' +) # Documents to append as an appendix to all manuals. latex_appendices = ['glossary', 'about', 'license', 'copyright'] @@ -317,6 +512,10 @@ epub_author = 'Python Documentation Authors' epub_publisher = 'Python Software Foundation' +# index pages are not valid xhtml +# https://github.com/sphinx-doc/sphinx/issues/12359 +epub_use_index = False + # Options for the coverage checker # -------------------------------- @@ -330,8 +529,7 @@ 'test($|_)', ] -coverage_ignore_classes = [ -] +coverage_ignore_classes = [] # Glob patterns for C source files for C API coverage, relative to this directory. coverage_c_path = [ @@ -340,15 +538,15 @@ # Regexes to find C items in the source files. coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), + 'cfunction': r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)', + 'data': r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)', + 'macro': r'^#define ([^_][\w_]+)\(.*\)[\s|\\]', } # The coverage checker will ignore all C items whose names match these regexes # (using re.match) -- the keys must be the same as in coverage_c_regexes. coverage_ignore_c_items = { -# 'cfunction': [...] + # 'cfunction': [...] } @@ -364,15 +562,19 @@ r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst r'http://www.python.org/$': 'https://www.python.org/$', - # Used in license page, keep as is - r'https://www.zope.org/': r'https://www.zope.dev/', # Microsoft's redirects to learn.microsoft.com r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', + # Debian's man page redirects to its current stable version + r'https://manpages.debian.org/\w+\(\d(\w+)?\)': r'https://manpages.debian.org/\w+/[\w/\-\.]*\.\d(\w+)?\.en\.html', # Language redirects r'https://toml.io': 'https://toml.io/en/', r'https://www.redhat.com': 'https://www.redhat.com/en', + # pypi.org project name normalization (upper to lowercase, underscore to hyphen) + r'https://pypi.org/project/[A-Za-z\d_\-\.]+/': r'https://pypi.org/project/[a-z\d\-\.]+/', + # Discourse title name expansion (text changes when title is edited) + r'https://discuss\.python\.org/t/\d+': r'https://discuss\.python\.org/t/.*/\d+', # Other redirects r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', @@ -399,20 +601,41 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] +# Options for sphinx.ext.extlinks +# ------------------------------- + +# This config is a dictionary of external sites, +# mapping unique short aliases to a base URL and a prefix. +# https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html +extlinks = { + "pypi": ("https://pypi.org/project/%s/", "%s"), + "source": (SOURCE_URI, "%s"), +} +extlinks_detect_hardcoded_links = True -# Options for extensions -# ---------------------- +# Options for c_annotations extension +# ----------------------------------- # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' -# sphinxext-opengraph config -ogp_site_url = 'https://docs.python.org/3/' +# Options for sphinxext-opengraph +# ------------------------------- + +ogp_canonical_url = 'https://docs.python.org/3/' ogp_site_name = 'Python documentation' -ogp_image = '_static/og-image.png' +ogp_social_cards = { # Used when matplotlib is installed + 'image': '_static/og-image.png', + 'line_color': '#3776ab', +} ogp_custom_meta_tags = [ - '<meta property="og:image:width" content="200" />', - '<meta property="og:image:height" content="200" />', - '<meta name="theme-color" content="#3776ab" />', + '<meta name="theme-color" content="#3776ab">', ] +if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += [ + '<meta property="og:image:width" content="200">', + '<meta property="og:image:height" content="200">', + ] diff --git a/Doc/constraints.txt b/Doc/constraints.txt index 54888eaab242ee..29cd4be1d3c8db 100644 --- a/Doc/constraints.txt +++ b/Doc/constraints.txt @@ -7,19 +7,18 @@ # Direct dependencies of Sphinx babel<3 colorama<0.5 -imagesize<1.5 -Jinja2<3.2 -packaging<24 -# Pygments==2.15.0 breaks CI -Pygments<2.16,!=2.15.0 +imagesize<2 +Jinja2<4 +packaging<25 +Pygments<3 requests<3 snowballstemmer<3 -sphinxcontrib-applehelp<1.1 -sphinxcontrib-devhelp<1.1 -sphinxcontrib-htmlhelp<2.1 -sphinxcontrib-jsmath<1.1 -sphinxcontrib-qthelp<1.1 -sphinxcontrib-serializinghtml<1.2 +sphinxcontrib-applehelp<3 +sphinxcontrib-devhelp<3 +sphinxcontrib-htmlhelp<3 +sphinxcontrib-jsmath<2 +sphinxcontrib-qthelp<3 +sphinxcontrib-serializinghtml<3 # Direct dependencies of Jinja2 (Jinja is a dependency of Sphinx, see above) -MarkupSafe<2.2 +MarkupSafe<3 diff --git a/Doc/contents.rst b/Doc/contents.rst index 464f93bdf85f95..b57f4b09a5dcb6 100644 --- a/Doc/contents.rst +++ b/Doc/contents.rst @@ -11,20 +11,13 @@ library/index.rst extending/index.rst c-api/index.rst - distributing/index.rst installing/index.rst howto/index.rst faq/index.rst + deprecations/index.rst glossary.rst about.rst bugs.rst copyright.rst license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - install/index.rst diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 9b71683155eebe..9210d5f50ed841 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2023 Python Software Foundation. All rights reserved. +Copyright © 2001 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ef9ac1617a284b..59b31ccf7bc471 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -180,7 +180,7 @@ PyCapsule_IsValid:const char*:name:: PyCapsule_New:PyObject*::+1: PyCapsule_New:void*:pointer:: PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: +PyCapsule_New:void (*)(PyObject *):destructor:: PyCapsule_SetContext:int::: PyCapsule_SetContext:PyObject*:self:0: @@ -349,11 +349,11 @@ PyComplex_CheckExact:int::: PyComplex_CheckExact:PyObject*:p:0: PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: +PyComplex_FromCComplex:Py_complex:v:: PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: +PyComplex_FromDoubles:double:real:: +PyComplex_FromDoubles:double:imag:: PyComplex_ImagAsDouble:double::: PyComplex_ImagAsDouble:PyObject*:op:0: @@ -364,8 +364,6 @@ PyComplex_RealAsDouble:PyObject*:op:0: PyContext_CheckExact:int::: PyContext_CheckExact:PyObject*:o:0: -PyContext_ClearFreeList:int::: - PyContext_Copy:PyObject*::+1: PyContext_Copy:PyObject*:ctx:0: @@ -402,6 +400,21 @@ PyContextVar_Reset:int::: PyContextVar_Reset:PyObject*:var:0: PyContextVar_Reset:PyObject*:token:-1: +PyCFunction_New:PyObject*::+1: +PyCFunction_New:PyMethodDef*:ml:: +PyCFunction_New:PyObject*:self:+1: + +PyCFunction_NewEx:PyObject*::+1: +PyCFunction_NewEx:PyMethodDef*:ml:: +PyCFunction_NewEx:PyObject*:self:+1: +PyCFunction_NewEx:PyObject*:module:+1: + +PyCMethod_New:PyObject*::+1: +PyCMethod_New:PyMethodDef*:ml:: +PyCMethod_New:PyObject*:self:+1: +PyCMethod_New:PyObject*:module:+1: +PyCMethod_New:PyObject*:cls:+1: + PyDate_Check:int::: PyDate_Check:PyObject*:ob:0: @@ -607,7 +620,9 @@ PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: PyErr_GetRaisedException:PyObject*::+1: -PyErr_SetRaisedException:::: + +PyErr_SetRaisedException:void::: +PyErr_SetRaisedException:PyObject *:exc:0:stolen PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: @@ -627,9 +642,9 @@ PyErr_NewExceptionWithDoc:PyObject*:dict:0: PyErr_NoMemory:PyObject*::null: PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? +PyErr_NormalizeException:PyObject**:exc:+1:??? +PyErr_NormalizeException:PyObject**:val:+1:??? +PyErr_NormalizeException:PyObject**:tb:+1:??? PyErr_Occurred:PyObject*::0: @@ -775,6 +790,12 @@ PyEval_GetGlobals:PyObject*::0: PyEval_GetFrame:PyObject*::0: +PyEval_GetFrameBuiltins:PyObject*::+1: + +PyEval_GetFrameLocals:PyObject*::+1: + +PyEval_GetFrameGlobals:PyObject*::+1: + PyEval_GetFuncDesc:const char*::: PyEval_GetFuncDesc:PyObject*:func:0: @@ -901,6 +922,32 @@ PyFloat_FromString:PyObject*:str:0: PyFloat_GetInfo:PyObject*::+1: PyFloat_GetInfo::void:: +PyFrame_GetBack:PyObject*::+1: +PyFrame_GetBack:PyFrameObject*:frame:0: + +PyFrame_GetBuiltins:PyObject*::+1: +PyFrame_GetBuiltins:PyFrameObject*:frame:0: + +PyFrame_GetCode:PyObject*::+1: +PyFrame_GetCode:PyFrameObject*:frame:0: + +PyFrame_GetGenerator:PyObject*::+1: +PyFrame_GetGenerator:PyFrameObject*:frame:0: + +PyFrame_GetGlobals:PyObject*::+1: +PyFrame_GetGlobals:PyFrameObject*:frame:0: + +PyFrame_GetLocals:PyObject*::+1: +PyFrame_GetLocals:PyFrameObject*:frame:0: + +PyFrame_GetVar:PyObject*::+1: +PyFrame_GetVar:PyFrameObject*:frame:0: +PyFrame_GetVar:PyObject*:name:0: + +PyFrame_GetVarString:PyObject*::+1: +PyFrame_GetVarString:PyFrameObject*:frame:0: +PyFrame_GetVarString:const char*:name:: + PyFrozenSet_Check:int::: PyFrozenSet_Check:PyObject*:p:0: @@ -983,8 +1030,6 @@ PyImport_AddModule:const char*:name:: PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules PyImport_AddModuleObject:PyObject*:name:0: -PyImport_Cleanup:void::: - PyImport_ExecCodeModule:PyObject*::+1: PyImport_ExecCodeModule:const char*:name:: PyImport_ExecCodeModule:PyObject*:co:0: @@ -1048,9 +1093,6 @@ PyImport_ImportModuleLevelObject:PyObject*:locals:0:??? PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:??? PyImport_ImportModuleLevelObject:int:level:: -PyImport_ImportModuleNoBlock:PyObject*::+1: -PyImport_ImportModuleNoBlock:const char*:name:: - PyImport_ReloadModule:PyObject*::+1: PyImport_ReloadModule:PyObject*:m:0: @@ -1089,6 +1131,10 @@ PyAIter_Check:PyObject*:o:0: PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: +PyIter_NextItem:int::: +PyIter_NextItem:PyObject*:iter:0: +PyIter_NextItem:PyObject**:item:+1: + PyIter_Send:int::: PyIter_Send:PyObject*:iter:0: PyIter_Send:PyObject*:arg:0: @@ -1118,6 +1164,10 @@ PyList_GetItem:PyObject*::0: PyList_GetItem:PyObject*:list:0: PyList_GetItem:Py_ssize_t:index:: +PyList_GetItemRef:PyObject*::+1: +PyList_GetItemRef:PyObject*:list:0: +PyList_GetItemRef:Py_ssize_t:index:: + PyList_GetSlice:PyObject*::+1: PyList_GetSlice:PyObject*:list:0: PyList_GetSlice:Py_ssize_t:low:: @@ -1233,6 +1283,26 @@ PyLong_FromUnsignedLong:unsignedlong:v:: PyLong_FromVoidPtr:PyObject*::+1: PyLong_FromVoidPtr:void*:p:: +PyLong_IsPositive:int::: +PyLong_IsPositive:PyObject*:obj:0: + +PyLong_IsNegative:int::: +PyLong_IsNegative:PyObject*:obj:0: + +PyLong_IsZero:int::: +PyLong_IsZero:PyObject*:obj:0: + +PyLong_GetSign:int::: +PyLong_GetSign:PyObject*:v:0: +PyLong_GetSign:int*:sign:: + +PyLong_Export:int::: +PyLong_Export:PyObject*:obj:0: +PyLong_Export:PyLongExport*:export_long:: + +PyLongWriter_Finish:PyObject*::+1: +PyLongWriter_Finish:PyLongWriter*:writer:: + PyMapping_Check:int::: PyMapping_Check:PyObject*:o:0: @@ -1250,7 +1320,7 @@ PyMapping_GetItemString:const char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: +PyMapping_HasKey:PyObject*:key:0: PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: @@ -1410,7 +1480,7 @@ PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: PyModule_New:PyObject*::+1: -PyModule_New::char* name:: +PyModule_New:char*:name:: PyModule_NewObject:PyObject*::+1: PyModule_NewObject:PyObject*:name:+1: @@ -1420,7 +1490,7 @@ PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:0: +PyModuleDef_Init:PyModuleDef*:def:: PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -1589,6 +1659,13 @@ PyObject_Call:PyObject*:callable_object:0: PyObject_Call:PyObject*:args:0: PyObject_Call:PyObject*:kw:0: +PyObject_CallNoArgs:PyObject*::+1: +PyObject_CallNoArgs:PyObject*:callable_object:0: + +PyObject_CallOneArg:PyObject*::+1: +PyObject_CallOneArg:PyObject*:callable_object:0: +PyObject_CallOneArg:PyObject*:arg:0: + PyObject_CallFunction:PyObject*::+1: PyObject_CallFunction:PyObject*:callable_object:0: PyObject_CallFunction:const char*:format:: @@ -1776,6 +1853,9 @@ PyObject_RichCompareBool:PyObject*:o1:0: PyObject_RichCompareBool:PyObject*:o2:0: PyObject_RichCompareBool:int:opid:: +PyObject_SelfIter:PyObject*::+1: +PyObject_SelfIter:PyObject*:obj:0: + PyObject_SetAttr:int::: PyObject_SetAttr:PyObject*:o:0: PyObject_SetAttr:PyObject*:attr_name:0: @@ -1913,10 +1993,10 @@ PyRun_StringFlags:PyObject*:locals:0: PyRun_StringFlags:PyCompilerFlags*:flags:: PySeqIter_Check:int::: -PySeqIter_Check::op:: +PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: +PySeqIter_New:PyObject*:seq:0: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2347,16 +2427,10 @@ PyUnicode_DATA:PyObject*:o:0: PyUnicode_GET_LENGTH:Py_ssize_t::: PyUnicode_GET_LENGTH:PyObject*:o:0: -PyUnicode_GET_SIZE:Py_ssize_t::: -PyUnicode_GET_SIZE:PyObject*:o:0: - -PyUnicode_GET_DATA_SIZE:Py_ssize_t::: -PyUnicode_GET_DATA_SIZE:PyObject*:o:0: - PyUnicode_KIND:int::: PyUnicode_KIND:PyObject*:o:0: -PyUnicode_MAX_CHAR_VALUE:::: +PyUnicode_MAX_CHAR_VALUE:Py_UCS4::: PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: Py_UNICODE_ISALNUM:int::: @@ -2423,7 +2497,7 @@ PyUnicode_FromWideChar:const wchar_t*:w:: PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: +PyUnicode_AsWideChar:PyObject*:unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: PyUnicode_AsWideChar:Py_ssize_t:size:: @@ -2476,7 +2550,7 @@ PyUnicode_AsUTF8String:PyObject*:unicode:0: PyUnicode_AsUTF8AndSize:const char*::: PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: -PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: +PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:: PyUnicode_AsUTF8:const char*::: PyUnicode_AsUTF8:PyObject*:unicode:0: @@ -2559,6 +2633,13 @@ PyUnicode_DecodeMBCSStateful:Py_ssize_t:size:: PyUnicode_DecodeMBCSStateful:const char*:errors:: PyUnicode_DecodeMBCSStateful:Py_ssize_t*:consumed:: +PyUnicode_DecodeCodePageStateful:PyObject*::+1: +PyUnicode_DecodeCodePageStateful:int:code_page:: +PyUnicode_DecodeCodePageStateful:const char*:s:: +PyUnicode_DecodeCodePageStateful:Py_ssize_t:size:: +PyUnicode_DecodeCodePageStateful:const char*:errors:: +PyUnicode_DecodeCodePageStateful:Py_ssize_t*:consumed:: + PyUnicode_EncodeCodePage:PyObject*::+1: PyUnicode_EncodeCodePage:int:code_page:: PyUnicode_EncodeCodePage:PyObject*:unicode:0: @@ -2571,13 +2652,26 @@ PyUnicode_Concat:PyObject*::+1: PyUnicode_Concat:PyObject*:left:0: PyUnicode_Concat:PyObject*:right:0: +PyUnicode_Partition:PyObject*::+1: +PyUnicode_Partition:PyObject*:unicode:0: +PyUnicode_Partition:PyObject*:sep:0: + +PyUnicode_RPartition:PyObject*::+1: +PyUnicode_RPartition:PyObject*:unicode:0: +PyUnicode_RPartition:PyObject*:sep:0: + +PyUnicode_RSplit:PyObject*::+1: +PyUnicode_RSplit:PyObject*:unicode:0: +PyUnicode_RSplit:PyObject*:sep:0: +PyUnicode_RSplit:Py_ssize_t:maxsplit:: + PyUnicode_Split:PyObject*::+1: -PyUnicode_Split:PyObject*:left:0: -PyUnicode_Split:PyObject*:right:0: +PyUnicode_Split:PyObject*:unicode:0: +PyUnicode_Split:PyObject*:sep:0: PyUnicode_Split:Py_ssize_t:maxsplit:: PyUnicode_Splitlines:PyObject*::+1: -PyUnicode_Splitlines:PyObject*:s:0: +PyUnicode_Splitlines:PyObject*:unicode:0: PyUnicode_Splitlines:int:keepend:: PyUnicode_Translate:PyObject*::+1: @@ -2673,6 +2767,23 @@ PyUnicode_FromFormatV:PyObject*::+1: PyUnicode_FromFormatV:const char*:format:: PyUnicode_FromFormatV:va_list:args:: +PyUnicode_FromOrdinal:PyObject*::+1: +PyUnicode_FromOrdinal:int:ordinal:: + +PyUnicode_Append:void::: +PyUnicode_Append:PyObject**:p_left:0: +PyUnicode_Append:PyObject*:right:: + +PyUnicode_AppendAndDel:void::: +PyUnicode_AppendAndDel:PyObject**:p_left:0: +PyUnicode_AppendAndDel:PyObject*:right:-1: + +PyUnicode_BuildEncodingMap:PyObject*::+1: +PyUnicode_BuildEncodingMap:PyObject*:string::: + +PyUnicode_GetDefaultEncoding:const char*::: +PyUnicode_GetDefaultEncoding::void:: + PyUnicode_GetLength:Py_ssize_t::: PyUnicode_GetLength:PyObject*:unicode:0: @@ -2683,6 +2794,10 @@ PyUnicode_CopyCharacters:PyObject*:from:0: PyUnicode_CopyCharacters:Py_ssize_t:from_start:: PyUnicode_CopyCharacters:Py_ssize_t:how_many:: +PyUnicode_Resize:int::: +PyUnicode_Resize:PyObject**:unicode:0: +PyUnicode_Resize:Py_ssize_t:length:: + PyUnicode_Fill:Py_ssize_t::: PyUnicode_Fill:PyObject*:unicode:0: PyUnicode_Fill:Py_ssize_t:start:: @@ -2799,13 +2914,13 @@ PyUnicodeDecodeError_SetStart:PyObject*:exc:0: PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: PyWeakref_Check:int::: -PyWeakref_Check:PyObject*:ob:: +PyWeakref_Check:PyObject*:ob:0: PyWeakref_CheckProxy:int::: -PyWeakref_CheckProxy:PyObject*:ob:: +PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: -PyWeakref_CheckRef:PyObject*:ob:: +PyWeakref_CheckRef:PyObject*:ob:0: PyWeakref_GET_OBJECT:PyObject*::0: PyWeakref_GET_OBJECT:PyObject*:ref:0: @@ -2892,18 +3007,8 @@ Py_GetCompiler:const char*::: Py_GetCopyright:const char*::: -Py_GetExecPrefix:wchar_t*::: - -Py_GetPath:wchar_t*::: - Py_GetPlatform:const char*::: -Py_GetPrefix:wchar_t*::: - -Py_GetProgramFullPath:wchar_t*::: - -Py_GetProgramName:wchar_t*::: - Py_GetVersion:const char*::: Py_INCREF:void::: @@ -2975,3 +3080,11 @@ _Py_c_quot:Py_complex:divisor:: _Py_c_sum:Py_complex::: _Py_c_sum:Py_complex:left:: _Py_c_sum:Py_complex:right:: + +PyImport_ImportModuleAttr:PyObject*::+1: +PyImport_ImportModuleAttr:PyObject*:mod_name:0: +PyImport_ImportModuleAttr:PyObject*:attr_name:0: + +PyImport_ImportModuleAttrString:PyObject*::+1: +PyImport_ImportModuleAttrString:const char *:mod_name:: +PyImport_ImportModuleAttrString:const char *:attr_name:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index ed415a4dc644a4..0d0dfb3843260e 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,857 +1,899 @@ role,name,added,ifdef_note,struct_abi_kind macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, -function,PyAIter_Check,3.10,, -function,PyArg_Parse,3.2,, -function,PyArg_ParseTuple,3.2,, -function,PyArg_ParseTupleAndKeywords,3.2,, -function,PyArg_UnpackTuple,3.2,, -function,PyArg_VaParse,3.2,, -function,PyArg_VaParseTupleAndKeywords,3.2,, -function,PyArg_ValidateKeywordArguments,3.2,, -var,PyBaseObject_Type,3.2,, -function,PyBool_FromLong,3.2,, -var,PyBool_Type,3.2,, -function,PyBuffer_FillContiguousStrides,3.11,, -function,PyBuffer_FillInfo,3.11,, -function,PyBuffer_FromContiguous,3.11,, -function,PyBuffer_GetPointer,3.11,, -function,PyBuffer_IsContiguous,3.11,, -function,PyBuffer_Release,3.11,, -function,PyBuffer_SizeFromFormat,3.11,, -function,PyBuffer_ToContiguous,3.11,, -var,PyByteArrayIter_Type,3.2,, -function,PyByteArray_AsString,3.2,, -function,PyByteArray_Concat,3.2,, -function,PyByteArray_FromObject,3.2,, -function,PyByteArray_FromStringAndSize,3.2,, -function,PyByteArray_Resize,3.2,, -function,PyByteArray_Size,3.2,, -var,PyByteArray_Type,3.2,, -var,PyBytesIter_Type,3.2,, -function,PyBytes_AsString,3.2,, -function,PyBytes_AsStringAndSize,3.2,, -function,PyBytes_Concat,3.2,, -function,PyBytes_ConcatAndDel,3.2,, -function,PyBytes_DecodeEscape,3.2,, -function,PyBytes_FromFormat,3.2,, -function,PyBytes_FromFormatV,3.2,, -function,PyBytes_FromObject,3.2,, -function,PyBytes_FromString,3.2,, -function,PyBytes_FromStringAndSize,3.2,, -function,PyBytes_Repr,3.2,, -function,PyBytes_Size,3.2,, -var,PyBytes_Type,3.2,, +func,PyAIter_Check,3.10,, +func,PyArg_Parse,3.2,, +func,PyArg_ParseTuple,3.2,, +func,PyArg_ParseTupleAndKeywords,3.2,, +func,PyArg_UnpackTuple,3.2,, +func,PyArg_VaParse,3.2,, +func,PyArg_VaParseTupleAndKeywords,3.2,, +func,PyArg_ValidateKeywordArguments,3.2,, +data,PyBaseObject_Type,3.2,, +func,PyBool_FromLong,3.2,, +data,PyBool_Type,3.2,, +func,PyBuffer_FillContiguousStrides,3.11,, +func,PyBuffer_FillInfo,3.11,, +func,PyBuffer_FromContiguous,3.11,, +func,PyBuffer_GetPointer,3.11,, +func,PyBuffer_IsContiguous,3.11,, +func,PyBuffer_Release,3.11,, +func,PyBuffer_SizeFromFormat,3.11,, +func,PyBuffer_ToContiguous,3.11,, +data,PyByteArrayIter_Type,3.2,, +func,PyByteArray_AsString,3.2,, +func,PyByteArray_Concat,3.2,, +func,PyByteArray_FromObject,3.2,, +func,PyByteArray_FromStringAndSize,3.2,, +func,PyByteArray_Resize,3.2,, +func,PyByteArray_Size,3.2,, +data,PyByteArray_Type,3.2,, +data,PyBytesIter_Type,3.2,, +func,PyBytes_AsString,3.2,, +func,PyBytes_AsStringAndSize,3.2,, +func,PyBytes_Concat,3.2,, +func,PyBytes_ConcatAndDel,3.2,, +func,PyBytes_DecodeEscape,3.2,, +func,PyBytes_FromFormat,3.2,, +func,PyBytes_FromFormatV,3.2,, +func,PyBytes_FromObject,3.2,, +func,PyBytes_FromString,3.2,, +func,PyBytes_FromStringAndSize,3.2,, +func,PyBytes_Repr,3.2,, +func,PyBytes_Size,3.2,, +data,PyBytes_Type,3.2,, type,PyCFunction,3.2,, +type,PyCFunctionFast,3.13,, +type,PyCFunctionFastWithKeywords,3.13,, type,PyCFunctionWithKeywords,3.2,, -function,PyCFunction_GetFlags,3.2,, -function,PyCFunction_GetFunction,3.2,, -function,PyCFunction_GetSelf,3.2,, -function,PyCFunction_New,3.4,, -function,PyCFunction_NewEx,3.2,, -var,PyCFunction_Type,3.2,, -function,PyCMethod_New,3.9,, -function,PyCallIter_New,3.2,, -var,PyCallIter_Type,3.2,, -function,PyCallable_Check,3.2,, +func,PyCFunction_GetFlags,3.2,, +func,PyCFunction_GetFunction,3.2,, +func,PyCFunction_GetSelf,3.2,, +func,PyCFunction_New,3.4,, +func,PyCFunction_NewEx,3.2,, +data,PyCFunction_Type,3.2,, +func,PyCMethod_New,3.9,, +func,PyCallIter_New,3.2,, +data,PyCallIter_Type,3.2,, +func,PyCallable_Check,3.2,, type,PyCapsule_Destructor,3.2,, -function,PyCapsule_GetContext,3.2,, -function,PyCapsule_GetDestructor,3.2,, -function,PyCapsule_GetName,3.2,, -function,PyCapsule_GetPointer,3.2,, -function,PyCapsule_Import,3.2,, -function,PyCapsule_IsValid,3.2,, -function,PyCapsule_New,3.2,, -function,PyCapsule_SetContext,3.2,, -function,PyCapsule_SetDestructor,3.2,, -function,PyCapsule_SetName,3.2,, -function,PyCapsule_SetPointer,3.2,, -var,PyCapsule_Type,3.2,, -var,PyClassMethodDescr_Type,3.2,, -function,PyCodec_BackslashReplaceErrors,3.2,, -function,PyCodec_Decode,3.2,, -function,PyCodec_Decoder,3.2,, -function,PyCodec_Encode,3.2,, -function,PyCodec_Encoder,3.2,, -function,PyCodec_IgnoreErrors,3.2,, -function,PyCodec_IncrementalDecoder,3.2,, -function,PyCodec_IncrementalEncoder,3.2,, -function,PyCodec_KnownEncoding,3.2,, -function,PyCodec_LookupError,3.2,, -function,PyCodec_NameReplaceErrors,3.7,, -function,PyCodec_Register,3.2,, -function,PyCodec_RegisterError,3.2,, -function,PyCodec_ReplaceErrors,3.2,, -function,PyCodec_StreamReader,3.2,, -function,PyCodec_StreamWriter,3.2,, -function,PyCodec_StrictErrors,3.2,, -function,PyCodec_Unregister,3.10,, -function,PyCodec_XMLCharRefReplaceErrors,3.2,, -function,PyComplex_FromDoubles,3.2,, -function,PyComplex_ImagAsDouble,3.2,, -function,PyComplex_RealAsDouble,3.2,, -var,PyComplex_Type,3.2,, -function,PyDescr_NewClassMethod,3.2,, -function,PyDescr_NewGetSet,3.2,, -function,PyDescr_NewMember,3.2,, -function,PyDescr_NewMethod,3.2,, -var,PyDictItems_Type,3.2,, -var,PyDictIterItem_Type,3.2,, -var,PyDictIterKey_Type,3.2,, -var,PyDictIterValue_Type,3.2,, -var,PyDictKeys_Type,3.2,, -function,PyDictProxy_New,3.2,, -var,PyDictProxy_Type,3.2,, -var,PyDictRevIterItem_Type,3.8,, -var,PyDictRevIterKey_Type,3.8,, -var,PyDictRevIterValue_Type,3.8,, -var,PyDictValues_Type,3.2,, -function,PyDict_Clear,3.2,, -function,PyDict_Contains,3.2,, -function,PyDict_Copy,3.2,, -function,PyDict_DelItem,3.2,, -function,PyDict_DelItemString,3.2,, -function,PyDict_GetItem,3.2,, -function,PyDict_GetItemRef,3.13,, -function,PyDict_GetItemString,3.2,, -function,PyDict_GetItemStringRef,3.13,, -function,PyDict_GetItemWithError,3.2,, -function,PyDict_Items,3.2,, -function,PyDict_Keys,3.2,, -function,PyDict_Merge,3.2,, -function,PyDict_MergeFromSeq2,3.2,, -function,PyDict_New,3.2,, -function,PyDict_Next,3.2,, -function,PyDict_SetItem,3.2,, -function,PyDict_SetItemString,3.2,, -function,PyDict_Size,3.2,, -var,PyDict_Type,3.2,, -function,PyDict_Update,3.2,, -function,PyDict_Values,3.2,, -var,PyEllipsis_Type,3.2,, -var,PyEnum_Type,3.2,, -function,PyErr_BadArgument,3.2,, -function,PyErr_BadInternalCall,3.2,, -function,PyErr_CheckSignals,3.2,, -function,PyErr_Clear,3.2,, -function,PyErr_Display,3.2,, -function,PyErr_DisplayException,3.12,, -function,PyErr_ExceptionMatches,3.2,, -function,PyErr_Fetch,3.2,, -function,PyErr_Format,3.2,, -function,PyErr_FormatV,3.5,, -function,PyErr_GetExcInfo,3.7,, -function,PyErr_GetHandledException,3.11,, -function,PyErr_GetRaisedException,3.12,, -function,PyErr_GivenExceptionMatches,3.2,, -function,PyErr_NewException,3.2,, -function,PyErr_NewExceptionWithDoc,3.2,, -function,PyErr_NoMemory,3.2,, -function,PyErr_NormalizeException,3.2,, -function,PyErr_Occurred,3.2,, -function,PyErr_Print,3.2,, -function,PyErr_PrintEx,3.2,, -function,PyErr_ProgramText,3.2,, -function,PyErr_ResourceWarning,3.6,, -function,PyErr_Restore,3.2,, -function,PyErr_SetExcFromWindowsErr,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, -function,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, -function,PyErr_SetExcInfo,3.7,, -function,PyErr_SetFromErrno,3.2,, -function,PyErr_SetFromErrnoWithFilename,3.2,, -function,PyErr_SetFromErrnoWithFilenameObject,3.2,, -function,PyErr_SetFromErrnoWithFilenameObjects,3.7,, -function,PyErr_SetFromWindowsErr,3.7,on Windows, -function,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, -function,PyErr_SetHandledException,3.11,, -function,PyErr_SetImportError,3.7,, -function,PyErr_SetImportErrorSubclass,3.6,, -function,PyErr_SetInterrupt,3.2,, -function,PyErr_SetInterruptEx,3.10,, -function,PyErr_SetNone,3.2,, -function,PyErr_SetObject,3.2,, -function,PyErr_SetRaisedException,3.12,, -function,PyErr_SetString,3.2,, -function,PyErr_SyntaxLocation,3.2,, -function,PyErr_SyntaxLocationEx,3.7,, -function,PyErr_WarnEx,3.2,, -function,PyErr_WarnExplicit,3.2,, -function,PyErr_WarnFormat,3.2,, -function,PyErr_WriteUnraisable,3.2,, -function,PyEval_AcquireThread,3.2,, -function,PyEval_EvalCode,3.2,, -function,PyEval_EvalCodeEx,3.2,, -function,PyEval_EvalFrame,3.2,, -function,PyEval_EvalFrameEx,3.2,, -function,PyEval_GetBuiltins,3.2,, -function,PyEval_GetFrame,3.2,, -function,PyEval_GetFuncDesc,3.2,, -function,PyEval_GetFuncName,3.2,, -function,PyEval_GetGlobals,3.2,, -function,PyEval_GetLocals,3.2,, -function,PyEval_ReleaseThread,3.2,, -function,PyEval_RestoreThread,3.2,, -function,PyEval_SaveThread,3.2,, -var,PyExc_ArithmeticError,3.2,, -var,PyExc_AssertionError,3.2,, -var,PyExc_AttributeError,3.2,, -var,PyExc_BaseException,3.2,, -var,PyExc_BaseExceptionGroup,3.11,, -var,PyExc_BlockingIOError,3.7,, -var,PyExc_BrokenPipeError,3.7,, -var,PyExc_BufferError,3.2,, -var,PyExc_BytesWarning,3.2,, -var,PyExc_ChildProcessError,3.7,, -var,PyExc_ConnectionAbortedError,3.7,, -var,PyExc_ConnectionError,3.7,, -var,PyExc_ConnectionRefusedError,3.7,, -var,PyExc_ConnectionResetError,3.7,, -var,PyExc_DeprecationWarning,3.2,, -var,PyExc_EOFError,3.2,, -var,PyExc_EncodingWarning,3.10,, -var,PyExc_EnvironmentError,3.2,, -var,PyExc_Exception,3.2,, -var,PyExc_FileExistsError,3.7,, -var,PyExc_FileNotFoundError,3.7,, -var,PyExc_FloatingPointError,3.2,, -var,PyExc_FutureWarning,3.2,, -var,PyExc_GeneratorExit,3.2,, -var,PyExc_IOError,3.2,, -var,PyExc_ImportError,3.2,, -var,PyExc_ImportWarning,3.2,, -var,PyExc_IndentationError,3.2,, -var,PyExc_IndexError,3.2,, -var,PyExc_InterruptedError,3.7,, -var,PyExc_IsADirectoryError,3.7,, -var,PyExc_KeyError,3.2,, -var,PyExc_KeyboardInterrupt,3.2,, -var,PyExc_LookupError,3.2,, -var,PyExc_MemoryError,3.2,, -var,PyExc_ModuleNotFoundError,3.6,, -var,PyExc_NameError,3.2,, -var,PyExc_NotADirectoryError,3.7,, -var,PyExc_NotImplementedError,3.2,, -var,PyExc_OSError,3.2,, -var,PyExc_OverflowError,3.2,, -var,PyExc_PendingDeprecationWarning,3.2,, -var,PyExc_PermissionError,3.7,, -var,PyExc_ProcessLookupError,3.7,, -var,PyExc_RecursionError,3.7,, -var,PyExc_ReferenceError,3.2,, -var,PyExc_ResourceWarning,3.7,, -var,PyExc_RuntimeError,3.2,, -var,PyExc_RuntimeWarning,3.2,, -var,PyExc_StopAsyncIteration,3.7,, -var,PyExc_StopIteration,3.2,, -var,PyExc_SyntaxError,3.2,, -var,PyExc_SyntaxWarning,3.2,, -var,PyExc_SystemError,3.2,, -var,PyExc_SystemExit,3.2,, -var,PyExc_TabError,3.2,, -var,PyExc_TimeoutError,3.7,, -var,PyExc_TypeError,3.2,, -var,PyExc_UnboundLocalError,3.2,, -var,PyExc_UnicodeDecodeError,3.2,, -var,PyExc_UnicodeEncodeError,3.2,, -var,PyExc_UnicodeError,3.2,, -var,PyExc_UnicodeTranslateError,3.2,, -var,PyExc_UnicodeWarning,3.2,, -var,PyExc_UserWarning,3.2,, -var,PyExc_ValueError,3.2,, -var,PyExc_Warning,3.2,, -var,PyExc_WindowsError,3.7,on Windows, -var,PyExc_ZeroDivisionError,3.2,, -function,PyExceptionClass_Name,3.8,, -function,PyException_GetArgs,3.12,, -function,PyException_GetCause,3.2,, -function,PyException_GetContext,3.2,, -function,PyException_GetTraceback,3.2,, -function,PyException_SetArgs,3.12,, -function,PyException_SetCause,3.2,, -function,PyException_SetContext,3.2,, -function,PyException_SetTraceback,3.2,, -function,PyFile_FromFd,3.2,, -function,PyFile_GetLine,3.2,, -function,PyFile_WriteObject,3.2,, -function,PyFile_WriteString,3.2,, -var,PyFilter_Type,3.2,, -function,PyFloat_AsDouble,3.2,, -function,PyFloat_FromDouble,3.2,, -function,PyFloat_FromString,3.2,, -function,PyFloat_GetInfo,3.2,, -function,PyFloat_GetMax,3.2,, -function,PyFloat_GetMin,3.2,, -var,PyFloat_Type,3.2,, +func,PyCapsule_GetContext,3.2,, +func,PyCapsule_GetDestructor,3.2,, +func,PyCapsule_GetName,3.2,, +func,PyCapsule_GetPointer,3.2,, +func,PyCapsule_Import,3.2,, +func,PyCapsule_IsValid,3.2,, +func,PyCapsule_New,3.2,, +func,PyCapsule_SetContext,3.2,, +func,PyCapsule_SetDestructor,3.2,, +func,PyCapsule_SetName,3.2,, +func,PyCapsule_SetPointer,3.2,, +data,PyCapsule_Type,3.2,, +data,PyClassMethodDescr_Type,3.2,, +func,PyCodec_BackslashReplaceErrors,3.2,, +func,PyCodec_Decode,3.2,, +func,PyCodec_Decoder,3.2,, +func,PyCodec_Encode,3.2,, +func,PyCodec_Encoder,3.2,, +func,PyCodec_IgnoreErrors,3.2,, +func,PyCodec_IncrementalDecoder,3.2,, +func,PyCodec_IncrementalEncoder,3.2,, +func,PyCodec_KnownEncoding,3.2,, +func,PyCodec_LookupError,3.2,, +func,PyCodec_NameReplaceErrors,3.7,, +func,PyCodec_Register,3.2,, +func,PyCodec_RegisterError,3.2,, +func,PyCodec_ReplaceErrors,3.2,, +func,PyCodec_StreamReader,3.2,, +func,PyCodec_StreamWriter,3.2,, +func,PyCodec_StrictErrors,3.2,, +func,PyCodec_Unregister,3.10,, +func,PyCodec_XMLCharRefReplaceErrors,3.2,, +func,PyComplex_FromDoubles,3.2,, +func,PyComplex_ImagAsDouble,3.2,, +func,PyComplex_RealAsDouble,3.2,, +data,PyComplex_Type,3.2,, +func,PyDescr_NewClassMethod,3.2,, +func,PyDescr_NewGetSet,3.2,, +func,PyDescr_NewMember,3.2,, +func,PyDescr_NewMethod,3.2,, +data,PyDictItems_Type,3.2,, +data,PyDictIterItem_Type,3.2,, +data,PyDictIterKey_Type,3.2,, +data,PyDictIterValue_Type,3.2,, +data,PyDictKeys_Type,3.2,, +func,PyDictProxy_New,3.2,, +data,PyDictProxy_Type,3.2,, +data,PyDictRevIterItem_Type,3.8,, +data,PyDictRevIterKey_Type,3.8,, +data,PyDictRevIterValue_Type,3.8,, +data,PyDictValues_Type,3.2,, +func,PyDict_Clear,3.2,, +func,PyDict_Contains,3.2,, +func,PyDict_Copy,3.2,, +func,PyDict_DelItem,3.2,, +func,PyDict_DelItemString,3.2,, +func,PyDict_GetItem,3.2,, +func,PyDict_GetItemRef,3.13,, +func,PyDict_GetItemString,3.2,, +func,PyDict_GetItemStringRef,3.13,, +func,PyDict_GetItemWithError,3.2,, +func,PyDict_Items,3.2,, +func,PyDict_Keys,3.2,, +func,PyDict_Merge,3.2,, +func,PyDict_MergeFromSeq2,3.2,, +func,PyDict_New,3.2,, +func,PyDict_Next,3.2,, +func,PyDict_SetItem,3.2,, +func,PyDict_SetItemString,3.2,, +func,PyDict_Size,3.2,, +data,PyDict_Type,3.2,, +func,PyDict_Update,3.2,, +func,PyDict_Values,3.2,, +data,PyEllipsis_Type,3.2,, +data,PyEnum_Type,3.2,, +func,PyErr_BadArgument,3.2,, +func,PyErr_BadInternalCall,3.2,, +func,PyErr_CheckSignals,3.2,, +func,PyErr_Clear,3.2,, +func,PyErr_Display,3.2,, +func,PyErr_DisplayException,3.12,, +func,PyErr_ExceptionMatches,3.2,, +func,PyErr_Fetch,3.2,, +func,PyErr_Format,3.2,, +func,PyErr_FormatV,3.5,, +func,PyErr_GetExcInfo,3.7,, +func,PyErr_GetHandledException,3.11,, +func,PyErr_GetRaisedException,3.12,, +func,PyErr_GivenExceptionMatches,3.2,, +func,PyErr_NewException,3.2,, +func,PyErr_NewExceptionWithDoc,3.2,, +func,PyErr_NoMemory,3.2,, +func,PyErr_NormalizeException,3.2,, +func,PyErr_Occurred,3.2,, +func,PyErr_Print,3.2,, +func,PyErr_PrintEx,3.2,, +func,PyErr_ProgramText,3.2,, +func,PyErr_ResourceWarning,3.6,, +func,PyErr_Restore,3.2,, +func,PyErr_SetExcFromWindowsErr,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObject,3.7,on Windows, +func,PyErr_SetExcFromWindowsErrWithFilenameObjects,3.7,on Windows, +func,PyErr_SetExcInfo,3.7,, +func,PyErr_SetFromErrno,3.2,, +func,PyErr_SetFromErrnoWithFilename,3.2,, +func,PyErr_SetFromErrnoWithFilenameObject,3.2,, +func,PyErr_SetFromErrnoWithFilenameObjects,3.7,, +func,PyErr_SetFromWindowsErr,3.7,on Windows, +func,PyErr_SetFromWindowsErrWithFilename,3.7,on Windows, +func,PyErr_SetHandledException,3.11,, +func,PyErr_SetImportError,3.7,, +func,PyErr_SetImportErrorSubclass,3.6,, +func,PyErr_SetInterrupt,3.2,, +func,PyErr_SetInterruptEx,3.10,, +func,PyErr_SetNone,3.2,, +func,PyErr_SetObject,3.2,, +func,PyErr_SetRaisedException,3.12,, +func,PyErr_SetString,3.2,, +func,PyErr_SyntaxLocation,3.2,, +func,PyErr_SyntaxLocationEx,3.7,, +func,PyErr_WarnEx,3.2,, +func,PyErr_WarnExplicit,3.2,, +func,PyErr_WarnFormat,3.2,, +func,PyErr_WriteUnraisable,3.2,, +func,PyEval_AcquireThread,3.2,, +func,PyEval_EvalCode,3.2,, +func,PyEval_EvalCodeEx,3.2,, +func,PyEval_EvalFrame,3.2,, +func,PyEval_EvalFrameEx,3.2,, +func,PyEval_GetBuiltins,3.2,, +func,PyEval_GetFrame,3.2,, +func,PyEval_GetFrameBuiltins,3.13,, +func,PyEval_GetFrameGlobals,3.13,, +func,PyEval_GetFrameLocals,3.13,, +func,PyEval_GetFuncDesc,3.2,, +func,PyEval_GetFuncName,3.2,, +func,PyEval_GetGlobals,3.2,, +func,PyEval_GetLocals,3.2,, +func,PyEval_InitThreads,3.2,, +func,PyEval_ReleaseThread,3.2,, +func,PyEval_RestoreThread,3.2,, +func,PyEval_SaveThread,3.2,, +data,PyExc_ArithmeticError,3.2,, +data,PyExc_AssertionError,3.2,, +data,PyExc_AttributeError,3.2,, +data,PyExc_BaseException,3.2,, +data,PyExc_BaseExceptionGroup,3.11,, +data,PyExc_BlockingIOError,3.7,, +data,PyExc_BrokenPipeError,3.7,, +data,PyExc_BufferError,3.2,, +data,PyExc_BytesWarning,3.2,, +data,PyExc_ChildProcessError,3.7,, +data,PyExc_ConnectionAbortedError,3.7,, +data,PyExc_ConnectionError,3.7,, +data,PyExc_ConnectionRefusedError,3.7,, +data,PyExc_ConnectionResetError,3.7,, +data,PyExc_DeprecationWarning,3.2,, +data,PyExc_EOFError,3.2,, +data,PyExc_EncodingWarning,3.10,, +data,PyExc_EnvironmentError,3.2,, +data,PyExc_Exception,3.2,, +data,PyExc_FileExistsError,3.7,, +data,PyExc_FileNotFoundError,3.7,, +data,PyExc_FloatingPointError,3.2,, +data,PyExc_FutureWarning,3.2,, +data,PyExc_GeneratorExit,3.2,, +data,PyExc_IOError,3.2,, +data,PyExc_ImportError,3.2,, +data,PyExc_ImportWarning,3.2,, +data,PyExc_IndentationError,3.2,, +data,PyExc_IndexError,3.2,, +data,PyExc_InterruptedError,3.7,, +data,PyExc_IsADirectoryError,3.7,, +data,PyExc_KeyError,3.2,, +data,PyExc_KeyboardInterrupt,3.2,, +data,PyExc_LookupError,3.2,, +data,PyExc_MemoryError,3.2,, +data,PyExc_ModuleNotFoundError,3.6,, +data,PyExc_NameError,3.2,, +data,PyExc_NotADirectoryError,3.7,, +data,PyExc_NotImplementedError,3.2,, +data,PyExc_OSError,3.2,, +data,PyExc_OverflowError,3.2,, +data,PyExc_PendingDeprecationWarning,3.2,, +data,PyExc_PermissionError,3.7,, +data,PyExc_ProcessLookupError,3.7,, +data,PyExc_RecursionError,3.7,, +data,PyExc_ReferenceError,3.2,, +data,PyExc_ResourceWarning,3.7,, +data,PyExc_RuntimeError,3.2,, +data,PyExc_RuntimeWarning,3.2,, +data,PyExc_StopAsyncIteration,3.7,, +data,PyExc_StopIteration,3.2,, +data,PyExc_SyntaxError,3.2,, +data,PyExc_SyntaxWarning,3.2,, +data,PyExc_SystemError,3.2,, +data,PyExc_SystemExit,3.2,, +data,PyExc_TabError,3.2,, +data,PyExc_TimeoutError,3.7,, +data,PyExc_TypeError,3.2,, +data,PyExc_UnboundLocalError,3.2,, +data,PyExc_UnicodeDecodeError,3.2,, +data,PyExc_UnicodeEncodeError,3.2,, +data,PyExc_UnicodeError,3.2,, +data,PyExc_UnicodeTranslateError,3.2,, +data,PyExc_UnicodeWarning,3.2,, +data,PyExc_UserWarning,3.2,, +data,PyExc_ValueError,3.2,, +data,PyExc_Warning,3.2,, +data,PyExc_WindowsError,3.7,on Windows, +data,PyExc_ZeroDivisionError,3.2,, +func,PyExceptionClass_Name,3.8,, +func,PyException_GetArgs,3.12,, +func,PyException_GetCause,3.2,, +func,PyException_GetContext,3.2,, +func,PyException_GetTraceback,3.2,, +func,PyException_SetArgs,3.12,, +func,PyException_SetCause,3.2,, +func,PyException_SetContext,3.2,, +func,PyException_SetTraceback,3.2,, +func,PyFile_FromFd,3.2,, +func,PyFile_GetLine,3.2,, +func,PyFile_WriteObject,3.2,, +func,PyFile_WriteString,3.2,, +data,PyFilter_Type,3.2,, +func,PyFloat_AsDouble,3.2,, +func,PyFloat_FromDouble,3.2,, +func,PyFloat_FromString,3.2,, +func,PyFloat_GetInfo,3.2,, +func,PyFloat_GetMax,3.2,, +func,PyFloat_GetMin,3.2,, +data,PyFloat_Type,3.2,, type,PyFrameObject,3.2,,opaque -function,PyFrame_GetCode,3.10,, -function,PyFrame_GetLineNumber,3.10,, -function,PyFrozenSet_New,3.2,, -var,PyFrozenSet_Type,3.2,, -function,PyGC_Collect,3.2,, -function,PyGC_Disable,3.10,, -function,PyGC_Enable,3.10,, -function,PyGC_IsEnabled,3.10,, -function,PyGILState_Ensure,3.2,, -function,PyGILState_GetThisThreadState,3.2,, -function,PyGILState_Release,3.2,, +func,PyFrame_GetCode,3.10,, +func,PyFrame_GetLineNumber,3.10,, +func,PyFrozenSet_New,3.2,, +data,PyFrozenSet_Type,3.2,, +func,PyGC_Collect,3.2,, +func,PyGC_Disable,3.10,, +func,PyGC_Enable,3.10,, +func,PyGC_IsEnabled,3.10,, +func,PyGILState_Ensure,3.2,, +func,PyGILState_GetThisThreadState,3.2,, +func,PyGILState_Release,3.2,, type,PyGILState_STATE,3.2,, type,PyGetSetDef,3.2,,full-abi -var,PyGetSetDescr_Type,3.2,, -function,PyImport_AddModule,3.2,, -function,PyImport_AddModuleObject,3.7,, -function,PyImport_AddModuleRef,3.13,, -function,PyImport_AppendInittab,3.2,, -function,PyImport_ExecCodeModule,3.2,, -function,PyImport_ExecCodeModuleEx,3.2,, -function,PyImport_ExecCodeModuleObject,3.7,, -function,PyImport_ExecCodeModuleWithPathnames,3.2,, -function,PyImport_GetImporter,3.2,, -function,PyImport_GetMagicNumber,3.2,, -function,PyImport_GetMagicTag,3.2,, -function,PyImport_GetModule,3.8,, -function,PyImport_GetModuleDict,3.2,, -function,PyImport_Import,3.2,, -function,PyImport_ImportFrozenModule,3.2,, -function,PyImport_ImportFrozenModuleObject,3.7,, -function,PyImport_ImportModule,3.2,, -function,PyImport_ImportModuleLevel,3.2,, -function,PyImport_ImportModuleLevelObject,3.7,, -function,PyImport_ImportModuleNoBlock,3.2,, -function,PyImport_ReloadModule,3.2,, -function,PyIndex_Check,3.8,, +data,PyGetSetDescr_Type,3.2,, +func,PyImport_AddModule,3.2,, +func,PyImport_AddModuleObject,3.7,, +func,PyImport_AddModuleRef,3.13,, +func,PyImport_AppendInittab,3.2,, +func,PyImport_ExecCodeModule,3.2,, +func,PyImport_ExecCodeModuleEx,3.2,, +func,PyImport_ExecCodeModuleObject,3.7,, +func,PyImport_ExecCodeModuleWithPathnames,3.2,, +func,PyImport_GetImporter,3.2,, +func,PyImport_GetMagicNumber,3.2,, +func,PyImport_GetMagicTag,3.2,, +func,PyImport_GetModule,3.8,, +func,PyImport_GetModuleDict,3.2,, +func,PyImport_Import,3.2,, +func,PyImport_ImportFrozenModule,3.2,, +func,PyImport_ImportFrozenModuleObject,3.7,, +func,PyImport_ImportModule,3.2,, +func,PyImport_ImportModuleLevel,3.2,, +func,PyImport_ImportModuleLevelObject,3.7,, +func,PyImport_ReloadModule,3.2,, +func,PyIndex_Check,3.8,, type,PyInterpreterState,3.2,,opaque -function,PyInterpreterState_Clear,3.2,, -function,PyInterpreterState_Delete,3.2,, -function,PyInterpreterState_Get,3.9,, -function,PyInterpreterState_GetDict,3.8,, -function,PyInterpreterState_GetID,3.7,, -function,PyInterpreterState_New,3.2,, -function,PyIter_Check,3.8,, -function,PyIter_Next,3.2,, -function,PyIter_Send,3.10,, -var,PyListIter_Type,3.2,, -var,PyListRevIter_Type,3.2,, -function,PyList_Append,3.2,, -function,PyList_AsTuple,3.2,, -function,PyList_GetItem,3.2,, -function,PyList_GetSlice,3.2,, -function,PyList_Insert,3.2,, -function,PyList_New,3.2,, -function,PyList_Reverse,3.2,, -function,PyList_SetItem,3.2,, -function,PyList_SetSlice,3.2,, -function,PyList_Size,3.2,, -function,PyList_Sort,3.2,, -var,PyList_Type,3.2,, +func,PyInterpreterState_Clear,3.2,, +func,PyInterpreterState_Delete,3.2,, +func,PyInterpreterState_Get,3.9,, +func,PyInterpreterState_GetDict,3.8,, +func,PyInterpreterState_GetID,3.7,, +func,PyInterpreterState_New,3.2,, +func,PyIter_Check,3.8,, +func,PyIter_Next,3.2,, +func,PyIter_NextItem,3.14,, +func,PyIter_Send,3.10,, +data,PyListIter_Type,3.2,, +data,PyListRevIter_Type,3.2,, +func,PyList_Append,3.2,, +func,PyList_AsTuple,3.2,, +func,PyList_GetItem,3.2,, +func,PyList_GetItemRef,3.13,, +func,PyList_GetSlice,3.2,, +func,PyList_Insert,3.2,, +func,PyList_New,3.2,, +func,PyList_Reverse,3.2,, +func,PyList_SetItem,3.2,, +func,PyList_SetSlice,3.2,, +func,PyList_Size,3.2,, +func,PyList_Sort,3.2,, +data,PyList_Type,3.2,, type,PyLongObject,3.2,,opaque -var,PyLongRangeIter_Type,3.2,, -function,PyLong_AsDouble,3.2,, -function,PyLong_AsLong,3.2,, -function,PyLong_AsLongAndOverflow,3.2,, -function,PyLong_AsLongLong,3.2,, -function,PyLong_AsLongLongAndOverflow,3.2,, -function,PyLong_AsSize_t,3.2,, -function,PyLong_AsSsize_t,3.2,, -function,PyLong_AsUnsignedLong,3.2,, -function,PyLong_AsUnsignedLongLong,3.2,, -function,PyLong_AsUnsignedLongLongMask,3.2,, -function,PyLong_AsUnsignedLongMask,3.2,, -function,PyLong_AsVoidPtr,3.2,, -function,PyLong_FromDouble,3.2,, -function,PyLong_FromLong,3.2,, -function,PyLong_FromLongLong,3.2,, -function,PyLong_FromSize_t,3.2,, -function,PyLong_FromSsize_t,3.2,, -function,PyLong_FromString,3.2,, -function,PyLong_FromUnsignedLong,3.2,, -function,PyLong_FromUnsignedLongLong,3.2,, -function,PyLong_FromVoidPtr,3.2,, -function,PyLong_GetInfo,3.2,, -var,PyLong_Type,3.2,, -var,PyMap_Type,3.2,, -function,PyMapping_Check,3.2,, -function,PyMapping_GetItemString,3.2,, -function,PyMapping_GetOptionalItem,3.13,, -function,PyMapping_GetOptionalItemString,3.13,, -function,PyMapping_HasKey,3.2,, -function,PyMapping_HasKeyString,3.2,, -function,PyMapping_Items,3.2,, -function,PyMapping_Keys,3.2,, -function,PyMapping_Length,3.2,, -function,PyMapping_SetItemString,3.2,, -function,PyMapping_Size,3.2,, -function,PyMapping_Values,3.2,, -function,PyMem_Calloc,3.7,, -function,PyMem_Free,3.2,, -function,PyMem_Malloc,3.2,, -function,PyMem_Realloc,3.2,, +data,PyLongRangeIter_Type,3.2,, +func,PyLong_AsDouble,3.2,, +func,PyLong_AsInt,3.13,, +func,PyLong_AsInt32,3.14,, +func,PyLong_AsInt64,3.14,, +func,PyLong_AsLong,3.2,, +func,PyLong_AsLongAndOverflow,3.2,, +func,PyLong_AsLongLong,3.2,, +func,PyLong_AsLongLongAndOverflow,3.2,, +func,PyLong_AsNativeBytes,3.14,, +func,PyLong_AsSize_t,3.2,, +func,PyLong_AsSsize_t,3.2,, +func,PyLong_AsUInt32,3.14,, +func,PyLong_AsUInt64,3.14,, +func,PyLong_AsUnsignedLong,3.2,, +func,PyLong_AsUnsignedLongLong,3.2,, +func,PyLong_AsUnsignedLongLongMask,3.2,, +func,PyLong_AsUnsignedLongMask,3.2,, +func,PyLong_AsVoidPtr,3.2,, +func,PyLong_FromDouble,3.2,, +func,PyLong_FromInt32,3.14,, +func,PyLong_FromInt64,3.14,, +func,PyLong_FromLong,3.2,, +func,PyLong_FromLongLong,3.2,, +func,PyLong_FromNativeBytes,3.14,, +func,PyLong_FromSize_t,3.2,, +func,PyLong_FromSsize_t,3.2,, +func,PyLong_FromString,3.2,, +func,PyLong_FromUInt32,3.14,, +func,PyLong_FromUInt64,3.14,, +func,PyLong_FromUnsignedLong,3.2,, +func,PyLong_FromUnsignedLongLong,3.2,, +func,PyLong_FromUnsignedNativeBytes,3.14,, +func,PyLong_FromVoidPtr,3.2,, +func,PyLong_GetInfo,3.2,, +data,PyLong_Type,3.2,, +data,PyMap_Type,3.2,, +func,PyMapping_Check,3.2,, +func,PyMapping_GetItemString,3.2,, +func,PyMapping_GetOptionalItem,3.13,, +func,PyMapping_GetOptionalItemString,3.13,, +func,PyMapping_HasKey,3.2,, +func,PyMapping_HasKeyString,3.2,, +func,PyMapping_HasKeyStringWithError,3.13,, +func,PyMapping_HasKeyWithError,3.13,, +func,PyMapping_Items,3.2,, +func,PyMapping_Keys,3.2,, +func,PyMapping_Length,3.2,, +func,PyMapping_SetItemString,3.2,, +func,PyMapping_Size,3.2,, +func,PyMapping_Values,3.2,, +func,PyMem_Calloc,3.7,, +func,PyMem_Free,3.2,, +func,PyMem_Malloc,3.2,, +func,PyMem_RawCalloc,3.13,, +func,PyMem_RawFree,3.13,, +func,PyMem_RawMalloc,3.13,, +func,PyMem_RawRealloc,3.13,, +func,PyMem_Realloc,3.2,, type,PyMemberDef,3.2,,full-abi -var,PyMemberDescr_Type,3.2,, -function,PyMember_GetOne,3.2,, -function,PyMember_SetOne,3.2,, -function,PyMemoryView_FromBuffer,3.11,, -function,PyMemoryView_FromMemory,3.7,, -function,PyMemoryView_FromObject,3.2,, -function,PyMemoryView_GetContiguous,3.2,, -var,PyMemoryView_Type,3.2,, +data,PyMemberDescr_Type,3.2,, +func,PyMember_GetOne,3.2,, +func,PyMember_SetOne,3.2,, +func,PyMemoryView_FromBuffer,3.11,, +func,PyMemoryView_FromMemory,3.7,, +func,PyMemoryView_FromObject,3.2,, +func,PyMemoryView_GetContiguous,3.2,, +data,PyMemoryView_Type,3.2,, type,PyMethodDef,3.2,,full-abi -var,PyMethodDescr_Type,3.2,, +data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi -function,PyModuleDef_Init,3.5,, -var,PyModuleDef_Type,3.5,, -function,PyModule_Add,3.13,, -function,PyModule_AddFunctions,3.7,, -function,PyModule_AddIntConstant,3.2,, -function,PyModule_AddObject,3.2,, -function,PyModule_AddObjectRef,3.10,, -function,PyModule_AddStringConstant,3.2,, -function,PyModule_AddType,3.10,, -function,PyModule_Create2,3.2,, -function,PyModule_ExecDef,3.7,, -function,PyModule_FromDefAndSpec2,3.7,, -function,PyModule_GetDef,3.2,, -function,PyModule_GetDict,3.2,, -function,PyModule_GetFilename,3.2,, -function,PyModule_GetFilenameObject,3.2,, -function,PyModule_GetName,3.2,, -function,PyModule_GetNameObject,3.7,, -function,PyModule_GetState,3.2,, -function,PyModule_New,3.2,, -function,PyModule_NewObject,3.7,, -function,PyModule_SetDocString,3.7,, -var,PyModule_Type,3.2,, -function,PyNumber_Absolute,3.2,, -function,PyNumber_Add,3.2,, -function,PyNumber_And,3.2,, -function,PyNumber_AsSsize_t,3.2,, -function,PyNumber_Check,3.2,, -function,PyNumber_Divmod,3.2,, -function,PyNumber_Float,3.2,, -function,PyNumber_FloorDivide,3.2,, -function,PyNumber_InPlaceAdd,3.2,, -function,PyNumber_InPlaceAnd,3.2,, -function,PyNumber_InPlaceFloorDivide,3.2,, -function,PyNumber_InPlaceLshift,3.2,, -function,PyNumber_InPlaceMatrixMultiply,3.7,, -function,PyNumber_InPlaceMultiply,3.2,, -function,PyNumber_InPlaceOr,3.2,, -function,PyNumber_InPlacePower,3.2,, -function,PyNumber_InPlaceRemainder,3.2,, -function,PyNumber_InPlaceRshift,3.2,, -function,PyNumber_InPlaceSubtract,3.2,, -function,PyNumber_InPlaceTrueDivide,3.2,, -function,PyNumber_InPlaceXor,3.2,, -function,PyNumber_Index,3.2,, -function,PyNumber_Invert,3.2,, -function,PyNumber_Long,3.2,, -function,PyNumber_Lshift,3.2,, -function,PyNumber_MatrixMultiply,3.7,, -function,PyNumber_Multiply,3.2,, -function,PyNumber_Negative,3.2,, -function,PyNumber_Or,3.2,, -function,PyNumber_Positive,3.2,, -function,PyNumber_Power,3.2,, -function,PyNumber_Remainder,3.2,, -function,PyNumber_Rshift,3.2,, -function,PyNumber_Subtract,3.2,, -function,PyNumber_ToBase,3.2,, -function,PyNumber_TrueDivide,3.2,, -function,PyNumber_Xor,3.2,, -function,PyOS_AfterFork,3.2,on platforms with fork(), -function,PyOS_AfterFork_Child,3.7,on platforms with fork(), -function,PyOS_AfterFork_Parent,3.7,on platforms with fork(), -function,PyOS_BeforeFork,3.7,on platforms with fork(), -function,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, -function,PyOS_FSPath,3.6,, -var,PyOS_InputHook,3.2,, -function,PyOS_InterruptOccurred,3.2,, -function,PyOS_double_to_string,3.2,, -function,PyOS_getsig,3.2,, -function,PyOS_mystricmp,3.2,, -function,PyOS_mystrnicmp,3.2,, -function,PyOS_setsig,3.2,, +func,PyModuleDef_Init,3.5,, +data,PyModuleDef_Type,3.5,, +func,PyModule_Add,3.13,, +func,PyModule_AddFunctions,3.7,, +func,PyModule_AddIntConstant,3.2,, +func,PyModule_AddObject,3.2,, +func,PyModule_AddObjectRef,3.10,, +func,PyModule_AddStringConstant,3.2,, +func,PyModule_AddType,3.10,, +func,PyModule_Create2,3.2,, +func,PyModule_ExecDef,3.7,, +func,PyModule_FromDefAndSpec2,3.7,, +func,PyModule_GetDef,3.2,, +func,PyModule_GetDict,3.2,, +func,PyModule_GetFilename,3.2,, +func,PyModule_GetFilenameObject,3.2,, +func,PyModule_GetName,3.2,, +func,PyModule_GetNameObject,3.7,, +func,PyModule_GetState,3.2,, +func,PyModule_New,3.2,, +func,PyModule_NewObject,3.7,, +func,PyModule_SetDocString,3.7,, +data,PyModule_Type,3.2,, +func,PyNumber_Absolute,3.2,, +func,PyNumber_Add,3.2,, +func,PyNumber_And,3.2,, +func,PyNumber_AsSsize_t,3.2,, +func,PyNumber_Check,3.2,, +func,PyNumber_Divmod,3.2,, +func,PyNumber_Float,3.2,, +func,PyNumber_FloorDivide,3.2,, +func,PyNumber_InPlaceAdd,3.2,, +func,PyNumber_InPlaceAnd,3.2,, +func,PyNumber_InPlaceFloorDivide,3.2,, +func,PyNumber_InPlaceLshift,3.2,, +func,PyNumber_InPlaceMatrixMultiply,3.7,, +func,PyNumber_InPlaceMultiply,3.2,, +func,PyNumber_InPlaceOr,3.2,, +func,PyNumber_InPlacePower,3.2,, +func,PyNumber_InPlaceRemainder,3.2,, +func,PyNumber_InPlaceRshift,3.2,, +func,PyNumber_InPlaceSubtract,3.2,, +func,PyNumber_InPlaceTrueDivide,3.2,, +func,PyNumber_InPlaceXor,3.2,, +func,PyNumber_Index,3.2,, +func,PyNumber_Invert,3.2,, +func,PyNumber_Long,3.2,, +func,PyNumber_Lshift,3.2,, +func,PyNumber_MatrixMultiply,3.7,, +func,PyNumber_Multiply,3.2,, +func,PyNumber_Negative,3.2,, +func,PyNumber_Or,3.2,, +func,PyNumber_Positive,3.2,, +func,PyNumber_Power,3.2,, +func,PyNumber_Remainder,3.2,, +func,PyNumber_Rshift,3.2,, +func,PyNumber_Subtract,3.2,, +func,PyNumber_ToBase,3.2,, +func,PyNumber_TrueDivide,3.2,, +func,PyNumber_Xor,3.2,, +func,PyOS_AfterFork,3.2,on platforms with fork(), +func,PyOS_AfterFork_Child,3.7,on platforms with fork(), +func,PyOS_AfterFork_Parent,3.7,on platforms with fork(), +func,PyOS_BeforeFork,3.7,on platforms with fork(), +func,PyOS_CheckStack,3.7,on platforms with USE_STACKCHECK, +func,PyOS_FSPath,3.6,, +data,PyOS_InputHook,3.2,, +func,PyOS_InterruptOccurred,3.2,, +func,PyOS_double_to_string,3.2,, +func,PyOS_getsig,3.2,, +func,PyOS_mystricmp,3.2,, +func,PyOS_mystrnicmp,3.2,, +func,PyOS_setsig,3.2,, type,PyOS_sighandler_t,3.2,, -function,PyOS_snprintf,3.2,, -function,PyOS_string_to_double,3.2,, -function,PyOS_strtol,3.2,, -function,PyOS_strtoul,3.2,, -function,PyOS_vsnprintf,3.2,, +func,PyOS_snprintf,3.2,, +func,PyOS_string_to_double,3.2,, +func,PyOS_strtol,3.2,, +func,PyOS_strtoul,3.2,, +func,PyOS_vsnprintf,3.2,, type,PyObject,3.2,,members member,PyObject.ob_refcnt,3.2,, member,PyObject.ob_type,3.2,, -function,PyObject_ASCII,3.2,, -function,PyObject_AsFileDescriptor,3.2,, -function,PyObject_Bytes,3.2,, -function,PyObject_Call,3.2,, -function,PyObject_CallFunction,3.2,, -function,PyObject_CallFunctionObjArgs,3.2,, -function,PyObject_CallMethod,3.2,, -function,PyObject_CallMethodObjArgs,3.2,, -function,PyObject_CallNoArgs,3.10,, -function,PyObject_CallObject,3.2,, -function,PyObject_Calloc,3.7,, -function,PyObject_CheckBuffer,3.11,, -function,PyObject_ClearWeakRefs,3.2,, -function,PyObject_CopyData,3.11,, -function,PyObject_DelAttr,3.13,, -function,PyObject_DelAttrString,3.13,, -function,PyObject_DelItem,3.2,, -function,PyObject_DelItemString,3.2,, -function,PyObject_Dir,3.2,, -function,PyObject_Format,3.2,, -function,PyObject_Free,3.2,, -function,PyObject_GC_Del,3.2,, -function,PyObject_GC_IsFinalized,3.9,, -function,PyObject_GC_IsTracked,3.9,, -function,PyObject_GC_Track,3.2,, -function,PyObject_GC_UnTrack,3.2,, -function,PyObject_GenericGetAttr,3.2,, -function,PyObject_GenericGetDict,3.10,, -function,PyObject_GenericSetAttr,3.2,, -function,PyObject_GenericSetDict,3.7,, -function,PyObject_GetAIter,3.10,, -function,PyObject_GetAttr,3.2,, -function,PyObject_GetAttrString,3.2,, -function,PyObject_GetBuffer,3.11,, -function,PyObject_GetItem,3.2,, -function,PyObject_GetIter,3.2,, -function,PyObject_GetOptionalAttr,3.13,, -function,PyObject_GetOptionalAttrString,3.13,, -function,PyObject_GetTypeData,3.12,, -function,PyObject_HasAttr,3.2,, -function,PyObject_HasAttrString,3.2,, -function,PyObject_Hash,3.2,, -function,PyObject_HashNotImplemented,3.2,, -function,PyObject_Init,3.2,, -function,PyObject_InitVar,3.2,, -function,PyObject_IsInstance,3.2,, -function,PyObject_IsSubclass,3.2,, -function,PyObject_IsTrue,3.2,, -function,PyObject_Length,3.2,, -function,PyObject_Malloc,3.2,, -function,PyObject_Not,3.2,, -function,PyObject_Realloc,3.2,, -function,PyObject_Repr,3.2,, -function,PyObject_RichCompare,3.2,, -function,PyObject_RichCompareBool,3.2,, -function,PyObject_SelfIter,3.2,, -function,PyObject_SetAttr,3.2,, -function,PyObject_SetAttrString,3.2,, -function,PyObject_SetItem,3.2,, -function,PyObject_Size,3.2,, -function,PyObject_Str,3.2,, -function,PyObject_Type,3.2,, -function,PyObject_Vectorcall,3.12,, -function,PyObject_VectorcallMethod,3.12,, -var,PyProperty_Type,3.2,, -var,PyRangeIter_Type,3.2,, -var,PyRange_Type,3.2,, -var,PyReversed_Type,3.2,, -function,PySeqIter_New,3.2,, -var,PySeqIter_Type,3.2,, -function,PySequence_Check,3.2,, -function,PySequence_Concat,3.2,, -function,PySequence_Contains,3.2,, -function,PySequence_Count,3.2,, -function,PySequence_DelItem,3.2,, -function,PySequence_DelSlice,3.2,, -function,PySequence_Fast,3.2,, -function,PySequence_GetItem,3.2,, -function,PySequence_GetSlice,3.2,, -function,PySequence_In,3.2,, -function,PySequence_InPlaceConcat,3.2,, -function,PySequence_InPlaceRepeat,3.2,, -function,PySequence_Index,3.2,, -function,PySequence_Length,3.2,, -function,PySequence_List,3.2,, -function,PySequence_Repeat,3.2,, -function,PySequence_SetItem,3.2,, -function,PySequence_SetSlice,3.2,, -function,PySequence_Size,3.2,, -function,PySequence_Tuple,3.2,, -var,PySetIter_Type,3.2,, -function,PySet_Add,3.2,, -function,PySet_Clear,3.2,, -function,PySet_Contains,3.2,, -function,PySet_Discard,3.2,, -function,PySet_New,3.2,, -function,PySet_Pop,3.2,, -function,PySet_Size,3.2,, -var,PySet_Type,3.2,, -function,PySlice_AdjustIndices,3.7,, -function,PySlice_GetIndices,3.2,, -function,PySlice_GetIndicesEx,3.2,, -function,PySlice_New,3.2,, -var,PySlice_Type,3.2,, -function,PySlice_Unpack,3.7,, -function,PyState_AddModule,3.3,, -function,PyState_FindModule,3.2,, -function,PyState_RemoveModule,3.3,, +func,PyObject_ASCII,3.2,, +func,PyObject_AsFileDescriptor,3.2,, +func,PyObject_Bytes,3.2,, +func,PyObject_Call,3.2,, +func,PyObject_CallFunction,3.2,, +func,PyObject_CallFunctionObjArgs,3.2,, +func,PyObject_CallMethod,3.2,, +func,PyObject_CallMethodObjArgs,3.2,, +func,PyObject_CallNoArgs,3.10,, +func,PyObject_CallObject,3.2,, +func,PyObject_Calloc,3.7,, +func,PyObject_CheckBuffer,3.11,, +func,PyObject_ClearWeakRefs,3.2,, +func,PyObject_CopyData,3.11,, +func,PyObject_DelAttr,3.13,, +func,PyObject_DelAttrString,3.13,, +func,PyObject_DelItem,3.2,, +func,PyObject_DelItemString,3.2,, +func,PyObject_Dir,3.2,, +func,PyObject_Format,3.2,, +func,PyObject_Free,3.2,, +func,PyObject_GC_Del,3.2,, +func,PyObject_GC_IsFinalized,3.9,, +func,PyObject_GC_IsTracked,3.9,, +func,PyObject_GC_Track,3.2,, +func,PyObject_GC_UnTrack,3.2,, +func,PyObject_GenericGetAttr,3.2,, +func,PyObject_GenericGetDict,3.10,, +func,PyObject_GenericSetAttr,3.2,, +func,PyObject_GenericSetDict,3.7,, +func,PyObject_GetAIter,3.10,, +func,PyObject_GetAttr,3.2,, +func,PyObject_GetAttrString,3.2,, +func,PyObject_GetBuffer,3.11,, +func,PyObject_GetItem,3.2,, +func,PyObject_GetIter,3.2,, +func,PyObject_GetOptionalAttr,3.13,, +func,PyObject_GetOptionalAttrString,3.13,, +func,PyObject_GetTypeData,3.12,, +func,PyObject_HasAttr,3.2,, +func,PyObject_HasAttrString,3.2,, +func,PyObject_HasAttrStringWithError,3.13,, +func,PyObject_HasAttrWithError,3.13,, +func,PyObject_Hash,3.2,, +func,PyObject_HashNotImplemented,3.2,, +func,PyObject_Init,3.2,, +func,PyObject_InitVar,3.2,, +func,PyObject_IsInstance,3.2,, +func,PyObject_IsSubclass,3.2,, +func,PyObject_IsTrue,3.2,, +func,PyObject_Length,3.2,, +func,PyObject_Malloc,3.2,, +func,PyObject_Not,3.2,, +func,PyObject_Realloc,3.2,, +func,PyObject_Repr,3.2,, +func,PyObject_RichCompare,3.2,, +func,PyObject_RichCompareBool,3.2,, +func,PyObject_SelfIter,3.2,, +func,PyObject_SetAttr,3.2,, +func,PyObject_SetAttrString,3.2,, +func,PyObject_SetItem,3.2,, +func,PyObject_Size,3.2,, +func,PyObject_Str,3.2,, +func,PyObject_Type,3.2,, +func,PyObject_Vectorcall,3.12,, +func,PyObject_VectorcallMethod,3.12,, +data,PyProperty_Type,3.2,, +data,PyRangeIter_Type,3.2,, +data,PyRange_Type,3.2,, +data,PyReversed_Type,3.2,, +func,PySeqIter_New,3.2,, +data,PySeqIter_Type,3.2,, +func,PySequence_Check,3.2,, +func,PySequence_Concat,3.2,, +func,PySequence_Contains,3.2,, +func,PySequence_Count,3.2,, +func,PySequence_DelItem,3.2,, +func,PySequence_DelSlice,3.2,, +func,PySequence_Fast,3.2,, +func,PySequence_GetItem,3.2,, +func,PySequence_GetSlice,3.2,, +func,PySequence_In,3.2,, +func,PySequence_InPlaceConcat,3.2,, +func,PySequence_InPlaceRepeat,3.2,, +func,PySequence_Index,3.2,, +func,PySequence_Length,3.2,, +func,PySequence_List,3.2,, +func,PySequence_Repeat,3.2,, +func,PySequence_SetItem,3.2,, +func,PySequence_SetSlice,3.2,, +func,PySequence_Size,3.2,, +func,PySequence_Tuple,3.2,, +data,PySetIter_Type,3.2,, +func,PySet_Add,3.2,, +func,PySet_Clear,3.2,, +func,PySet_Contains,3.2,, +func,PySet_Discard,3.2,, +func,PySet_New,3.2,, +func,PySet_Pop,3.2,, +func,PySet_Size,3.2,, +data,PySet_Type,3.2,, +func,PySlice_AdjustIndices,3.7,, +func,PySlice_GetIndices,3.2,, +func,PySlice_GetIndicesEx,3.2,, +func,PySlice_New,3.2,, +data,PySlice_Type,3.2,, +func,PySlice_Unpack,3.7,, +func,PyState_AddModule,3.3,, +func,PyState_FindModule,3.2,, +func,PyState_RemoveModule,3.3,, type,PyStructSequence_Desc,3.2,,full-abi type,PyStructSequence_Field,3.2,,full-abi -function,PyStructSequence_GetItem,3.2,, -function,PyStructSequence_New,3.2,, -function,PyStructSequence_NewType,3.2,, -function,PyStructSequence_SetItem,3.2,, -var,PyStructSequence_UnnamedField,3.11,, -var,PySuper_Type,3.2,, -function,PySys_FormatStderr,3.2,, -function,PySys_FormatStdout,3.2,, -function,PySys_GetObject,3.2,, -function,PySys_GetXOptions,3.7,, -function,PySys_ResetWarnOptions,3.2,, -function,PySys_SetObject,3.2,, -function,PySys_WriteStderr,3.2,, -function,PySys_WriteStdout,3.2,, +func,PyStructSequence_GetItem,3.2,, +func,PyStructSequence_New,3.2,, +func,PyStructSequence_NewType,3.2,, +func,PyStructSequence_SetItem,3.2,, +data,PyStructSequence_UnnamedField,3.11,, +data,PySuper_Type,3.2,, +func,PySys_Audit,3.13,, +func,PySys_AuditTuple,3.13,, +func,PySys_FormatStderr,3.2,, +func,PySys_FormatStdout,3.2,, +func,PySys_GetAttr,3.15,, +func,PySys_GetAttrString,3.15,, +func,PySys_GetObject,3.2,, +func,PySys_GetOptionalAttr,3.15,, +func,PySys_GetOptionalAttrString,3.15,, +func,PySys_GetXOptions,3.7,, +func,PySys_ResetWarnOptions,3.2,, +func,PySys_SetArgv,3.2,, +func,PySys_SetArgvEx,3.2,, +func,PySys_SetObject,3.2,, +func,PySys_WriteStderr,3.2,, +func,PySys_WriteStdout,3.2,, type,PyThreadState,3.2,,opaque -function,PyThreadState_Clear,3.2,, -function,PyThreadState_Delete,3.2,, -function,PyThreadState_Get,3.2,, -function,PyThreadState_GetDict,3.2,, -function,PyThreadState_GetFrame,3.10,, -function,PyThreadState_GetID,3.10,, -function,PyThreadState_GetInterpreter,3.10,, -function,PyThreadState_New,3.2,, -function,PyThreadState_SetAsyncExc,3.2,, -function,PyThreadState_Swap,3.2,, -function,PyThread_GetInfo,3.3,, -function,PyThread_ReInitTLS,3.2,, -function,PyThread_acquire_lock,3.2,, -function,PyThread_acquire_lock_timed,3.2,, -function,PyThread_allocate_lock,3.2,, -function,PyThread_create_key,3.2,, -function,PyThread_delete_key,3.2,, -function,PyThread_delete_key_value,3.2,, -function,PyThread_exit_thread,3.2,, -function,PyThread_free_lock,3.2,, -function,PyThread_get_key_value,3.2,, -function,PyThread_get_stacksize,3.2,, -function,PyThread_get_thread_ident,3.2,, -function,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, -function,PyThread_init_thread,3.2,, -function,PyThread_release_lock,3.2,, -function,PyThread_set_key_value,3.2,, -function,PyThread_set_stacksize,3.2,, -function,PyThread_start_new_thread,3.2,, -function,PyThread_tss_alloc,3.7,, -function,PyThread_tss_create,3.7,, -function,PyThread_tss_delete,3.7,, -function,PyThread_tss_free,3.7,, -function,PyThread_tss_get,3.7,, -function,PyThread_tss_is_created,3.7,, -function,PyThread_tss_set,3.7,, -function,PyTraceBack_Here,3.2,, -function,PyTraceBack_Print,3.2,, -var,PyTraceBack_Type,3.2,, -var,PyTupleIter_Type,3.2,, -function,PyTuple_GetItem,3.2,, -function,PyTuple_GetSlice,3.2,, -function,PyTuple_New,3.2,, -function,PyTuple_Pack,3.2,, -function,PyTuple_SetItem,3.2,, -function,PyTuple_Size,3.2,, -var,PyTuple_Type,3.2,, +func,PyThreadState_Clear,3.2,, +func,PyThreadState_Delete,3.2,, +func,PyThreadState_Get,3.2,, +func,PyThreadState_GetDict,3.2,, +func,PyThreadState_GetFrame,3.10,, +func,PyThreadState_GetID,3.10,, +func,PyThreadState_GetInterpreter,3.10,, +func,PyThreadState_New,3.2,, +func,PyThreadState_SetAsyncExc,3.2,, +func,PyThreadState_Swap,3.2,, +func,PyThread_GetInfo,3.3,, +func,PyThread_ReInitTLS,3.2,, +func,PyThread_acquire_lock,3.2,, +func,PyThread_acquire_lock_timed,3.2,, +func,PyThread_allocate_lock,3.2,, +func,PyThread_create_key,3.2,, +func,PyThread_delete_key,3.2,, +func,PyThread_delete_key_value,3.2,, +func,PyThread_exit_thread,3.2,, +func,PyThread_free_lock,3.2,, +func,PyThread_get_key_value,3.2,, +func,PyThread_get_stacksize,3.2,, +func,PyThread_get_thread_ident,3.2,, +func,PyThread_get_thread_native_id,3.2,on platforms with native thread IDs, +func,PyThread_init_thread,3.2,, +func,PyThread_release_lock,3.2,, +func,PyThread_set_key_value,3.2,, +func,PyThread_set_stacksize,3.2,, +func,PyThread_start_new_thread,3.2,, +func,PyThread_tss_alloc,3.7,, +func,PyThread_tss_create,3.7,, +func,PyThread_tss_delete,3.7,, +func,PyThread_tss_free,3.7,, +func,PyThread_tss_get,3.7,, +func,PyThread_tss_is_created,3.7,, +func,PyThread_tss_set,3.7,, +func,PyTraceBack_Here,3.2,, +func,PyTraceBack_Print,3.2,, +data,PyTraceBack_Type,3.2,, +data,PyTupleIter_Type,3.2,, +func,PyTuple_GetItem,3.2,, +func,PyTuple_GetSlice,3.2,, +func,PyTuple_New,3.2,, +func,PyTuple_Pack,3.2,, +func,PyTuple_SetItem,3.2,, +func,PyTuple_Size,3.2,, +data,PyTuple_Type,3.2,, type,PyTypeObject,3.2,,opaque -function,PyType_ClearCache,3.2,, -function,PyType_FromMetaclass,3.12,, -function,PyType_FromModuleAndSpec,3.10,, -function,PyType_FromSpec,3.2,, -function,PyType_FromSpecWithBases,3.3,, -function,PyType_GenericAlloc,3.2,, -function,PyType_GenericNew,3.2,, -function,PyType_GetFlags,3.2,, -function,PyType_GetModule,3.10,, -function,PyType_GetModuleState,3.10,, -function,PyType_GetName,3.11,, -function,PyType_GetQualName,3.11,, -function,PyType_GetSlot,3.4,, -function,PyType_GetTypeDataSize,3.12,, -function,PyType_IsSubtype,3.2,, -function,PyType_Modified,3.2,, -function,PyType_Ready,3.2,, +func,PyType_ClearCache,3.2,, +func,PyType_Freeze,3.14,, +func,PyType_FromMetaclass,3.12,, +func,PyType_FromModuleAndSpec,3.10,, +func,PyType_FromSpec,3.2,, +func,PyType_FromSpecWithBases,3.3,, +func,PyType_GenericAlloc,3.2,, +func,PyType_GenericNew,3.2,, +func,PyType_GetBaseByToken,3.14,, +func,PyType_GetFlags,3.2,, +func,PyType_GetFullyQualifiedName,3.13,, +func,PyType_GetModule,3.10,, +func,PyType_GetModuleByDef,3.13,, +func,PyType_GetModuleName,3.13,, +func,PyType_GetModuleState,3.10,, +func,PyType_GetName,3.11,, +func,PyType_GetQualName,3.11,, +func,PyType_GetSlot,3.4,, +func,PyType_GetTypeDataSize,3.12,, +func,PyType_IsSubtype,3.2,, +func,PyType_Modified,3.2,, +func,PyType_Ready,3.2,, type,PyType_Slot,3.2,,full-abi type,PyType_Spec,3.2,,full-abi -var,PyType_Type,3.2,, -function,PyUnicodeDecodeError_Create,3.2,, -function,PyUnicodeDecodeError_GetEncoding,3.2,, -function,PyUnicodeDecodeError_GetEnd,3.2,, -function,PyUnicodeDecodeError_GetObject,3.2,, -function,PyUnicodeDecodeError_GetReason,3.2,, -function,PyUnicodeDecodeError_GetStart,3.2,, -function,PyUnicodeDecodeError_SetEnd,3.2,, -function,PyUnicodeDecodeError_SetReason,3.2,, -function,PyUnicodeDecodeError_SetStart,3.2,, -function,PyUnicodeEncodeError_GetEncoding,3.2,, -function,PyUnicodeEncodeError_GetEnd,3.2,, -function,PyUnicodeEncodeError_GetObject,3.2,, -function,PyUnicodeEncodeError_GetReason,3.2,, -function,PyUnicodeEncodeError_GetStart,3.2,, -function,PyUnicodeEncodeError_SetEnd,3.2,, -function,PyUnicodeEncodeError_SetReason,3.2,, -function,PyUnicodeEncodeError_SetStart,3.2,, -var,PyUnicodeIter_Type,3.2,, -function,PyUnicodeTranslateError_GetEnd,3.2,, -function,PyUnicodeTranslateError_GetObject,3.2,, -function,PyUnicodeTranslateError_GetReason,3.2,, -function,PyUnicodeTranslateError_GetStart,3.2,, -function,PyUnicodeTranslateError_SetEnd,3.2,, -function,PyUnicodeTranslateError_SetReason,3.2,, -function,PyUnicodeTranslateError_SetStart,3.2,, -function,PyUnicode_Append,3.2,, -function,PyUnicode_AppendAndDel,3.2,, -function,PyUnicode_AsASCIIString,3.2,, -function,PyUnicode_AsCharmapString,3.2,, -function,PyUnicode_AsDecodedObject,3.2,, -function,PyUnicode_AsDecodedUnicode,3.2,, -function,PyUnicode_AsEncodedObject,3.2,, -function,PyUnicode_AsEncodedString,3.2,, -function,PyUnicode_AsEncodedUnicode,3.2,, -function,PyUnicode_AsLatin1String,3.2,, -function,PyUnicode_AsMBCSString,3.7,on Windows, -function,PyUnicode_AsRawUnicodeEscapeString,3.2,, -function,PyUnicode_AsUCS4,3.7,, -function,PyUnicode_AsUCS4Copy,3.7,, -function,PyUnicode_AsUTF16String,3.2,, -function,PyUnicode_AsUTF32String,3.2,, -function,PyUnicode_AsUTF8AndSize,3.10,, -function,PyUnicode_AsUTF8String,3.2,, -function,PyUnicode_AsUnicodeEscapeString,3.2,, -function,PyUnicode_AsWideChar,3.2,, -function,PyUnicode_AsWideCharString,3.7,, -function,PyUnicode_BuildEncodingMap,3.2,, -function,PyUnicode_Compare,3.2,, -function,PyUnicode_CompareWithASCIIString,3.2,, -function,PyUnicode_Concat,3.2,, -function,PyUnicode_Contains,3.2,, -function,PyUnicode_Count,3.2,, -function,PyUnicode_Decode,3.2,, -function,PyUnicode_DecodeASCII,3.2,, -function,PyUnicode_DecodeCharmap,3.2,, -function,PyUnicode_DecodeCodePageStateful,3.7,on Windows, -function,PyUnicode_DecodeFSDefault,3.2,, -function,PyUnicode_DecodeFSDefaultAndSize,3.2,, -function,PyUnicode_DecodeLatin1,3.2,, -function,PyUnicode_DecodeLocale,3.7,, -function,PyUnicode_DecodeLocaleAndSize,3.7,, -function,PyUnicode_DecodeMBCS,3.7,on Windows, -function,PyUnicode_DecodeMBCSStateful,3.7,on Windows, -function,PyUnicode_DecodeRawUnicodeEscape,3.2,, -function,PyUnicode_DecodeUTF16,3.2,, -function,PyUnicode_DecodeUTF16Stateful,3.2,, -function,PyUnicode_DecodeUTF32,3.2,, -function,PyUnicode_DecodeUTF32Stateful,3.2,, -function,PyUnicode_DecodeUTF7,3.2,, -function,PyUnicode_DecodeUTF7Stateful,3.2,, -function,PyUnicode_DecodeUTF8,3.2,, -function,PyUnicode_DecodeUTF8Stateful,3.2,, -function,PyUnicode_DecodeUnicodeEscape,3.2,, -function,PyUnicode_EncodeCodePage,3.7,on Windows, -function,PyUnicode_EncodeFSDefault,3.2,, -function,PyUnicode_EncodeLocale,3.7,, -function,PyUnicode_FSConverter,3.2,, -function,PyUnicode_FSDecoder,3.2,, -function,PyUnicode_Find,3.2,, -function,PyUnicode_FindChar,3.7,, -function,PyUnicode_Format,3.2,, -function,PyUnicode_FromEncodedObject,3.2,, -function,PyUnicode_FromFormat,3.2,, -function,PyUnicode_FromFormatV,3.2,, -function,PyUnicode_FromObject,3.2,, -function,PyUnicode_FromOrdinal,3.2,, -function,PyUnicode_FromString,3.2,, -function,PyUnicode_FromStringAndSize,3.2,, -function,PyUnicode_FromWideChar,3.2,, -function,PyUnicode_GetDefaultEncoding,3.2,, -function,PyUnicode_GetLength,3.7,, -function,PyUnicode_InternFromString,3.2,, -function,PyUnicode_InternInPlace,3.2,, -function,PyUnicode_IsIdentifier,3.2,, -function,PyUnicode_Join,3.2,, -function,PyUnicode_Partition,3.2,, -function,PyUnicode_RPartition,3.2,, -function,PyUnicode_RSplit,3.2,, -function,PyUnicode_ReadChar,3.7,, -function,PyUnicode_Replace,3.2,, -function,PyUnicode_Resize,3.2,, -function,PyUnicode_RichCompare,3.2,, -function,PyUnicode_Split,3.2,, -function,PyUnicode_Splitlines,3.2,, -function,PyUnicode_Substring,3.7,, -function,PyUnicode_Tailmatch,3.2,, -function,PyUnicode_Translate,3.2,, -var,PyUnicode_Type,3.2,, -function,PyUnicode_WriteChar,3.7,, +data,PyType_Type,3.2,, +func,PyUnicodeDecodeError_Create,3.2,, +func,PyUnicodeDecodeError_GetEncoding,3.2,, +func,PyUnicodeDecodeError_GetEnd,3.2,, +func,PyUnicodeDecodeError_GetObject,3.2,, +func,PyUnicodeDecodeError_GetReason,3.2,, +func,PyUnicodeDecodeError_GetStart,3.2,, +func,PyUnicodeDecodeError_SetEnd,3.2,, +func,PyUnicodeDecodeError_SetReason,3.2,, +func,PyUnicodeDecodeError_SetStart,3.2,, +func,PyUnicodeEncodeError_GetEncoding,3.2,, +func,PyUnicodeEncodeError_GetEnd,3.2,, +func,PyUnicodeEncodeError_GetObject,3.2,, +func,PyUnicodeEncodeError_GetReason,3.2,, +func,PyUnicodeEncodeError_GetStart,3.2,, +func,PyUnicodeEncodeError_SetEnd,3.2,, +func,PyUnicodeEncodeError_SetReason,3.2,, +func,PyUnicodeEncodeError_SetStart,3.2,, +data,PyUnicodeIter_Type,3.2,, +func,PyUnicodeTranslateError_GetEnd,3.2,, +func,PyUnicodeTranslateError_GetObject,3.2,, +func,PyUnicodeTranslateError_GetReason,3.2,, +func,PyUnicodeTranslateError_GetStart,3.2,, +func,PyUnicodeTranslateError_SetEnd,3.2,, +func,PyUnicodeTranslateError_SetReason,3.2,, +func,PyUnicodeTranslateError_SetStart,3.2,, +func,PyUnicode_Append,3.2,, +func,PyUnicode_AppendAndDel,3.2,, +func,PyUnicode_AsASCIIString,3.2,, +func,PyUnicode_AsCharmapString,3.2,, +func,PyUnicode_AsEncodedString,3.2,, +func,PyUnicode_AsLatin1String,3.2,, +func,PyUnicode_AsMBCSString,3.7,on Windows, +func,PyUnicode_AsRawUnicodeEscapeString,3.2,, +func,PyUnicode_AsUCS4,3.7,, +func,PyUnicode_AsUCS4Copy,3.7,, +func,PyUnicode_AsUTF16String,3.2,, +func,PyUnicode_AsUTF32String,3.2,, +func,PyUnicode_AsUTF8AndSize,3.10,, +func,PyUnicode_AsUTF8String,3.2,, +func,PyUnicode_AsUnicodeEscapeString,3.2,, +func,PyUnicode_AsWideChar,3.2,, +func,PyUnicode_AsWideCharString,3.7,, +func,PyUnicode_BuildEncodingMap,3.2,, +func,PyUnicode_Compare,3.2,, +func,PyUnicode_CompareWithASCIIString,3.2,, +func,PyUnicode_Concat,3.2,, +func,PyUnicode_Contains,3.2,, +func,PyUnicode_Count,3.2,, +func,PyUnicode_Decode,3.2,, +func,PyUnicode_DecodeASCII,3.2,, +func,PyUnicode_DecodeCharmap,3.2,, +func,PyUnicode_DecodeCodePageStateful,3.7,on Windows, +func,PyUnicode_DecodeFSDefault,3.2,, +func,PyUnicode_DecodeFSDefaultAndSize,3.2,, +func,PyUnicode_DecodeLatin1,3.2,, +func,PyUnicode_DecodeLocale,3.7,, +func,PyUnicode_DecodeLocaleAndSize,3.7,, +func,PyUnicode_DecodeMBCS,3.7,on Windows, +func,PyUnicode_DecodeMBCSStateful,3.7,on Windows, +func,PyUnicode_DecodeRawUnicodeEscape,3.2,, +func,PyUnicode_DecodeUTF16,3.2,, +func,PyUnicode_DecodeUTF16Stateful,3.2,, +func,PyUnicode_DecodeUTF32,3.2,, +func,PyUnicode_DecodeUTF32Stateful,3.2,, +func,PyUnicode_DecodeUTF7,3.2,, +func,PyUnicode_DecodeUTF7Stateful,3.2,, +func,PyUnicode_DecodeUTF8,3.2,, +func,PyUnicode_DecodeUTF8Stateful,3.2,, +func,PyUnicode_DecodeUnicodeEscape,3.2,, +func,PyUnicode_EncodeCodePage,3.7,on Windows, +func,PyUnicode_EncodeFSDefault,3.2,, +func,PyUnicode_EncodeLocale,3.7,, +func,PyUnicode_Equal,3.14,, +func,PyUnicode_EqualToUTF8,3.13,, +func,PyUnicode_EqualToUTF8AndSize,3.13,, +func,PyUnicode_FSConverter,3.2,, +func,PyUnicode_FSDecoder,3.2,, +func,PyUnicode_Find,3.2,, +func,PyUnicode_FindChar,3.7,, +func,PyUnicode_Format,3.2,, +func,PyUnicode_FromEncodedObject,3.2,, +func,PyUnicode_FromFormat,3.2,, +func,PyUnicode_FromFormatV,3.2,, +func,PyUnicode_FromObject,3.2,, +func,PyUnicode_FromOrdinal,3.2,, +func,PyUnicode_FromString,3.2,, +func,PyUnicode_FromStringAndSize,3.2,, +func,PyUnicode_FromWideChar,3.2,, +func,PyUnicode_GetDefaultEncoding,3.2,, +func,PyUnicode_GetLength,3.7,, +func,PyUnicode_InternFromString,3.2,, +func,PyUnicode_InternInPlace,3.2,, +func,PyUnicode_IsIdentifier,3.2,, +func,PyUnicode_Join,3.2,, +func,PyUnicode_Partition,3.2,, +func,PyUnicode_RPartition,3.2,, +func,PyUnicode_RSplit,3.2,, +func,PyUnicode_ReadChar,3.7,, +func,PyUnicode_Replace,3.2,, +func,PyUnicode_Resize,3.2,, +func,PyUnicode_RichCompare,3.2,, +func,PyUnicode_Split,3.2,, +func,PyUnicode_Splitlines,3.2,, +func,PyUnicode_Substring,3.7,, +func,PyUnicode_Tailmatch,3.2,, +func,PyUnicode_Translate,3.2,, +data,PyUnicode_Type,3.2,, +func,PyUnicode_WriteChar,3.7,, type,PyVarObject,3.2,,members member,PyVarObject.ob_base,3.2,, member,PyVarObject.ob_size,3.2,, -function,PyVectorcall_Call,3.12,, -function,PyVectorcall_NARGS,3.12,, +func,PyVectorcall_Call,3.12,, +func,PyVectorcall_NARGS,3.12,, type,PyWeakReference,3.2,,opaque -function,PyWeakref_GetObject,3.2,, -function,PyWeakref_GetRef,3.13,, -function,PyWeakref_NewProxy,3.2,, -function,PyWeakref_NewRef,3.2,, -var,PyWrapperDescr_Type,3.2,, -function,PyWrapper_New,3.2,, -var,PyZip_Type,3.2,, -function,Py_AddPendingCall,3.2,, -function,Py_AtExit,3.2,, +func,PyWeakref_GetObject,3.2,, +func,PyWeakref_GetRef,3.13,, +func,PyWeakref_NewProxy,3.2,, +func,PyWeakref_NewRef,3.2,, +data,PyWrapperDescr_Type,3.2,, +func,PyWrapper_New,3.2,, +data,PyZip_Type,3.2,, +func,Py_AddPendingCall,3.2,, +func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, macro,Py_BLOCK_THREADS,3.2,, -function,Py_BuildValue,3.2,, -function,Py_BytesMain,3.8,, -function,Py_CompileString,3.2,, -function,Py_DecRef,3.2,, -function,Py_DecodeLocale,3.7,, +func,Py_BuildValue,3.2,, +func,Py_BytesMain,3.8,, +func,Py_CompileString,3.2,, +func,Py_DecRef,3.2,, +func,Py_DecodeLocale,3.7,, macro,Py_END_ALLOW_THREADS,3.2,, -function,Py_EncodeLocale,3.7,, -function,Py_EndInterpreter,3.2,, -function,Py_EnterRecursiveCall,3.9,, -function,Py_Exit,3.2,, -function,Py_FatalError,3.2,, -var,Py_FileSystemDefaultEncodeErrors,3.10,, -var,Py_FileSystemDefaultEncoding,3.2,, -function,Py_Finalize,3.2,, -function,Py_FinalizeEx,3.6,, -function,Py_GenericAlias,3.9,, -var,Py_GenericAliasType,3.9,, -function,Py_GetBuildInfo,3.2,, -function,Py_GetCompiler,3.2,, -function,Py_GetCopyright,3.2,, -function,Py_GetExecPrefix,3.2,, -function,Py_GetPath,3.2,, -function,Py_GetPlatform,3.2,, -function,Py_GetPrefix,3.2,, -function,Py_GetProgramFullPath,3.2,, -function,Py_GetProgramName,3.2,, -function,Py_GetPythonHome,3.2,, -function,Py_GetRecursionLimit,3.2,, -function,Py_GetVersion,3.2,, -var,Py_HasFileSystemDefaultEncoding,3.2,, -function,Py_IncRef,3.2,, -function,Py_Initialize,3.2,, -function,Py_InitializeEx,3.2,, -function,Py_Is,3.10,, -function,Py_IsFalse,3.10,, -function,Py_IsInitialized,3.2,, -function,Py_IsNone,3.10,, -function,Py_IsTrue,3.10,, -function,Py_LeaveRecursiveCall,3.9,, -function,Py_Main,3.2,, -function,Py_MakePendingCalls,3.2,, -function,Py_NewInterpreter,3.2,, -function,Py_NewRef,3.10,, -function,Py_ReprEnter,3.2,, -function,Py_ReprLeave,3.2,, -function,Py_SetRecursionLimit,3.2,, +func,Py_EncodeLocale,3.7,, +func,Py_EndInterpreter,3.2,, +func,Py_EnterRecursiveCall,3.9,, +func,Py_Exit,3.2,, +func,Py_FatalError,3.2,, +data,Py_FileSystemDefaultEncodeErrors,3.10,, +data,Py_FileSystemDefaultEncoding,3.2,, +func,Py_Finalize,3.2,, +func,Py_FinalizeEx,3.6,, +func,Py_GenericAlias,3.9,, +data,Py_GenericAliasType,3.9,, +func,Py_GetBuildInfo,3.2,, +func,Py_GetCompiler,3.2,, +func,Py_GetConstant,3.13,, +func,Py_GetConstantBorrowed,3.13,, +func,Py_GetCopyright,3.2,, +func,Py_GetPlatform,3.2,, +func,Py_GetRecursionLimit,3.2,, +func,Py_GetVersion,3.2,, +data,Py_HasFileSystemDefaultEncoding,3.2,, +func,Py_IncRef,3.2,, +func,Py_Initialize,3.2,, +func,Py_InitializeEx,3.2,, +func,Py_Is,3.10,, +func,Py_IsFalse,3.10,, +func,Py_IsFinalizing,3.13,, +func,Py_IsInitialized,3.2,, +func,Py_IsNone,3.10,, +func,Py_IsTrue,3.10,, +func,Py_LeaveRecursiveCall,3.9,, +func,Py_Main,3.2,, +func,Py_MakePendingCalls,3.2,, +func,Py_NewInterpreter,3.2,, +func,Py_NewRef,3.10,, +func,Py_PACK_FULL_VERSION,3.14,, +func,Py_PACK_VERSION,3.14,, +func,Py_REFCNT,3.14,, +func,Py_ReprEnter,3.2,, +func,Py_ReprLeave,3.2,, +func,Py_SetProgramName,3.2,, +func,Py_SetPythonHome,3.2,, +func,Py_SetRecursionLimit,3.2,, +func,Py_TYPE,3.14,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, -var,Py_UTF8Mode,3.8,, -function,Py_VaBuildValue,3.2,, -var,Py_Version,3.11,, -function,Py_XNewRef,3.10,, +data,Py_UTF8Mode,3.8,, +func,Py_VaBuildValue,3.2,, +data,Py_Version,3.11,, +func,Py_XNewRef,3.10,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, type,Py_ssize_t,3.2,, diff --git a/Doc/deprecations/c-api-pending-removal-in-3.14.rst b/Doc/deprecations/c-api-pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..c805074669811a --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.14.rst @@ -0,0 +1,8 @@ +Pending removal in Python 3.14 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The ``ma_version_tag`` field in :c:type:`PyDictObject` for extension modules + (:pep:`699`; :gh:`101193`). + +* Creating :c:data:`immutable types <Py_TPFLAGS_IMMUTABLETYPE>` with mutable + bases (:gh:`95388`). diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..b87f0a5ecde06f --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -0,0 +1,136 @@ +Pending removal in Python 3.15 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdecimal``. +* The :c:func:`!PyImport_ImportModuleNoBlock`: + Use :c:func:`PyImport_ImportModule` instead. +* :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: + Use :c:func:`PyWeakref_GetRef` instead. The `pythoncapi-compat project + <https://github.com/python/pythoncapi-compat/>`__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. +* :c:type:`Py_UNICODE` type and the :c:macro:`!Py_UNICODE_WIDE` macro: + Use :c:type:`wchar_t` instead. +* :c:func:`!PyUnicode_AsDecodedObject`: + Use :c:func:`PyCodec_Decode` instead. +* :c:func:`!PyUnicode_AsDecodedUnicode`: + Use :c:func:`PyCodec_Decode` instead; Note that some codecs (for example, "base64") + may return a type other than :class:`str`, such as :class:`bytes`. +* :c:func:`!PyUnicode_AsEncodedObject`: + Use :c:func:`PyCodec_Encode` instead. +* :c:func:`!PyUnicode_AsEncodedUnicode`: + Use :c:func:`PyCodec_Encode` instead; Note that some codecs (for example, "base64") + may return a type other than :class:`bytes`, such as :class:`str`. +* Python initialization functions, deprecated in Python 3.13: + + * :c:func:`!Py_GetPath`: + Use :c:func:`PyConfig_Get("module_search_paths") <PyConfig_Get>` + (:data:`sys.path`) instead. + * :c:func:`!Py_GetPrefix`: + Use :c:func:`PyConfig_Get("base_prefix") <PyConfig_Get>` + (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") + <PyConfig_Get>` (:data:`sys.prefix`) if :ref:`virtual environments + <venv-def>` need to be handled. + * :c:func:`!Py_GetExecPrefix`: + Use :c:func:`PyConfig_Get("base_exec_prefix") <PyConfig_Get>` + (:data:`sys.base_exec_prefix`) instead. Use + :c:func:`PyConfig_Get("exec_prefix") <PyConfig_Get>` + (:data:`sys.exec_prefix`) if :ref:`virtual environments <venv-def>` need to + be handled. + * :c:func:`!Py_GetProgramFullPath`: + Use :c:func:`PyConfig_Get("executable") <PyConfig_Get>` + (:data:`sys.executable`) instead. + * :c:func:`!Py_GetProgramName`: + Use :c:func:`PyConfig_Get("executable") <PyConfig_Get>` + (:data:`sys.executable`) instead. + * :c:func:`!Py_GetPythonHome`: + Use :c:func:`PyConfig_Get("home") <PyConfig_Get>` or the + :envvar:`PYTHONHOME` environment variable instead. + + The `pythoncapi-compat project + <https://github.com/python/pythoncapi-compat/>`__ can be used to get + :c:func:`PyConfig_Get` on Python 3.13 and older. + +* Functions to configure Python's initialization, deprecated in Python 3.11: + + * :c:func:`!PySys_SetArgvEx()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!PySys_SetArgv()`: + Set :c:member:`PyConfig.argv` instead. + * :c:func:`!Py_SetProgramName()`: + Set :c:member:`PyConfig.program_name` instead. + * :c:func:`!Py_SetPythonHome()`: + Set :c:member:`PyConfig.home` instead. + * :c:func:`PySys_ResetWarnOptions`: + Clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` instead. + +* Global configuration variables: + + * :c:var:`Py_DebugFlag`: + Use :c:member:`PyConfig.parser_debug` or + :c:func:`PyConfig_Get("parser_debug") <PyConfig_Get>` instead. + * :c:var:`Py_VerboseFlag`: + Use :c:member:`PyConfig.verbose` or + :c:func:`PyConfig_Get("verbose") <PyConfig_Get>` instead. + * :c:var:`Py_QuietFlag`: + Use :c:member:`PyConfig.quiet` or + :c:func:`PyConfig_Get("quiet") <PyConfig_Get>` instead. + * :c:var:`Py_InteractiveFlag`: + Use :c:member:`PyConfig.interactive` or + :c:func:`PyConfig_Get("interactive") <PyConfig_Get>` instead. + * :c:var:`Py_InspectFlag`: + Use :c:member:`PyConfig.inspect` or + :c:func:`PyConfig_Get("inspect") <PyConfig_Get>` instead. + * :c:var:`Py_OptimizeFlag`: + Use :c:member:`PyConfig.optimization_level` or + :c:func:`PyConfig_Get("optimization_level") <PyConfig_Get>` instead. + * :c:var:`Py_NoSiteFlag`: + Use :c:member:`PyConfig.site_import` or + :c:func:`PyConfig_Get("site_import") <PyConfig_Get>` instead. + * :c:var:`Py_BytesWarningFlag`: + Use :c:member:`PyConfig.bytes_warning` or + :c:func:`PyConfig_Get("bytes_warning") <PyConfig_Get>` instead. + * :c:var:`Py_FrozenFlag`: + Use :c:member:`PyConfig.pathconfig_warnings` or + :c:func:`PyConfig_Get("pathconfig_warnings") <PyConfig_Get>` instead. + * :c:var:`Py_IgnoreEnvironmentFlag`: + Use :c:member:`PyConfig.use_environment` or + :c:func:`PyConfig_Get("use_environment") <PyConfig_Get>` instead. + * :c:var:`Py_DontWriteBytecodeFlag`: + Use :c:member:`PyConfig.write_bytecode` or + :c:func:`PyConfig_Get("write_bytecode") <PyConfig_Get>` instead. + * :c:var:`Py_NoUserSiteDirectory`: + Use :c:member:`PyConfig.user_site_directory` or + :c:func:`PyConfig_Get("user_site_directory") <PyConfig_Get>` instead. + * :c:var:`Py_UnbufferedStdioFlag`: + Use :c:member:`PyConfig.buffered_stdio` or + :c:func:`PyConfig_Get("buffered_stdio") <PyConfig_Get>` instead. + * :c:var:`Py_HashRandomizationFlag`: + Use :c:member:`PyConfig.use_hash_seed` + and :c:member:`PyConfig.hash_seed` or + :c:func:`PyConfig_Get("hash_seed") <PyConfig_Get>` instead. + * :c:var:`Py_IsolatedFlag`: + Use :c:member:`PyConfig.isolated` or + :c:func:`PyConfig_Get("isolated") <PyConfig_Get>` instead. + * :c:var:`Py_LegacyWindowsFSEncodingFlag`: + Use :c:member:`PyPreConfig.legacy_windows_fs_encoding` or + :c:func:`PyConfig_Get("legacy_windows_fs_encoding") <PyConfig_Get>` instead. + * :c:var:`Py_LegacyWindowsStdioFlag`: + Use :c:member:`PyConfig.legacy_windows_stdio` or + :c:func:`PyConfig_Get("legacy_windows_stdio") <PyConfig_Get>` instead. + * :c:var:`!Py_FileSystemDefaultEncoding`, :c:var:`!Py_HasFileSystemDefaultEncoding`: + Use :c:member:`PyConfig.filesystem_encoding` or + :c:func:`PyConfig_Get("filesystem_encoding") <PyConfig_Get>` instead. + * :c:var:`!Py_FileSystemDefaultEncodeErrors`: + Use :c:member:`PyConfig.filesystem_errors` or + :c:func:`PyConfig_Get("filesystem_errors") <PyConfig_Get>` instead. + * :c:var:`!Py_UTF8Mode`: + Use :c:member:`PyPreConfig.utf8_mode` or + :c:func:`PyConfig_Get("utf8_mode") <PyConfig_Get>` instead. + (see :c:func:`Py_PreInitialize`) + + The :c:func:`Py_InitializeFromConfig` API should be used with + :c:type:`PyConfig` to set these options. Or :c:func:`PyConfig_Get` can be + used to get these options at runtime. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst new file mode 100644 index 00000000000000..564cfb79a0b513 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -0,0 +1,45 @@ +Pending removal in Python 3.18 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* Deprecated private functions (:gh:`128863`): + + * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. + * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. + * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. + * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. + * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: + use :c:func:`PyLongWriter_Create`. + * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. + * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. + * :c:func:`!_PyUnicodeWriter_Init`: + replace ``_PyUnicodeWriter_Init(&writer)`` with + :c:func:`writer = PyUnicodeWriter_Create(0) <PyUnicodeWriter_Create>`. + * :c:func:`!_PyUnicodeWriter_Finish`: + replace ``_PyUnicodeWriter_Finish(&writer)`` with + :c:func:`PyUnicodeWriter_Finish(writer) <PyUnicodeWriter_Finish>`. + * :c:func:`!_PyUnicodeWriter_Dealloc`: + replace ``_PyUnicodeWriter_Dealloc(&writer)`` with + :c:func:`PyUnicodeWriter_Discard(writer) <PyUnicodeWriter_Discard>`. + * :c:func:`!_PyUnicodeWriter_WriteChar`: + replace ``_PyUnicodeWriter_WriteChar(&writer, ch)`` with + :c:func:`PyUnicodeWriter_WriteChar(writer, ch) <PyUnicodeWriter_WriteChar>`. + * :c:func:`!_PyUnicodeWriter_WriteStr`: + replace ``_PyUnicodeWriter_WriteStr(&writer, str)`` with + :c:func:`PyUnicodeWriter_WriteStr(writer, str) <PyUnicodeWriter_WriteStr>`. + * :c:func:`!_PyUnicodeWriter_WriteSubstring`: + replace ``_PyUnicodeWriter_WriteSubstring(&writer, str, start, end)`` with + :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) <PyUnicodeWriter_WriteSubstring>`. + * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: + replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with + :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) <PyUnicodeWriter_WriteUTF8>`. + * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: + replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with + :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) <PyUnicodeWriter_WriteUTF8>`. + * :c:func:`!_PyUnicodeWriter_Prepare`: (no replacement). + * :c:func:`!_PyUnicodeWriter_PrepareKind`: (no replacement). + * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. + * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. + + The `pythoncapi-compat project + <https://github.com/python/pythoncapi-compat/>`__ can be used to get these + new public functions on Python 3.13 and older. diff --git a/Doc/deprecations/c-api-pending-removal-in-future.rst b/Doc/deprecations/c-api-pending-removal-in-future.rst new file mode 100644 index 00000000000000..841d1b455b6bec --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-future.rst @@ -0,0 +1,42 @@ +Pending removal in future versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following APIs are deprecated and will be removed, +although there is currently no date scheduled for their removal. + +* :c:macro:`Py_TPFLAGS_HAVE_FINALIZE`: + Unneeded since Python 3.8. +* :c:func:`PyErr_Fetch`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_NormalizeException`: + Use :c:func:`PyErr_GetRaisedException` instead. +* :c:func:`PyErr_Restore`: + Use :c:func:`PyErr_SetRaisedException` instead. +* :c:func:`PyModule_GetFilename`: + Use :c:func:`PyModule_GetFilenameObject` instead. +* :c:func:`PyOS_AfterFork`: + Use :c:func:`PyOS_AfterFork_Child` instead. +* :c:func:`PySlice_GetIndicesEx`: + Use :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` instead. +* :c:func:`PyUnicode_READY`: + Unneeded since Python 3.12 +* :c:func:`!PyErr_Display`: + Use :c:func:`PyErr_DisplayException` instead. +* :c:func:`!_PyErr_ChainExceptions`: + Use :c:func:`!_PyErr_ChainExceptions1` instead. +* :c:member:`!PyBytesObject.ob_shash` member: + call :c:func:`PyObject_Hash` instead. +* Thread Local Storage (TLS) API: + + * :c:func:`PyThread_create_key`: + Use :c:func:`PyThread_tss_alloc` instead. + * :c:func:`PyThread_delete_key`: + Use :c:func:`PyThread_tss_free` instead. + * :c:func:`PyThread_set_key_value`: + Use :c:func:`PyThread_tss_set` instead. + * :c:func:`PyThread_get_key_value`: + Use :c:func:`PyThread_tss_get` instead. + * :c:func:`PyThread_delete_key_value`: + Use :c:func:`PyThread_tss_delete` instead. + * :c:func:`PyThread_ReInitTLS`: + Unneeded since Python 3.7. diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst new file mode 100644 index 00000000000000..d064f2bec42c22 --- /dev/null +++ b/Doc/deprecations/index.rst @@ -0,0 +1,21 @@ +Deprecations +============ + +.. include:: pending-removal-in-3.15.rst + +.. include:: pending-removal-in-3.16.rst + +.. include:: pending-removal-in-3.17.rst + +.. include:: pending-removal-in-3.19.rst + +.. include:: pending-removal-in-future.rst + +C API deprecations +------------------ + +.. include:: c-api-pending-removal-in-3.15.rst + +.. include:: c-api-pending-removal-in-3.18.rst + +.. include:: c-api-pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst new file mode 100644 index 00000000000000..2fd2f12cc6a2c4 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -0,0 +1,52 @@ +Pending removal in Python 3.13 +------------------------------ + +Modules (see :pep:`594`): + +* :mod:`!aifc` +* :mod:`!audioop` +* :mod:`!cgi` +* :mod:`!cgitb` +* :mod:`!chunk` +* :mod:`!crypt` +* :mod:`!imghdr` +* :mod:`!mailcap` +* :mod:`!msilib` +* :mod:`!nis` +* :mod:`!nntplib` +* :mod:`!ossaudiodev` +* :mod:`!pipes` +* :mod:`!sndhdr` +* :mod:`!spwd` +* :mod:`!sunau` +* :mod:`!telnetlib` +* :mod:`!uu` +* :mod:`!xdrlib` + +Other modules: + +* :mod:`!lib2to3`, and the :program:`2to3` program (:gh:`84540`) + +APIs: + +* :class:`!configparser.LegacyInterpolation` (:gh:`90765`) +* ``locale.resetlocale()`` (:gh:`90817`) +* :meth:`!turtle.RawTurtle.settiltangle` (:gh:`50096`) +* :func:`!unittest.findTestCases` (:gh:`50096`) +* :func:`!unittest.getTestCaseNames` (:gh:`50096`) +* :func:`!unittest.makeSuite` (:gh:`50096`) +* :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) +* :class:`!webbrowser.MacOSX` (:gh:`86421`) +* :class:`classmethod` descriptor chaining (:gh:`89519`) +* :mod:`importlib.resources` deprecated methods: + + * ``contents()`` + * ``is_resource()`` + * ``open_binary()`` + * ``open_text()`` + * ``path()`` + * ``read_binary()`` + * ``read_text()`` + + Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy + <https://importlib-resources.readthedocs.io/en/latest/using.html#migrating-from-legacy>`_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst new file mode 100644 index 00000000000000..9aac10840a663f --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -0,0 +1,105 @@ +Pending removal in Python 3.14 +------------------------------ + +* :mod:`argparse`: The *type*, *choices*, and *metavar* parameters + of :class:`!argparse.BooleanOptionalAction` are deprecated + and will be removed in 3.14. + (Contributed by Nikita Sobolev in :gh:`92248`.) + +* :mod:`ast`: The following features have been deprecated in documentation + since Python 3.8, now cause a :exc:`DeprecationWarning` to be emitted at + runtime when they are accessed or used, and will be removed in Python 3.14: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. + (Contributed by Serhiy Storchaka in :gh:`90953`.) + +* :mod:`asyncio`: + + * The child watcher classes :class:`!asyncio.MultiLoopChildWatcher`, + :class:`!asyncio.FastChildWatcher`, :class:`!asyncio.AbstractChildWatcher` + and :class:`!asyncio.SafeChildWatcher` are deprecated and + will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * :func:`!asyncio.set_child_watcher`, :func:`!asyncio.get_child_watcher`, + :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` and + :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` are deprecated + and will be removed in Python 3.14. + (Contributed by Kumar Aditya in :gh:`94597`.) + + * The :meth:`~asyncio.get_event_loop` method of the + default event loop policy now emits a :exc:`DeprecationWarning` if there + is no current event loop set and it decides to create one. + (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) + +* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. + Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. + For use in typing, prefer a union, like ``bytes | bytearray``, + or :class:`collections.abc.Buffer`. + (Contributed by Shantanu Jain in :gh:`91896`.) + +* :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. + (Contributed by Alan Williams in :gh:`72346`.) + +* :mod:`importlib.abc` deprecated classes: + + * :class:`!importlib.abc.ResourceReader` + * :class:`!importlib.abc.Traversable` + * :class:`!importlib.abc.TraversableResources` + + Use :mod:`importlib.resources.abc` classes instead: + + * :class:`importlib.resources.abc.Traversable` + * :class:`importlib.resources.abc.TraversableResources` + + (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + +* :mod:`itertools` had undocumented, inefficient, historically buggy, + and inconsistent support for copy, deepcopy, and pickle operations. + This will be removed in 3.14 for a significant reduction in code + volume and maintenance burden. + (Contributed by Raymond Hettinger in :gh:`101588`.) + +* :mod:`multiprocessing`: The default start method will change to a safer one on + Linux, BSDs, and other non-macOS POSIX platforms where ``'fork'`` is currently + the default (:gh:`84559`). Adding a runtime warning about this was deemed too + disruptive as the majority of code is not expected to care. Use the + :func:`~multiprocessing.get_context` or + :func:`~multiprocessing.set_start_method` APIs to explicitly specify when + your code *requires* ``'fork'``. See :ref:`multiprocessing-start-methods`. + +* :mod:`pathlib`: :meth:`~pathlib.PurePath.is_relative_to` and + :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is + deprecated. + +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` + now raise :exc:`DeprecationWarning`; + use :func:`importlib.util.find_spec` instead. + (Contributed by Nikita Sobolev in :gh:`97850`.) + +* :mod:`pty`: + + * ``master_open()``: use :func:`pty.openpty`. + * ``slave_open()``: use :func:`pty.openpty`. + +* :mod:`sqlite3`: + + * :data:`!version` and :data:`!version_info`. + + * :meth:`~sqlite3.Cursor.execute` and :meth:`~sqlite3.Cursor.executemany` + if :ref:`named placeholders <sqlite3-placeholders>` are used and + *parameters* is a sequence instead of a :class:`dict`. + +* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, + now causes a :exc:`DeprecationWarning` to be emitted when it is used. + +* :mod:`urllib`: + :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a + public API. + (Contributed by Gregory P. Smith in :gh:`88168`.) diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst new file mode 100644 index 00000000000000..a76d06cce1278a --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -0,0 +1,110 @@ +Pending removal in Python 3.15 +------------------------------ + +* The import system: + + * Setting :attr:`~module.__cached__` on a module while + failing to set :attr:`__spec__.cached <importlib.machinery.ModuleSpec.cached>` + is deprecated. In Python 3.15, :attr:`!__cached__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + + * Setting :attr:`~module.__package__` on a module while + failing to set :attr:`__spec__.parent <importlib.machinery.ModuleSpec.parent>` + is deprecated. In Python 3.15, :attr:`!__package__` will cease to be set or + take into consideration by the import system or standard library. (:gh:`97879`) + +* :mod:`ctypes`: + + * The undocumented :func:`!ctypes.SetPointerType` function + has been deprecated since Python 3.13. + +* :mod:`http.server`: + + * The obsolete and rarely used :class:`!CGIHTTPRequestHandler` + has been deprecated since Python 3.13. + No direct replacement exists. + *Anything* is better than CGI to interface + a web server with a request handler. + + * The :option:`!--cgi` flag to the :program:`python -m http.server` + command-line interface has been deprecated since Python 3.13. + +* :mod:`importlib`: + + * ``load_module()`` method: use ``exec_module()`` instead. + +* :class:`locale`: + + * The :func:`~locale.getdefaultlocale` function + has been deprecated since Python 3.11. + Its removal was originally planned for Python 3.13 (:gh:`90817`), + but has been postponed to Python 3.15. + Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, + and :func:`~locale.getencoding` instead. + (Contributed by Hugo van Kemenade in :gh:`111187`.) + +* :mod:`pathlib`: + + * :meth:`.PurePath.is_reserved` + has been deprecated since Python 3.13. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + +* :mod:`platform`: + + * :func:`!platform.java_ver` has been deprecated since Python 3.13. + This function is only useful for Jython support, has a confusing API, + and is largely untested. + +* :mod:`sysconfig`: + + * The *check_home* argument of :func:`sysconfig.is_python_build` has been + deprecated since Python 3.12. + +* :mod:`threading`: + + * :func:`~threading.RLock` will take no arguments in Python 3.15. + Passing any arguments has been deprecated since Python 3.14, + as the Python version does not permit any arguments, + but the C version allows any number of positional or keyword arguments, + ignoring every argument. + +* :mod:`types`: + + * :class:`types.CodeType`: Accessing :attr:`~codeobject.co_lnotab` was + deprecated in :pep:`626` + since 3.10 and was planned to be removed in 3.12, + but it only got a proper :exc:`DeprecationWarning` in 3.12. + May be removed in 3.15. + (Contributed by Nikita Sobolev in :gh:`101866`.) + +* :mod:`typing`: + + * The undocumented keyword argument syntax for creating + :class:`~typing.NamedTuple` classes + (for example, ``Point = NamedTuple("Point", x=int, y=int)``) + has been deprecated since Python 3.13. + Use the class-based syntax or the functional syntax instead. + + * When using the functional syntax of :class:`~typing.TypedDict`\s, failing + to pass a value to the *fields* parameter (``TD = TypedDict("TD")``) or + passing ``None`` (``TD = TypedDict("TD", None)``) has been deprecated + since Python 3.13. + Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` + to create a TypedDict with zero field. + + * The :func:`typing.no_type_check_decorator` decorator function + has been deprecated since Python 3.13. + After eight years in the :mod:`typing` module, + it has yet to be supported by any major type checker. + +* :mod:`wave`: + + * The ``getmark()``, ``setmark()`` and ``getmarkers()`` methods of + the :class:`~wave.Wave_read` and :class:`~wave.Wave_write` classes + have been deprecated since Python 3.13. + +* :mod:`zipimport`: + + * :meth:`~zipimport.zipimporter.load_module` has been deprecated since + Python 3.10. Use :meth:`~zipimport.zipimporter.exec_module` instead. + (Contributed by Jiahao Li in :gh:`125746`.) diff --git a/Doc/deprecations/pending-removal-in-3.16.rst b/Doc/deprecations/pending-removal-in-3.16.rst new file mode 100644 index 00000000000000..cdd76ee693f31b --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.16.rst @@ -0,0 +1,105 @@ +Pending removal in Python 3.16 +------------------------------ + +* The import system: + + * Setting :attr:`~module.__loader__` on a module while + failing to set :attr:`__spec__.loader <importlib.machinery.ModuleSpec.loader>` + is deprecated. In Python 3.16, :attr:`!__loader__` will cease to be set or + taken into consideration by the import system or the standard library. + +* :mod:`array`: + + * The ``'u'`` format code (:c:type:`wchar_t`) + has been deprecated in documentation since Python 3.3 + and at runtime since Python 3.13. + Use the ``'w'`` format code (:c:type:`Py_UCS4`) + for Unicode characters instead. + +* :mod:`asyncio`: + + * :func:`!asyncio.iscoroutinefunction` is deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + + * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: + + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` + + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + *loop_factory* to use the desired event loop implementation. + + For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: + + import asyncio + + async def main(): + ... + + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) + + (Contributed by Kumar Aditya in :gh:`127949`.) + +* :mod:`builtins`: + + * Bitwise inversion on boolean types, ``~True`` or ``~False`` + has been deprecated since Python 3.12, + as it produces surprising and unintuitive results (``-2`` and ``-1``). + Use ``not x`` instead for the logical negation of a Boolean. + In the rare case that you need the bitwise inversion of + the underlying integer, convert to ``int`` explicitly (``~int(x)``). + +* :mod:`functools`: + + * Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments has been deprecated since Python 3.14. + +* :mod:`logging`: + + Support for custom logging handlers with the *strm* argument is deprecated + and scheduled for removal in Python 3.16. Define handlers with the *stream* + argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) + +* :mod:`mimetypes`: + + * Valid extensions start with a '.' or are empty for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions are deprecated and will + raise a :exc:`ValueError` in Python 3.16. + (Contributed by Hugo van Kemenade in :gh:`75223`.) + +* :mod:`shutil`: + + * The :class:`!ExecError` exception + has been deprecated since Python 3.14. + It has not been used by any function in :mod:`!shutil` since Python 3.4, + and is now an alias of :exc:`RuntimeError`. + +* :mod:`symtable`: + + * The :meth:`Class.get_methods <symtable.Class.get_methods>` method + has been deprecated since Python 3.14. + +* :mod:`sys`: + + * The :func:`~sys._enablelegacywindowsfsencoding` function + has been deprecated since Python 3.13. + Use the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable instead. + +* :mod:`sysconfig`: + + * The :func:`!sysconfig.expand_makefile_vars` function + has been deprecated since Python 3.14. + Use the ``vars`` argument of :func:`sysconfig.get_paths` instead. + +* :mod:`tarfile`: + + * The undocumented and unused :attr:`!TarFile.tarfile` attribute + has been deprecated since Python 3.13. diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst new file mode 100644 index 00000000000000..370b98307e5228 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -0,0 +1,10 @@ +Pending removal in Python 3.17 +------------------------------ + +* :mod:`typing`: + + - Before Python 3.14, old-style unions were implemented using the private class + ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, + but it has been retained for backward compatibility, with removal scheduled for Python + 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` + and :func:`typing.get_args` instead of relying on private implementation details. diff --git a/Doc/deprecations/pending-removal-in-3.19.rst b/Doc/deprecations/pending-removal-in-3.19.rst new file mode 100644 index 00000000000000..25f9cba390de68 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.19.rst @@ -0,0 +1,24 @@ +Pending removal in Python 3.19 +------------------------------ + +* :mod:`ctypes`: + + * Implicitly switching to the MSVC-compatible struct layout by setting + :attr:`~ctypes.Structure._pack_` but not :attr:`~ctypes.Structure._layout_` + on non-Windows platforms. + +* :mod:`hashlib`: + + - In hash function constructors such as :func:`~hashlib.new` or the + direct hash-named constructors such as :func:`~hashlib.md5` and + :func:`~hashlib.sha256`, their optional initial data parameter could + also be passed a keyword argument named ``data=`` or ``string=`` in + various :mod:`!hashlib` implementations. + + Support for the ``string`` keyword argument name is now deprecated + and slated for removal in Python 3.19. + + Before Python 3.13, the ``string`` keyword parameter was not correctly + supported depending on the backend implementation of hash functions. + Prefer passing the initial data as a positional argument for maximum + backwards compatibility. diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst new file mode 100644 index 00000000000000..4c4a368baca955 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -0,0 +1,158 @@ +Pending removal in future versions +---------------------------------- + +The following APIs will be removed in the future, +although there is currently no date scheduled for their removal. + +* :mod:`argparse`: + + * Nesting argument groups and nesting mutually exclusive + groups are deprecated. + * Passing the undocumented keyword argument *prefix_chars* to + :meth:`~argparse.ArgumentParser.add_argument_group` is now + deprecated. + * The :class:`argparse.FileType` type converter is deprecated. + +* :mod:`builtins`: + + * ``bool(NotImplemented)``. + * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` + signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, + the single argument signature. + * Currently Python accepts numeric literals immediately followed by keywords, + for example ``0in x``, ``1or x``, ``0if 1else 2``. It allows confusing and + ambiguous expressions like ``[0x1for x in y]`` (which can be interpreted as + ``[0x1 for x in y]`` or ``[0x1f or x in y]``). A syntax warning is raised + if the numeric literal is immediately followed by one of keywords + :keyword:`and`, :keyword:`else`, :keyword:`for`, :keyword:`if`, + :keyword:`in`, :keyword:`is` and :keyword:`or`. In a future release it + will be changed to a syntax error. (:gh:`87999`) + * Support for ``__index__()`` and ``__int__()`` method returning non-int type: + these methods will be required to return an instance of a strict subclass of + :class:`int`. + * Support for ``__float__()`` method returning a strict subclass of + :class:`float`: these methods will be required to return an instance of + :class:`float`. + * Support for ``__complex__()`` method returning a strict subclass of + :class:`complex`: these methods will be required to return an instance of + :class:`complex`. + * Delegation of ``int()`` to ``__trunc__()`` method. + * Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; it should only be passed + as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) + +* :mod:`calendar`: ``calendar.January`` and ``calendar.February`` constants are + deprecated and replaced by :data:`calendar.JANUARY` and + :data:`calendar.FEBRUARY`. + (Contributed by Prince Roshan in :gh:`103636`.) + +* :mod:`codecs`: use :func:`open` instead of :func:`codecs.open`. (:gh:`133038`) + +* :attr:`codeobject.co_lnotab`: use the :meth:`codeobject.co_lines` method + instead. + +* :mod:`datetime`: + + * :meth:`~datetime.datetime.utcnow`: + use ``datetime.datetime.now(tz=datetime.UTC)``. + * :meth:`~datetime.datetime.utcfromtimestamp`: + use ``datetime.datetime.fromtimestamp(timestamp, tz=datetime.UTC)``. + +* :mod:`gettext`: Plural value must be an integer. + +* :mod:`importlib`: + + * :func:`~importlib.util.cache_from_source` *debug_override* parameter is + deprecated: use the *optimization* parameter instead. + +* :mod:`importlib.metadata`: + + * ``EntryPoints`` tuple interface. + * Implicit ``None`` on return values. + +* :mod:`logging`: the ``warn()`` method has been deprecated + since Python 3.3, use :meth:`~logging.warning` instead. + +* :mod:`mailbox`: Use of StringIO input and text mode is deprecated, use + BytesIO and binary mode instead. + +* :mod:`os`: Calling :func:`os.register_at_fork` in multi-threaded process. + +* :class:`!pydoc.ErrorDuringImport`: A tuple value for *exc_info* parameter is + deprecated, use an exception instance. + +* :mod:`re`: More strict rules are now applied for numerical group references + and group names in regular expressions. Only sequence of ASCII digits is now + accepted as a numerical reference. The group name in bytes patterns and + replacement strings can now only contain ASCII letters and digits and + underscore. + (Contributed by Serhiy Storchaka in :gh:`91760`.) + +* :mod:`!sre_compile`, :mod:`!sre_constants` and :mod:`!sre_parse` modules. + +* :mod:`shutil`: :func:`~shutil.rmtree`'s *onerror* parameter is deprecated in + Python 3.12; use the *onexc* parameter instead. + +* :mod:`ssl` options and protocols: + + * :class:`ssl.SSLContext` without protocol argument is deprecated. + * :class:`ssl.SSLContext`: :meth:`~ssl.SSLContext.set_npn_protocols` and + :meth:`!selected_npn_protocol` are deprecated: use ALPN + instead. + * ``ssl.OP_NO_SSL*`` options + * ``ssl.OP_NO_TLS*`` options + * ``ssl.PROTOCOL_SSLv3`` + * ``ssl.PROTOCOL_TLS`` + * ``ssl.PROTOCOL_TLSv1`` + * ``ssl.PROTOCOL_TLSv1_1`` + * ``ssl.PROTOCOL_TLSv1_2`` + * ``ssl.TLSVersion.SSLv3`` + * ``ssl.TLSVersion.TLSv1`` + * ``ssl.TLSVersion.TLSv1_1`` + +* :mod:`threading` methods: + + * :meth:`!threading.Condition.notifyAll`: use :meth:`~threading.Condition.notify_all`. + * :meth:`!threading.Event.isSet`: use :meth:`~threading.Event.is_set`. + * :meth:`!threading.Thread.isDaemon`, :meth:`threading.Thread.setDaemon`: + use :attr:`threading.Thread.daemon` attribute. + * :meth:`!threading.Thread.getName`, :meth:`threading.Thread.setName`: + use :attr:`threading.Thread.name` attribute. + * :meth:`!threading.currentThread`: use :meth:`threading.current_thread`. + * :meth:`!threading.activeCount`: use :meth:`threading.active_count`. + +* :class:`typing.Text` (:gh:`92332`). + +* The internal class ``typing._UnionGenericAlias`` is no longer used to implement + :class:`typing.Union`. To preserve compatibility with users using this private + class, a compatibility shim will be provided until at least Python 3.17. (Contributed by + Jelle Zijlstra in :gh:`105499`.) + +* :class:`unittest.IsolatedAsyncioTestCase`: it is deprecated to return a value + that is not ``None`` from a test case. + +* :mod:`urllib.parse` deprecated functions: :func:`~urllib.parse.urlparse` instead + + * ``splitattr()`` + * ``splithost()`` + * ``splitnport()`` + * ``splitpasswd()`` + * ``splitport()`` + * ``splitquery()`` + * ``splittag()`` + * ``splittype()`` + * ``splituser()`` + * ``splitvalue()`` + * ``to_bytes()`` + +* :mod:`wsgiref`: ``SimpleHandler.stdout.write()`` should not do partial + writes. + +* :mod:`xml.etree.ElementTree`: Testing the truth value of an + :class:`~xml.etree.ElementTree.Element` is deprecated. In a future release it + will always return ``True``. Prefer explicit ``len(elem)`` or + ``elem is not None`` tests instead. + +* :func:`sys._clear_type_cache` is deprecated: + use :func:`sys._clear_internal_caches` instead. diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index d237f8f082d87b..2430564b45d6d2 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -1,174 +1,19 @@ +:orphan: + +.. This page is retained solely for existing links to /distributing/index.html. + Direct readers to the PPUG instead. + .. _distributing-index: ############################### Distributing Python Modules ############################### -:Email: distutils-sig@python.org - - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the distribution part of the process. For a guide to -installing other Python projects, refer to the -:ref:`installation guide <installing-index>`. - .. note:: - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* the `Python Package Index <https://pypi.org>`__ is a public - repository of open source licensed packages made available for use by - other Python users -* the `Python Packaging Authority - <https://www.pypa.io/>`__ are the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation - and issue trackers on `GitHub <https://github.com/pypa>`__. -* ``distutils`` is the original build and distribution system first added - to the Python standard library in 1998. While direct use of ``distutils`` - is being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). -* `setuptools`_ is a (largely) drop-in replacement for ``distutils`` first - published in 2004. Its most notable addition over the unmodified - ``distutils`` tools was the ability to declare dependencies on other - packages. It is currently recommended as a more regularly updated - alternative to ``distutils`` that offers consistent support for more - recent packaging standards across a wide range of Python versions. -* `wheel`_ (in this context) is a project that adds the ``bdist_wheel`` - command to ``distutils``/`setuptools`_. This produces a cross platform - binary packaging format (called "wheels" or "wheel files" and defined in - :pep:`427`) that allows Python libraries, even those including binary - extensions, to be installed on a system without needing to be built - locally. - -.. _setuptools: https://setuptools.readthedocs.io/en/latest/ -.. _wheel: https://wheel.readthedocs.io/ - -Open source licensing and collaboration -======================================= - -In most parts of the world, software is automatically covered by copyright. -This means that other developers require explicit permission to copy, use, -modify and redistribute the software. - -Open source licensing is a way of explicitly granting such permission in a -relatively consistent way, allowing developers to share and collaborate -efficiently by making common solutions to various problems freely available. -This leaves many developers free to spend more time focusing on the problems -that are relatively unique to their specific situation. - -The distribution tools provided with Python are designed to make it -reasonably straightforward for developers to make their own contributions -back to that common pool of software if they choose to do so. - -The same distribution tools can also be used to distribute software within -an organisation, regardless of whether that software is published as open -source software or not. - - -Installing the tools -==================== - -The standard library does not include build tools that support modern -Python packaging standards, as the core development team has found that it -is important to have standard tools that work consistently, even on older -versions of Python. - -The currently recommended build and distribution tools can be installed -by invoking the ``pip`` module at the command line:: - - python -m pip install setuptools wheel twine - -.. note:: - - For POSIX users (including macOS and Linux users), these instructions - assume the use of a :term:`virtual environment`. - - For Windows users, these instructions assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -The Python Packaging User Guide includes more details on the `currently -recommended tools`_. - -.. _currently recommended tools: https://packaging.python.org/guides/tool-recommendations/#packaging-tool-recommendations - -.. index:: - single: Python Package Index (PyPI) - single: PyPI; (see Python Package Index (PyPI)) - -.. _publishing-python-packages: - -Reading the Python Packaging User Guide -======================================= - -The Python Packaging User Guide covers the various key steps and elements -involved in creating and publishing a project: - -* `Project structure`_ -* `Building and packaging the project`_ -* `Uploading the project to the Python Package Index`_ -* `The .pypirc file`_ - -.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ - - -How do I...? -============ - -These are quick answers or links for some common tasks. - -... choose a name for my project? ---------------------------------- - -This isn't an easy topic, but here are a few tips: - -* check the Python Package Index to see if the name is already in use -* check popular hosting sites like GitHub, Bitbucket, etc to see if there - is already a project with that name -* check what comes up in a web search for the name you're considering -* avoid particularly common words, especially ones with multiple meanings, - as they can make it difficult for users to find your software when - searching for it - - -... create and distribute binary extensions? --------------------------------------------- - -This is actually quite a complex topic, with a variety of alternatives -available depending on exactly what you're aiming to achieve. See the -Python Packaging User Guide for more information and recommendations. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - <https://packaging.python.org/guides/packaging-binary-extensions/>`__ - -.. other topics: + Information and guidance on distributing Python modules and packages + has been moved to the `Python Packaging User Guide`_, + and the tutorial on `packaging Python projects`_. - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + .. _Python Packaging User Guide: https://packaging.python.org/ + .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index 880bb33ee56718..a58eb40d431c59 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -23,10 +23,10 @@ instance. See :ref:`initializing-modules` for details. .. highlight:: python For modules with ASCII-only names, the function must be named -``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names +:samp:`PyInit_{<name>}`, with ``<name>`` replaced by the name of the module. +When using :ref:`multi-phase-initialization`, non-ASCII module names are allowed. In this case, the initialization function name is -``PyInitU_<modulename>``, with ``<modulename>`` encoded using Python's +:samp:`PyInitU_{<name>}`, with ``<name>`` encoded using Python's *punycode* encoding with hyphens replaced by underscores. In Python:: def initfunc_name(name): @@ -45,6 +45,7 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. .. highlight:: c +.. _install-index: .. _setuptools-index: Building C and C++ Extensions with setuptools diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index 20397dc5add5db..cb41889437c8b0 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -196,8 +196,8 @@ interesting part with respect to embedding Python starts with :: After initializing the interpreter, the script is loaded using :c:func:`PyImport_Import`. This routine needs a Python string as its argument, -which is constructed using the :c:func:`PyUnicode_FromString` data conversion -routine. :: +which is constructed using the :c:func:`PyUnicode_DecodeFSDefault` data +conversion routine. :: pFunc = PyObject_GetAttrString(pModule, argv[2]); /* pFunc is a new reference */ @@ -245,21 +245,23 @@ Python extension. For example:: return PyLong_FromLong(numargs); } - static PyMethodDef EmbMethods[] = { + static PyMethodDef emb_module_methods[] = { {"numargs", emb_numargs, METH_VARARGS, "Return the number of arguments received by the process."}, {NULL, NULL, 0, NULL} }; - static PyModuleDef EmbModule = { - PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods, - NULL, NULL, NULL, NULL + static struct PyModuleDef emb_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "emb", + .m_size = 0, + .m_methods = emb_module_methods, }; static PyObject* PyInit_emb(void) { - return PyModule_Create(&EmbModule); + return PyModuleDef_Init(&emb_module); } Insert the above code just above the :c:func:`main` function. Also, insert the diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index f58b4f28113e8c..fd63495674651b 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -203,31 +203,57 @@ function usually raises :c:data:`PyExc_TypeError`. If you have an argument whos value must be in a particular range or must satisfy other conditions, :c:data:`PyExc_ValueError` is appropriate. -You can also define a new exception that is unique to your module. For this, you -usually declare a static object variable at the beginning of your file:: +You can also define a new exception that is unique to your module. +The simplest way to do this is to declare a static global object variable at +the beginning of the file:: - static PyObject *SpamError; + static PyObject *SpamError = NULL; -and initialize it in your module's initialization function (:c:func:`!PyInit_spam`) -with an exception object:: +and initialize it by calling :c:func:`PyErr_NewException` in the module's +:c:data:`Py_mod_exec` function (:c:func:`!spam_module_exec`):: - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; + SpamError = PyErr_NewException("spam.error", NULL, NULL); - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; +Since :c:data:`!SpamError` is a global variable, it will be overwitten every time +the module is reinitialized, when the :c:data:`Py_mod_exec` function is called. + +For now, let's avoid the issue: we will block repeated initialization by raising an +:py:exc:`ImportError`:: + static PyObject *SpamError = NULL; + + static int + spam_module_exec(PyObject *m) + { + if (SpamError != NULL) { + PyErr_SetString(PyExc_ImportError, + "cannot initialize spam module more than once"); + return -1; + } SpamError = PyErr_NewException("spam.error", NULL, NULL); - if (PyModule_AddObjectRef(m, "error", SpamError) < 0) { - Py_CLEAR(SpamError); - Py_DECREF(m); - return NULL; + if (PyModule_AddObjectRef(m, "SpamError", SpamError) < 0) { + return -1; } - return m; + return 0; + } + + static PyModuleDef_Slot spam_module_slots[] = { + {Py_mod_exec, spam_module_exec}, + {0, NULL} + }; + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + .m_size = 0, // non-negative + .m_slots = spam_module_slots, + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); } Note that the Python name for the exception object is :exc:`!spam.error`. The @@ -242,7 +268,12 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. -We discuss the use of ``PyMODINIT_FUNC`` as a function return type later in this +For now, the :c:func:`Py_DECREF` call to remove this reference is missing. +Even when the Python interpreter shuts down, the global :c:data:`!SpamError` +variable will not be garbage-collected. It will "leak". +We did, however, ensure that this will happen at most once per process. + +We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. The :exc:`!spam.error` exception can be raised in your extension module using a @@ -318,7 +349,7 @@ The Module's Method Table and Initialization Function I promised to show how :c:func:`!spam_system` is called from Python programs. First, we need to list its name and address in a "method table":: - static PyMethodDef SpamMethods[] = { + static PyMethodDef spam_methods[] = { ... {"system", spam_system, METH_VARARGS, "Execute a shell command."}, @@ -343,13 +374,10 @@ function. The method table must be referenced in the module definition structure:: - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods + static struct PyModuleDef spam_module = { + ... + .m_methods = spam_methods, + ... }; This structure, in turn, must be passed to the interpreter in the module's @@ -360,23 +388,17 @@ only non-\ ``static`` item defined in the module file:: PyMODINIT_FUNC PyInit_spam(void) { - return PyModule_Create(&spammodule); + return PyModuleDef_Init(&spam_module); } -Note that PyMODINIT_FUNC declares the function as ``PyObject *`` return type, +Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. -When the Python program imports module :mod:`!spam` for the first time, -:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.) -It calls :c:func:`PyModule_Create`, which returns a module object, and -inserts built-in function objects into the newly created module based upon the -table (an array of :c:type:`PyMethodDef` structures) found in the module definition. -:c:func:`PyModule_Create` returns a pointer to the module object -that it creates. It may abort with a fatal error for -certain errors, or return ``NULL`` if the module could not be initialized -satisfactorily. The init function must return the module object to its caller, -so that it then gets inserted into ``sys.modules``. +:c:func:`!PyInit_spam` is called when each interpreter imports its module +:mod:`!spam` for the first time. (See below for comments about embedding Python.) +A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`, +so that the import machinery can create the module and store it in ``sys.modules``. When embedding Python, the :c:func:`!PyInit_spam` function is not called automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. @@ -433,23 +455,19 @@ optionally followed by an import of the module:: .. note:: - Removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process (or following a :c:func:`fork` without an - intervening :c:func:`exec`) can create problems for some extension modules. - Extension module authors should exercise caution when initializing internal data - structures. + If you declare a global variable or a local static one, the module may + experience unintended side-effects on re-initialisation, for example when + removing entries from ``sys.modules`` or importing compiled modules into + multiple interpreters within a process + (or following a :c:func:`fork` without an intervening :c:func:`exec`). + If module state is not yet fully :ref:`isolated <isolating-extensions-howto>`, + authors should consider marking the module as having no support for subinterpreters + (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`). A more substantial example module is included in the Python source distribution -as :file:`Modules/xxmodule.c`. This file may be used as a template or simply +as :file:`Modules/xxlimited.c`. This file may be used as a template or simply read as an example. -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - .. _compilation: @@ -547,7 +565,7 @@ reference count of an object and are safe in the presence of ``NULL`` pointers (but note that *temp* will not be ``NULL`` in this context). More info on them in section :ref:`refcounts`. -.. index:: single: PyObject_CallObject() +.. index:: single: PyObject_CallObject (C function) Later, when it is time to call the function, you call the C function :c:func:`PyObject_CallObject`. This function has two arguments, both pointers to @@ -638,7 +656,7 @@ the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. : Extracting Parameters in Extension Functions ============================================ -.. index:: single: PyArg_ParseTuple() +.. index:: single: PyArg_ParseTuple (C function) The :c:func:`PyArg_ParseTuple` function is declared as follows:: @@ -730,12 +748,12 @@ Some example calls:: Keyword Parameters for Extension Functions ========================================== -.. index:: single: PyArg_ParseTupleAndKeywords() +.. index:: single: PyArg_ParseTupleAndKeywords (C function) The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); + const char *format, char * const *kwlist, ...); The *arg* and *format* parameters are identical to those of the :c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of @@ -790,18 +808,17 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods + static struct PyModuleDef keywdarg_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "keywdarg", + .m_size = 0, + .m_methods = keywdarg_methods, }; PyMODINIT_FUNC PyInit_keywdarg(void) { - return PyModule_Create(&keywdargmodule); + return PyModuleDef_Init(&keywdarg_module); } @@ -868,7 +885,7 @@ It is important to call :c:func:`free` at the right time. If a block's address is forgotten but :c:func:`free` is not called for it, the memory it occupies cannot be reused until the program terminates. This is called a :dfn:`memory leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block +continues to use the block, it creates a conflict with reuse of the block through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. It has the same bad consequences as referencing uninitialized data --- core dumps, wrong results, mysterious crashes. @@ -1072,8 +1089,9 @@ why his :meth:`!__del__` methods would fail... The second case of problems with a borrowed reference is a variant involving threads. Normally, multiple threads in the Python interpreter can't get in each -other's way, because there is a global lock protecting Python's entire object -space. However, it is possible to temporarily release this lock using the macro +other's way, because there is a :term:`global lock <global interpreter lock>` +protecting Python's entire object space. +However, it is possible to temporarily release this lock using the macro :c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using :c:macro:`Py_END_ALLOW_THREADS`. This is common around blocking I/O calls, to let other threads use the processor while waiting for the I/O to complete. @@ -1259,20 +1277,15 @@ two more lines must be added:: #include "spammodule.h" The ``#define`` is used to tell the header file that it is being included in the -exporting module, not a client module. Finally, the module's initialization -function must take care of initializing the C API pointer array:: +exporting module, not a client module. Finally, the module's :c:data:`mod_exec +<Py_mod_exec>` function must take care of initializing the C API pointer array:: - PyMODINIT_FUNC - PyInit_spam(void) + static int + spam_module_exec(PyObject *m) { - PyObject *m; static void *PySpam_API[PySpam_API_pointers]; PyObject *c_api_object; - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - /* Initialize the C API pointer array */ PySpam_API[PySpam_System_NUM] = (void *)PySpam_System; @@ -1280,11 +1293,10 @@ function must take care of initializing the C API pointer array:: c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Note that ``PySpam_API`` is declared ``static``; otherwise the pointer @@ -1343,20 +1355,16 @@ like this:: All that a client module must do in order to have access to the function :c:func:`!PySpam_System` is to call the function (or rather macro) -:c:func:`!import_spam` in its initialization function:: +:c:func:`!import_spam` in its :c:data:`mod_exec <Py_mod_exec>` function:: - PyMODINIT_FUNC - PyInit_client(void) + static int + client_module_exec(PyObject *m) { - PyObject *m; - - m = PyModule_Create(&clientmodule); - if (m == NULL) - return NULL; - if (import_spam() < 0) - return NULL; + if (import_spam() < 0) { + return -1; + } /* additional initialization can happen here */ - return m; + return 0; } The main disadvantage of this approach is that the file :file:`spammodule.h` is diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 01b4df6d44acff..4cc2c96d8d5b47 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -26,19 +26,9 @@ Recommended third party tools ============================= This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like -`Cython <https://cython.org/>`_, `cffi <https://cffi.readthedocs.io>`_, -`SWIG <https://www.swig.org>`_ and `Numba <https://numba.pydata.org/>`_ -offer both simpler and more sophisticated approaches to creating C and C++ -extensions for Python. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/guides/packaging-binary-extensions/>`_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. +as part of this version of CPython. Some :ref:`third party tools +<c-api-tools>` offer both simpler and more sophisticated approaches to creating +C and C++ extensions for Python. Creating extensions without third party tools @@ -49,6 +39,10 @@ assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. +.. seealso:: + + :pep:`489` -- Multi-phase extension module initialization + .. toctree:: :maxdepth: 2 :numbered: diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 386b3c8f4452c3..e3612f3a1875ca 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -70,27 +70,29 @@ object itself needs to be freed here as well. Here is an example of this function:: static void - newdatatype_dealloc(newdatatypeobject *obj) + newdatatype_dealloc(PyObject *op) { - free(obj->obj_UnderlyingDatatypePtr); - Py_TYPE(obj)->tp_free((PyObject *)obj); + newdatatypeobject *self = (newdatatypeobject *) op; + free(self->obj_UnderlyingDatatypePtr); + Py_TYPE(self)->tp_free(self); } If your type supports garbage collection, the destructor should call :c:func:`PyObject_GC_UnTrack` before clearing any member fields:: static void - newdatatype_dealloc(newdatatypeobject *obj) + newdatatype_dealloc(PyObject *op) { - PyObject_GC_UnTrack(obj); - Py_CLEAR(obj->other_obj); + newdatatypeobject *self = (newdatatypeobject *) op; + PyObject_GC_UnTrack(op); + Py_CLEAR(self->other_obj); ... - Py_TYPE(obj)->tp_free((PyObject *)obj); + Py_TYPE(self)->tp_free(self); } .. index:: - single: PyErr_Fetch() - single: PyErr_Restore() + single: PyErr_Fetch (C function) + single: PyErr_Restore (C function) One important requirement of the deallocator function is that it leaves any pending exceptions alone. This is important since deallocators are frequently @@ -117,17 +119,19 @@ done. This can be done using the :c:func:`PyErr_Fetch` and PyErr_Fetch(&err_type, &err_value, &err_traceback); cbresult = PyObject_CallNoArgs(self->my_callback); - if (cbresult == NULL) - PyErr_WriteUnraisable(self->my_callback); - else + if (cbresult == NULL) { + PyErr_WriteUnraisable(self->my_callback); + } + else { Py_DECREF(cbresult); + } /* This restores the saved exception state */ PyErr_Restore(err_type, err_value, err_traceback); Py_DECREF(self->my_callback); } - Py_TYPE(obj)->tp_free((PyObject*)self); + Py_TYPE(self)->tp_free(self); } .. note:: @@ -168,10 +172,11 @@ representation of the instance for which it is called. Here is a simple example:: static PyObject * - newdatatype_repr(newdatatypeobject *obj) + newdatatype_repr(PyObject *op) { + newdatatypeobject *self = (newdatatypeobject *) op; return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); + self->obj_UnderlyingDatatypePtr->size); } If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a @@ -188,10 +193,11 @@ used instead. Here is a simple example:: static PyObject * - newdatatype_str(newdatatypeobject *obj) + newdatatype_str(PyObject *op) { + newdatatypeobject *self = (newdatatypeobject *) op; return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); + self->obj_UnderlyingDatatypePtr->size); } @@ -296,9 +302,9 @@ An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`__doc__` attribute. +class object, and get the doc string using its :attr:`~type.__doc__` attribute. -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value of ``NULL`` is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -329,16 +335,16 @@ method of a class would be called. Here is an example:: static PyObject * - newdatatype_getattr(newdatatypeobject *obj, char *name) + newdatatype_getattr(PyObject *op, char *name) { - if (strcmp(name, "data") == 0) - { - return PyLong_FromLong(obj->data); + newdatatypeobject *self = (newdatatypeobject *) op; + if (strcmp(name, "data") == 0) { + return PyLong_FromLong(self->data); } PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%.400s'", - Py_TYPE(obj)->tp_name, name); + Py_TYPE(self)->tp_name, name); return NULL; } @@ -349,7 +355,7 @@ example that simply raises an exception; if this were really all you wanted, the :c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. :: static int - newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) + newdatatype_setattr(PyObject *op, char *name, PyObject *v) { PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name); return -1; @@ -379,8 +385,10 @@ Here is a sample implementation, for a datatype that is considered equal if the size of an internal pointer is equal:: static PyObject * - newdatatype_richcmp(newdatatypeobject *obj1, newdatatypeobject *obj2, int op) + newdatatype_richcmp(PyObject *lhs, PyObject *rhs, int op) { + newdatatypeobject *obj1 = (newdatatypeobject *) lhs; + newdatatypeobject *obj2 = (newdatatypeobject *) rhs; PyObject *result; int c, size1, size2; @@ -399,8 +407,7 @@ size of an internal pointer is equal:: case Py_GE: c = size1 >= size2; break; } result = c ? Py_True : Py_False; - Py_INCREF(result); - return result; + return Py_NewRef(result); } @@ -439,12 +446,14 @@ This function, if you choose to provide it, should return a hash number for an instance of your data type. Here is a simple example:: static Py_hash_t - newdatatype_hash(newdatatypeobject *obj) + newdatatype_hash(PyObject *op) { + newdatatypeobject *self = (newdatatypeobject *) op; Py_hash_t result; - result = obj->some_size + 32767 * obj->some_number; - if (result == -1) - result = -2; + result = self->some_size + 32767 * self->some_number; + if (result == -1) { + result = -2; + } return result; } @@ -478,8 +487,9 @@ This function takes three arguments: Here is a toy ``tp_call`` implementation:: static PyObject * - newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *kwds) + newdatatype_call(PyObject *op, PyObject *args, PyObject *kwds) { + newdatatypeobject *self = (newdatatypeobject *) op; PyObject *result; const char *arg1; const char *arg2; @@ -490,7 +500,7 @@ Here is a toy ``tp_call`` implementation:: } result = PyUnicode_FromFormat( "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n", - obj->obj_UnderlyingDatatypePtr->size, + self->obj_UnderlyingDatatypePtr->size, arg1, arg2, arg3); return result; } @@ -545,7 +555,7 @@ performance-critical objects (such as numbers). .. seealso:: Documentation for the :mod:`weakref` module. -For an object to be weakly referencable, the extension type must set the +For an object to be weakly referenceable, the extension type must set the ``Py_TPFLAGS_MANAGED_WEAKREF`` bit of the :c:member:`~PyTypeObject.tp_flags` field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. @@ -563,12 +573,12 @@ The only further addition is that ``tp_dealloc`` needs to clear any weak references (by calling :c:func:`PyObject_ClearWeakRefs`):: static void - Trivial_dealloc(TrivialObject *self) + Trivial_dealloc(PyObject *op) { /* Clear weakrefs first before calling any destructors */ - PyObject_ClearWeakRefs((PyObject *) self); + PyObject_ClearWeakRefs(op); /* ... remainder of destruction code omitted for brevity ... */ - Py_TYPE(self)->tp_free((PyObject *) self); + Py_TYPE(op)->tp_free(op); } diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 209a4ab76d226d..f14690de4f86e8 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -45,7 +45,7 @@ extension module :mod:`!custom`: allows defining heap-allocated extension types using the :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. -.. literalinclude:: ../includes/custom.c +.. literalinclude:: ../includes/newtypes/custom.c Now that's quite a bit to take in at once, but hopefully bits will seem familiar from the previous chapter. This file defines three things: @@ -55,8 +55,10 @@ from the previous chapter. This file defines three things: #. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct, which defines a set of flags and function pointers that the interpreter inspects when specific operations are requested. -#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom`` - function and the associated ``custommodule`` struct. +#. How to define and execute the :mod:`!custom` module: this is the + ``PyInit_custom`` function and the associated ``custom_module`` struct for + defining the module, and the ``custom_module_exec`` function to set up + a fresh module object. The first bit is:: @@ -144,7 +146,7 @@ only used for variable-sized objects and should otherwise be zero. If you want your type to be subclassable from Python, and your type has the same :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's + in its :attr:`~type.__bases__`, or else it will not be able to call your type's :meth:`~object.__new__` method without getting an error. You can avoid this problem by ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your @@ -171,18 +173,18 @@ implementation provided by the API function :c:func:`PyType_GenericNew`. :: .tp_new = PyType_GenericNew, Everything else in the file should be familiar, except for some code in -:c:func:`!PyInit_custom`:: +:c:func:`!custom_module_exec`:: - if (PyType_Ready(&CustomType) < 0) - return; + if (PyType_Ready(&CustomType) < 0) { + return -1; + } This initializes the :class:`!Custom` type, filling in a number of members to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially set to ``NULL``. :: if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } This adds the type to the module dictionary. This allows us to create @@ -194,36 +196,32 @@ This adds the type to the module dictionary. This allows us to create >>> mycustom = custom.Custom() That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: +:file:`custom.c`, + +.. literalinclude:: ../includes/newtypes/pyproject.toml + +in a file called :file:`pyproject.toml`, and .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) + from setuptools import Extension, setup + setup(ext_modules=[Extension("custom", ["custom.c"])]) in a file called :file:`setup.py`; then typing .. code-block:: shell-session - $ python setup.py build + $ python -m pip install . -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. +in a shell should produce a file :file:`custom.so` in a subdirectory +and install it; now fire up Python --- you should be able to ``import custom`` +and play around with ``Custom`` objects. That wasn't so hard, was it? Of course, the current Custom type is pretty uninteresting. It has no data and doesn't do anything. It can't even be subclassed. -.. note:: - While this documentation showcases the standard :mod:`!distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide <https://packaging.python.org/tutorials/distributing-packages/>`_. - Adding data and methods to the Basic example ============================================ @@ -232,7 +230,7 @@ Let's extend the basic example to add some data and methods. Let's also make the type usable as a base class. We'll create a new module, :mod:`!custom2` that adds these capabilities: -.. literalinclude:: ../includes/custom2.c +.. literalinclude:: ../includes/newtypes/custom2.c This version of the module has a number of changes. @@ -254,16 +252,17 @@ Because we now have data to manage, we have to be more careful about object allocation and deallocation. At a minimum, we need a deallocation method:: static void - Custom_dealloc(CustomObject *self) + Custom_dealloc(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_XDECREF(self->first); Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); + Py_TYPE(self)->tp_free(self); } which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: - .tp_dealloc = (destructor) Custom_dealloc, + .tp_dealloc = Custom_dealloc, This method first clears the reference counts of the two Python attributes. :c:func:`Py_XDECREF` correctly handles the case where its argument is @@ -274,11 +273,31 @@ the object's type might not be :class:`!CustomType`, because the object may be an instance of a subclass. .. note:: - The explicit cast to ``destructor`` above is needed because we defined - ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc`` - function pointer expects to receive a ``PyObject *`` argument. Otherwise, - the compiler will emit a warning. This is object-oriented polymorphism, - in C! + + The explicit cast to ``CustomObject *`` above is needed because we defined + ``Custom_dealloc`` to take a ``PyObject *`` argument, as the ``tp_dealloc`` + function pointer expects to receive a ``PyObject *`` argument. + By assigning to the the ``tp_dealloc`` slot of a type, we declare + that it can only be called with instances of our ``CustomObject`` + class, so the cast to ``(CustomObject *)`` is safe. + This is object-oriented polymorphism, in C! + + In existing code, or in previous versions of this tutorial, + you might see similar functions take a pointer to the subtype + object structure (``CustomObject*``) directly, like this:: + + Custom_dealloc(CustomObject *self) + { + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); + } + ... + .tp_dealloc = (destructor) Custom_dealloc, + + This does the same thing on all architectures that CPython + supports, but according to the C standard, it invokes + undefined behavior. We want to make sure that the first and last names are initialized to empty strings, so we provide a ``tp_new`` implementation:: @@ -356,8 +375,9 @@ We also define an initialization function which accepts arguments to provide initial values for our instance:: static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) + Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL, *tmp; @@ -383,7 +403,7 @@ initial values for our instance:: by filling the :c:member:`~PyTypeObject.tp_init` slot. :: - .tp_init = (initproc) Custom_init, + .tp_init = Custom_init, The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`~object.__init__` method. It is used to initialize an object after it's @@ -407,8 +427,8 @@ the new attribute values. We might be tempted, for example to assign the But this would be risky. Our type doesn't restrict the type of the ``first`` member, so it could be any kind of object. It could have a destructor that causes code to be executed that tries to access the -``first`` member; or that destructor could release the -:term:`Global interpreter Lock <GIL>` and let arbitrary code run in other +``first`` member; or that destructor could detach the +:term:`thread state <attached thread state>` and let arbitrary code run in other threads that accesses and modifies our object. To be paranoid and protect ourselves against this possibility, we almost @@ -417,8 +437,8 @@ don't we have to do this? * when we absolutely know that the reference count is greater than 1; -* when we know that deallocation of the object [#]_ will neither release - the :term:`GIL` nor cause any calls back into our type's code; +* when we know that deallocation of the object [#]_ will neither detach + the :term:`thread state <attached thread state>` nor cause any calls back into our type's code; * when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler on a type which doesn't support cyclic garbage collection [#]_. @@ -451,12 +471,13 @@ Further, the attributes can be deleted, setting the C pointers to ``NULL``. Eve though we can make sure the members are initialized to non-``NULL`` values, the members can be set to ``NULL`` if the attributes are deleted. -We define a single method, :meth:`!Custom.name()`, that outputs the objects name as the +We define a single method, :meth:`!Custom.name`, that outputs the objects name as the concatenation of the first and last names. :: static PyObject * - Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) + Custom_name(PyObject *op, PyObject *Py_UNUSED(dummy)) { + CustomObject *self = (CustomObject *) op; if (self->first == NULL) { PyErr_SetString(PyExc_AttributeError, "first"); return NULL; @@ -490,7 +511,7 @@ Now that we've defined the method, we need to create an array of method definitions:: static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, + {"name", Custom_name, METH_NOARGS, "Return the name, combining the first and last name" }, {NULL} /* Sentinel */ @@ -514,17 +535,21 @@ We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the module name in the :c:type:`PyModuleDef` struct, and update the full class name in the :c:type:`PyTypeObject` struct. -Finally, we update our :file:`setup.py` file to build the new module: +Finally, we update our :file:`setup.py` file to include the new module, .. code-block:: python - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) + from setuptools import Extension, setup + setup(ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + ]) + +and then we re-install so that we can ``import custom2``: + +.. code-block:: shell-session + $ python -m pip install . Providing finer control over data attributes ============================================ @@ -535,7 +560,7 @@ version of our module, the instance variables :attr:`!first` and :attr:`!last` could be set to non-string values or even deleted. We want to make sure that these attributes always contain strings. -.. literalinclude:: ../includes/custom3.c +.. literalinclude:: ../includes/newtypes/custom3.c To provide greater control, over the :attr:`!first` and :attr:`!last` attributes, @@ -543,15 +568,17 @@ we'll use custom getter and setter functions. Here are the functions for getting and setting the :attr:`!first` attribute:: static PyObject * - Custom_getfirst(CustomObject *self, void *closure) + Custom_getfirst(PyObject *op, void *closure) { + CustomObject *self = (CustomObject *) op; Py_INCREF(self->first); return self->first; } static int - Custom_setfirst(CustomObject *self, PyObject *value, void *closure) + Custom_setfirst(PyObject *op, PyObject *value, void *closure) { + CustomObject *self = (CustomObject *) op; PyObject *tmp; if (value == NULL) { PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); @@ -583,9 +610,9 @@ new value is not a string. We create an array of :c:type:`PyGetSetDef` structures:: static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + {"first", Custom_getfirst, Custom_setfirst, "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, + {"last", Custom_getlast, Custom_setlast, "last name", NULL}, {NULL} /* Sentinel */ }; @@ -609,8 +636,9 @@ We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to be passed:: static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) + Custom_init(PyObject *op, PyObject *args, PyObject *kwds) { + CustomObject *self = (CustomObject *) op; static char *kwlist[] = {"first", "last", "number", NULL}; PyObject *first = NULL, *last = NULL, *tmp; @@ -682,15 +710,16 @@ To allow a :class:`!Custom` instance participating in a reference cycle to be properly detected and collected by the cyclic GC, our :class:`!Custom` type needs to fill two additional slots and to enable a flag that enables these slots: -.. literalinclude:: ../includes/custom4.c +.. literalinclude:: ../includes/newtypes/custom4.c First, the traversal method lets the cyclic GC know about subobjects that could participate in cycles:: static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) + Custom_traverse(PyObject *op, visitproc visit, void *arg) { + CustomObject *self = (CustomObject *) op; int vret; if (self->first) { vret = visit(self->first, arg); @@ -716,8 +745,9 @@ functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate in ``Custom_traverse``:: static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) + Custom_traverse(PyObject *op, visitproc visit, void *arg) { + CustomObject *self = (CustomObject *) op; Py_VISIT(self->first); Py_VISIT(self->last); return 0; @@ -731,8 +761,9 @@ Second, we need to provide a method for clearing any subobjects that can participate in cycles:: static int - Custom_clear(CustomObject *self) + Custom_clear(PyObject *op) { + CustomObject *self = (CustomObject *) op; Py_CLEAR(self->first); Py_CLEAR(self->last); return 0; @@ -765,11 +796,11 @@ Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack` and ``Custom_clear``:: static void - Custom_dealloc(CustomObject *self) + Custom_dealloc(PyObject *op) { - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); + PyObject_GC_UnTrack(op); + (void)Custom_clear(op); + Py_TYPE(op)->tp_free(op); } Finally, we add the :c:macro:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: @@ -806,7 +837,7 @@ increases an internal counter: >>> print(s.increment()) 2 -.. literalinclude:: ../includes/sublist.c +.. literalinclude:: ../includes/newtypes/sublist.c As you can see, the source code closely resembles the :class:`!Custom` examples in @@ -825,9 +856,10 @@ When a Python object is a :class:`!SubList` instance, its ``PyObject *`` pointer can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: static int - SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) + SubList_init(PyObject *op, PyObject *args, PyObject *kwds) { - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) + SubListObject *self = (SubListObject *) op; + if (PyList_Type.tp_init(op, args, kwds) < 0) return -1; self->state = 0; return 0; @@ -845,27 +877,22 @@ but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_ The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` specifying the type's concrete base class. Due to cross-platform compiler issues, you can't fill that field directly with a reference to -:c:type:`PyList_Type`; it should be done later in the module initialization +:c:type:`PyList_Type`; it should be done in the :c:data:`Py_mod_exec` function:: - PyMODINIT_FUNC - PyInit_sublist(void) + static int + sublist_module_exec(PyObject *m) { - PyObject* m; SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; + if (PyType_Ready(&SubListType) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Before calling :c:func:`PyType_Ready`, the type structure must have the diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 1129b0968bc4e6..a97c6182553c30 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -96,6 +96,13 @@ gives you access to spam's names, but does not create a separate copy. On Unix, linking with a library is more like ``from spam import *``; it does create a separate copy. +.. c:macro:: Py_NO_LINK_LIB + + Turn off the implicit, ``#pragma``-based linkage with the Python + library, performed inside CPython header files. + + .. versionadded:: 3.14 + .. _win-dlls: @@ -108,21 +115,46 @@ Using DLLs in Practice Windows Python is built in Microsoft Visual C++; using other compilers may or may not work. The rest of this section is MSVC++ specific. -When creating DLLs in Windows, you must pass :file:`pythonXY.lib` to the linker. -To build two DLLs, spam and ni (which uses C functions found in spam), you could -use these commands:: +When creating DLLs in Windows, you can use the CPython library in two ways: + +1. By default, inclusion of :file:`PC/pyconfig.h` directly or via + :file:`Python.h` triggers an implicit, configure-aware link with the + library. The header file chooses :file:`pythonXY_d.lib` for Debug, + :file:`pythonXY.lib` for Release, and :file:`pythonX.lib` for Release with + the :ref:`Limited API <stable-application-binary-interface>` enabled. + + To build two DLLs, spam and ni (which uses C functions found in spam), you + could use these commands:: + + cl /LD /I/python/include spam.c + cl /LD /I/python/include ni.c spam.lib + + The first command created three files: :file:`spam.obj`, :file:`spam.dll` + and :file:`spam.lib`. :file:`Spam.dll` does not contain any Python + functions (such as :c:func:`PyArg_ParseTuple`), but it does know how to find + the Python code thanks to the implicitly linked :file:`pythonXY.lib`. + + The second command created :file:`ni.dll` (and :file:`.obj` and + :file:`.lib`), which knows how to find the necessary functions from spam, + and also from the Python executable. + +2. Manually by defining :c:macro:`Py_NO_LINK_LIB` macro before including + :file:`Python.h`. You must pass :file:`pythonXY.lib` to the linker. + + To build two DLLs, spam and ni (which uses C functions found in spam), you + could use these commands:: - cl /LD /I/python/include spam.c ../libs/pythonXY.lib - cl /LD /I/python/include ni.c spam.lib ../libs/pythonXY.lib + cl /LD /DPy_NO_LINK_LIB /I/python/include spam.c ../libs/pythonXY.lib + cl /LD /DPy_NO_LINK_LIB /I/python/include ni.c spam.lib ../libs/pythonXY.lib -The first command created three files: :file:`spam.obj`, :file:`spam.dll` and -:file:`spam.lib`. :file:`Spam.dll` does not contain any Python functions (such -as :c:func:`PyArg_ParseTuple`), but it does know how to find the Python code -thanks to :file:`pythonXY.lib`. + The first command created three files: :file:`spam.obj`, :file:`spam.dll` + and :file:`spam.lib`. :file:`Spam.dll` does not contain any Python + functions (such as :c:func:`PyArg_ParseTuple`), but it does know how to find + the Python code thanks to :file:`pythonXY.lib`. -The second command created :file:`ni.dll` (and :file:`.obj` and :file:`.lib`), -which knows how to find the necessary functions from spam, and also from the -Python executable. + The second command created :file:`ni.dll` (and :file:`.obj` and + :file:`.lib`), which knows how to find the necessary functions from spam, + and also from the Python executable. Not every identifier is exported to the lookup table. If you want any other modules (including Python) to be able to see your identifiers, you have to say @@ -132,4 +164,4 @@ modules (including Python) to be able to see your identifiers, you have to say Developer Studio will throw in a lot of import libraries that you do not really need, adding about 100K to your executable. To get rid of them, use the Project Settings dialog, Link tab, to specify *ignore default libraries*. Add the -correct :file:`msvcrtxx.lib` to the list of libraries. +correct :file:`msvcrt{xx}.lib` to the list of libraries. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 11d01374dc1e79..c758c019ca437b 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -70,7 +70,7 @@ operations. This means that as far as floating-point operations are concerned, Python behaves like many popular languages including C and Java. Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: +exactly in binary floating point. For example, after:: >>> x = 1.2 @@ -87,7 +87,7 @@ which is exactly:: The typical precision of 53 bits provides Python floats with 15--16 decimal digits of accuracy. -For a fuller explanation, please see the :ref:`floating point arithmetic +For a fuller explanation, please see the :ref:`floating-point arithmetic <tut-fp-issues>` chapter in the Python tutorial. @@ -259,9 +259,11 @@ is evaluated in all cases. Why isn't there a switch or case statement in Python? ----------------------------------------------------- -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. +In general, structured switch statements execute one block of code +when an expression has a particular value or set of values. +Since Python 3.10 one can easily match literal values, or constants +within a namespace, with a ``match ... case`` statement. +An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, you can create a dictionary mapping case values to functions to call. For @@ -290,6 +292,9 @@ It's suggested that you use a prefix for the method names, such as ``visit_`` in this example. Without such a prefix, if values are coming from an untrusted source, an attacker would be able to call any method on your object. +Imitating switch with fallthrough, as with C's switch-case-default, +is possible, much harder, and less needed. + Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? -------------------------------------------------------------------------------------------------------- @@ -323,7 +328,7 @@ Can Python be compiled to machine code, C or some other language? ----------------------------------------------------------------- `Cython <https://cython.org/>`_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka <https://www.nuitka.net/>`_ is +optional annotations into C extensions. `Nuitka <https://nuitka.net/>`_ is an up-and-coming compiler of Python into C++ code, aiming to support the full Python language. @@ -340,7 +345,7 @@ to perform a garbage collection, obtain debugging statistics, and tune the collector's parameters. Other implementations (such as `Jython <https://www.jython.org>`_ or -`PyPy <https://www.pypy.org>`_), however, can rely on a different mechanism +`PyPy <https://pypy.org>`_), however, can rely on a different mechanism such as a full-blown garbage collector. This difference can cause some subtle porting problems if your Python code depends on the behavior of the reference counting implementation. @@ -415,10 +420,12 @@ strings representing the files in the current directory. Functions which operate on this output would generally not break if you added another file or two to the directory. -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. +Tuples are :term:`immutable`, meaning that once a tuple has been created, you can't +replace any of its elements with a new value. Lists are :term:`mutable`, meaning that +you can always change a list's elements. Only :term:`hashable` objects can +be used as dictionary keys. Most immutable types are hashable, which is why +tuples, but not lists, can be used as keys. Note, however, that a tuple is +only hashable if all of its elements are hashable. How are lists implemented in CPython? @@ -451,7 +458,7 @@ on the key and a per-process seed; for example, ``'Python'`` could hash to to ``1142331976``. The hash code is then used to calculate a location in an internal array where the value will be stored. Assuming that you're storing keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. +constant time -- *O*\ (1), in Big-O notation -- to retrieve a key. Why must dictionary keys be immutable? @@ -584,9 +591,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`list.append` method is expected to add new elements +program. For example, the :meth:`!list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`list.append` implementation will actually do this correctly, but it's +your :meth:`!list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 2a8b976925d042..1d5abed2317b0c 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -37,24 +37,9 @@ Writing C is hard; are there any alternatives? ---------------------------------------------- There are a number of alternatives to writing your own C extensions, depending -on what you're trying to do. - -.. XXX make sure these all work - -`Cython <https://cython.org>`_ and its relative `Pyrex -<https://www.csse.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers -that accept a slightly modified form of Python and generate the corresponding -C code. Cython and Pyrex make it possible to write an extension without having -to learn Python's C API. - -If you need to interface to some C or C++ library for which no Python extension -currently exists, you can try wrapping the library's data types and functions -with a tool such as `SWIG <https://www.swig.org>`_. `SIP -<https://riverbankcomputing.com/software/sip/intro>`__, `CXX -<https://cxx.sourceforge.net/>`_ `Boost -<https://www.boost.org/libs/python/doc/index.html>`_, or `Weave -<https://github.com/scipy/weave>`_ are also -alternatives for wrapping C++ libraries. +on what you're trying to do. :ref:`Recommended third party tools <c-api-tools>` +offer both simpler and more sophisticated approaches to creating C and C++ +extensions for Python. How can I execute arbitrary Python statements from C? @@ -246,13 +231,12 @@ Then, when you run GDB: I want to compile a Python module on my Linux system, but some files are missing. Why? -------------------------------------------------------------------------------------- -Most packaged versions of Python don't include the -:file:`/usr/lib/python2.{x}/config/` directory, which contains various files +Most packaged versions of Python omit some files required for compiling Python extensions. -For Red Hat, install the python-devel RPM to get the necessary files. +For Red Hat, install the python3-devel RPM to get the necessary files. -For Debian, run ``apt-get install python-dev``. +For Debian, run ``apt-get install python3-dev``. How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------ diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 8727332594bda6..578777d7f23621 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -122,6 +122,8 @@ available. Consult `the Python Package Index <https://pypi.org>`_ to find packages of interest to you. +.. _faq-version-numbering-scheme: + How does the Python version numbering scheme work? -------------------------------------------------- @@ -133,8 +135,6 @@ Python versions are numbered "A.B.C" or "A.B": changes. * *C* is the micro version number -- it is incremented for each bugfix release. -See :pep:`6` for more information about bugfix releases. - Not all releases are bugfix releases. In the run-up to a new feature release, a series of development releases are made, denoted as alpha, beta, or release candidate. Alphas are early releases in which interfaces aren't yet finalized; @@ -157,7 +157,11 @@ unreleased versions, built directly from the CPython development repository. In practice, after a final minor release is made, the version is incremented to the next minor version, which becomes the "a0" version, e.g. "2.4a0". -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and +See the `Developer's Guide +<https://devguide.python.org/developer-workflow/development-cycle/>`__ +for more information about the development cycle, and +:pep:`387` to learn more about Python's backward compatibility policy. See also +the documentation for :data:`sys.version`, :data:`sys.hexversion`, and :data:`sys.version_info`. @@ -181,8 +185,6 @@ information on getting the source code and compiling it. How do I get documentation on Python? ------------------------------------- -.. XXX mention py3k - The standard documentation for the current stable version of Python is available at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. @@ -307,10 +309,9 @@ guaranteed that interfaces will remain the same throughout a series of bugfix releases. The latest stable releases can always be found on the `Python download page -<https://www.python.org/downloads/>`_. There are two production-ready versions -of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, `it is not -maintained anymore <https://peps.python.org/pep-0373/>`_. +<https://www.python.org/downloads/>`_. +Python 3.x is the recommended version and supported by most widely used libraries. +Python 2.x :pep:`is not maintained anymore <373>`. How many people are using Python? --------------------------------- diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 0a372342862d2f..cfa60feceb31b7 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -43,7 +43,7 @@ applications, the applications will not be truly stand-alone, as the application will still need the Tcl and Tk libraries. One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`TCL_LIBRARY` and :envvar:`TK_LIBRARY` +to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY` environment variables. Various third-party freeze libraries such as py2exe and cx_Freeze have @@ -55,7 +55,7 @@ Can I have Tk events handled while waiting for I/O? On platforms other than Windows, yes, and you don't even need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`XtAddInput()` call, which allows you +code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you to register a callback function which will be called from the Tk mainloop when I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. @@ -63,8 +63,9 @@ I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. I can't get key bindings to work in Tkinter: why? ------------------------------------------------- -An often-heard complaint is that event handlers bound to events with the -:meth:`bind` method don't get handled even when the appropriate key is pressed. +An often-heard complaint is that event handlers :ref:`bound <bindings-and-events>` +to events with the :meth:`!bind` method +don't get handled even when the appropriate key is pressed. The most common cause is that the widget to which the binding applies doesn't have "keyboard focus". Check out the Tk documentation for the focus command. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index 9e3727456bb96e..d8d75ca6f2ec96 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -111,7 +111,7 @@ Is there an equivalent to C's onexit() in Python? ------------------------------------------------- The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`onexit`. +:c:func:`!onexit`. Why don't my signal handlers work? @@ -397,7 +397,7 @@ These aren't:: D[x] = D[x] + 1 Operations that replace other objects may invoke those other objects' -:meth:`__del__` method when their reference count reaches zero, and that can +:meth:`~object.__del__` method when their reference count reaches zero, and that can affect things. This is especially true for the mass updates to dictionaries and lists. When in doubt, use a mutex! @@ -405,22 +405,37 @@ lists. When in doubt, use a mutex! Can't we get rid of the Global Interpreter Lock? ------------------------------------------------ -.. XXX link to dbeazley's talk about GIL? - The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's deployment on high-end multiprocessor server machines, because a multi-threaded Python program effectively only uses one CPU, due to the insistence that (almost) all Python code can only run while the GIL is held. -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive +With the approval of :pep:`703` work is now underway to remove the GIL from the +CPython implementation of Python. Initially it will be implemented as an +optional compiler flag when building the interpreter, and so separate +builds will be available with and without the GIL. Long-term, the hope is +to settle on a single build, once the performance implications of removing the +GIL are fully understood. Python 3.13 is likely to be the first release +containing this work, although it may not be completely functional in this +release. + +The current work to remove the GIL is based on a +`fork of Python 3.9 with the GIL removed <https://github.com/colesbury/nogil>`_ +by Sam Gross. +Prior to that, +in the days of Python 1.5, Greg Stein actually implemented a comprehensive patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment +with fine-grained locking. Adam Olsen did a similar experiment in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread +project. Unfortunately, both of these earlier experiments exhibited a sharp +drop in single-thread performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. +necessary to compensate for the removal of the GIL. The Python 3.9 fork +is the first attempt at removing the GIL with an acceptable performance +impact. -This doesn't mean that you can't make good use of Python on multi-CPU machines! +The presence of the GIL in current Python releases +doesn't mean that you can't make good use of Python on multi-CPU machines! You just have to be creative with dividing the work up between multiple *processes* rather than multiple *threads*. The :class:`~concurrent.futures.ProcessPoolExecutor` class in the new @@ -434,22 +449,13 @@ thread of execution is in the C code and allow other threads to get some work done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` already do this. -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? +An alternative approach to reducing the impact of the GIL is +to make the GIL a per-interpreter-state lock rather than truly global. +This was :ref:`first implemented in Python 3.12 <whatsnew312-pep684>` and is +available in the C API. A Python interface to it is expected in Python 3.13. +The main limitation to it at the moment is likely to be 3rd party extension +modules, since these must be written with multiple interpreters in mind in +order to be usable, so many older extension modules will not be usable. Input and Output @@ -535,91 +541,12 @@ Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need use ``p.read(n)``. -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from https://expectpy.sourceforge.net. A pure Python - solution that works like expect is `pexpect - <https://pypi.org/project/pexpect/>`_. - - How do I access the serial (RS232) port? ---------------------------------------- For Win32, OSX, Linux, BSD, Jython, IronPython: - https://pypi.org/project/pyserial/ + :pypi:`pyserial` For Unix, see a Usenet post by Mitch Chapman: @@ -730,14 +657,17 @@ The :mod:`select` module is commonly used to help with asynchronous I/O on sockets. To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`socket.connect`, you will either connect immediately +mode. Then when you do the :meth:`~socket.socket.connect`, +you will either connect immediately (unlikely) or get an exception that contains the error number as ``.errno``. ``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't finished yet. Different OSes will return different values, so you're going to have to check what's returned on your system. -You can use the :meth:`socket.connect_ex` method to avoid creating an exception. It will -just return the errno value. To poll, you can call :meth:`socket.connect_ex` again later +You can use the :meth:`~socket.socket.connect_ex` method +to avoid creating an exception. +It will just return the errno value. +To poll, you can call :meth:`~socket.socket.connect_ex` again later -- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this socket to :meth:`select.select` to check if it's writable. @@ -788,12 +718,12 @@ is simple:: import random random.random() -This returns a random floating point number in the range [0, 1). +This returns a random floating-point number in the range [0, 1). There are also many other specialized generators in this module, such as: * ``randrange(a, b)`` chooses an integer in the range [a, b). -* ``uniform(a, b)`` chooses a floating point number in the range [a, b). +* ``uniform(a, b)`` chooses a floating-point number in the range [a, b). * ``normalvariate(mean, sdev)`` samples the normal (Gaussian) distribution. Some higher-level functions operate on sequences directly, such as: diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 6e1812504a184f..9f9e4fab685b19 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -454,7 +454,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`~list.append`, the content of the mutable object has +After the call to :meth:`!append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -869,7 +869,7 @@ How do I convert a string to a number? -------------------------------------- For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') -== 144``. Similarly, :func:`float` converts to floating-point, +== 144``. Similarly, :func:`float` converts to a floating-point number, e.g. ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == @@ -986,8 +986,8 @@ There are various techniques. f() -Is there an equivalent to Perl's chomp() for removing trailing newlines from strings? -------------------------------------------------------------------------------------- +Is there an equivalent to Perl's ``chomp()`` for removing trailing newlines from strings? +----------------------------------------------------------------------------------------- You can use ``S.rstrip("\r\n")`` to remove all occurrences of any line terminator from the end of the string ``S`` without removing other trailing @@ -1005,23 +1005,23 @@ Since this is typically only desired when reading text one line at a time, using ``S.rstrip()`` this way works well. -Is there a scanf() or sscanf() equivalent? ------------------------------------------- +Is there a ``scanf()`` or ``sscanf()`` equivalent? +-------------------------------------------------- Not as such. For simple input parsing, the easiest approach is usually to split the line into whitespace-delimited words using the :meth:`~str.split` method of string objects and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. :meth:`!split()` supports an optional "sep" parameter which is useful +:func:`float`. :meth:`!split` supports an optional "sep" parameter which is useful if the line uses something other than whitespace as a separator. For more complicated input parsing, regular expressions are more powerful than C's ``sscanf`` and better suited for the task. -What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? -------------------------------------------------------------------- +What does ``UnicodeDecodeError`` or ``UnicodeEncodeError`` error mean? +---------------------------------------------------------------------- See the :ref:`unicode-howto`. @@ -1036,7 +1036,7 @@ A raw string ending with an odd number of backslashes will escape the string's q >>> r'C:\this\will\not\work\' File "<stdin>", line 1 r'C:\this\will\not\work\' - ^ + ^ SyntaxError: unterminated string literal (detected at line 1) There are several workarounds for this. One is to use regular strings and double @@ -1397,7 +1397,7 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`~list.extend` on the list +and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list and returning the list. That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`!list.extend`:: @@ -1613,9 +1613,16 @@ method too, and it must do so carefully. The basic implementation of self.__dict__[name] = value ... -Most :meth:`!__setattr__` implementations must modify -:meth:`self.__dict__ <object.__dict__>` to store -local state for self without causing an infinite recursion. +Many :meth:`~object.__setattr__` implementations call :meth:`!object.__setattr__` to set +an attribute on self without causing infinite recursion:: + + class X: + def __setattr__(self, name, value): + # Custom logic here... + object.__setattr__(self, name, value) + +Alternatively, it is possible to set attributes by inserting +entries into :attr:`self.__dict__ <object.__dict__>` directly. How do I call a method defined in a base class from a derived class that extends it? @@ -1741,11 +1748,31 @@ but effective way to define class private variables. Any identifier of the form is textually replaced with ``_classname__spam``, where ``classname`` is the current class name with any leading underscores stripped. -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. +The identifier can be used unchanged within the class, but to access it outside +the class, the mangled name must be used: + +.. code-block:: python + class A: + def __one(self): + return 1 + def two(self): + return 2 * self.__one() + + class B(A): + def three(self): + return 3 * self._A__one() + + four = 4 * A()._A__one() + +In particular, this does not guarantee privacy since an outside user can still +deliberately access the private attribute; many Python programmers never bother +to use private variable names at all. + +.. seealso:: + + The :ref:`private name mangling specifications <private-name-mangling>` + for details and special cases. My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1841,15 +1868,15 @@ object identity is assured. Generally, there are three circumstances where identity is guaranteed: 1) Assignments create new names but do not change object identity. After the -assignment ``new = old``, it is guaranteed that ``new is old``. + assignment ``new = old``, it is guaranteed that ``new is old``. 2) Putting an object in a container that stores object references does not -change object identity. After the list assignment ``s[0] = x``, it is -guaranteed that ``s[0] is x``. + change object identity. After the list assignment ``s[0] = x``, it is + guaranteed that ``s[0] is x``. 3) If an object is a singleton, it means that only one instance of that object -can exist. After the assignments ``a = None`` and ``b = None``, it is -guaranteed that ``a is b`` because ``None`` is a singleton. + can exist. After the assignments ``a = None`` and ``b = None``, it is + guaranteed that ``a is b`` because ``None`` is a singleton. In most other circumstances, identity tests are inadvisable and equality tests are preferred. In particular, identity tests should not be used to check @@ -1879,31 +1906,33 @@ In the standard library code, you will see several common patterns for correctly using identity tests: 1) As recommended by :pep:`8`, an identity test is the preferred way to check -for ``None``. This reads like plain English in code and avoids confusion with -other objects that may have boolean values that evaluate to false. + for ``None``. This reads like plain English in code and avoids confusion + with other objects that may have boolean values that evaluate to false. 2) Detecting optional arguments can be tricky when ``None`` is a valid input -value. In those situations, you can create a singleton sentinel object -guaranteed to be distinct from other objects. For example, here is how -to implement a method that behaves like :meth:`dict.pop`:: + value. In those situations, you can create a singleton sentinel object + guaranteed to be distinct from other objects. For example, here is how + to implement a method that behaves like :meth:`dict.pop`: + + .. code-block:: python - _sentinel = object() + _sentinel = object() - def pop(self, key, default=_sentinel): - if key in self: - value = self[key] - del self[key] - return value - if default is _sentinel: - raise KeyError(key) - return default + def pop(self, key, default=_sentinel): + if key in self: + value = self[key] + del self[key] + return value + if default is _sentinel: + raise KeyError(key) + return default 3) Container implementations sometimes need to augment equality tests with -identity tests. This prevents the code from being confused by objects such as -``float('NaN')`` that are not equal to themselves. + identity tests. This prevents the code from being confused by objects + such as ``float('NaN')`` that are not equal to themselves. For example, here is the implementation of -:meth:`collections.abc.Sequence.__contains__`:: +:meth:`!collections.abc.Sequence.__contains__`:: def __contains__(self, value): for v in self: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index a4cd05b0cf019d..c5c7994f1262a9 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -35,6 +36,12 @@ Glossary and loaders (in the :mod:`importlib.abc` module). You can create your own ABCs with the :mod:`abc` module. + annotate function + A function that can be called to retrieve the :term:`annotations <annotation>` + of an object. This function is accessible as the :attr:`~object.__annotate__` + attribute of functions, classes, and modules. Annotate functions are a + subset of :term:`evaluate functions <evaluate function>`. + annotation A label associated with a variable, a class attribute or a function parameter or return value, @@ -42,12 +49,11 @@ Glossary Annotations of local variables cannot be accessed at runtime, but annotations of global variables, class attributes, and functions - are stored in the :attr:`__annotations__` - special attribute of modules, classes, and functions, - respectively. + can be retrieved by calling :func:`annotationlib.get_annotations` + on modules, classes, and functions, respectively. - See :term:`variable annotation`, :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. + See :term:`variable annotation`, :term:`function annotation`, :pep:`484`, + :pep:`526`, and :pep:`649`, which describe this functionality. Also see :ref:`annotations-howto` for best practices on working with annotations. @@ -109,7 +115,7 @@ Glossary :keyword:`yield` expression. Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending + execution state (including local variables and pending try-statements). When the *asynchronous generator iterator* effectively resumes with another awaitable returned by :meth:`~object.__anext__`, it picks up where it left off. See :pep:`492` and :pep:`525`. @@ -126,6 +132,28 @@ Glossary iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + attached thread state + + A :term:`thread state` that is active for the current OS thread. + + When a :term:`thread state` is attached, the OS thread has + access to the full Python C API and can safely invoke the + bytecode interpreter. + + Unless a function explicitly notes otherwise, attempting to call + the C API without an attached thread state will result in a fatal + error or undefined behavior. A thread state can be attached and detached + explicitly by the user through the C API, or implicitly by the runtime, + including during blocking C calls and by the bytecode interpreter in between + calls. + + On most builds of Python, having an attached thread state implies that the + caller holds the :term:`GIL` for the current interpreter, so only + one OS thread can have an attached thread state at a given moment. In + :term:`free-threaded <free threading>` builds of Python, threads can concurrently + hold an attached thread state, allowing for true parallelism of the bytecode + interpreter. + attribute A value associated with an object which is usually referenced by name using dotted expressions. @@ -151,9 +179,9 @@ Glossary A :term:`file object` able to read and write :term:`bytes-like objects <bytes-like object>`. Examples of binary files are files opened in binary mode (``'rb'``, - ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, - :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and - :class:`gzip.GzipFile`. + ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer <sys.stdin>`, + :data:`sys.stdout.buffer <sys.stdout>`, and instances of + :class:`io.BytesIO` and :class:`gzip.GzipFile`. See also :term:`text file` for a file object able to read and write :class:`str` objects. @@ -225,6 +253,28 @@ Glossary A variable defined in a class and intended to be modified only at class level (i.e., not in an instance of the class). + closure variable + A :term:`free variable` referenced from a :term:`nested scope` that is defined in an outer + scope rather than being resolved at runtime from the globals or builtin namespaces. + May be explicitly defined with the :keyword:`nonlocal` keyword to allow write access, + or implicitly defined if the variable is only being read. + + For example, in the ``inner`` function in the following code, both ``x`` and ``print`` are + :term:`free variables <free variable>`, but only ``x`` is a *closure variable*:: + + def outer(): + x = 0 + def inner(): + nonlocal x + x += 1 + print(x) + return inner + + Due to the :attr:`codeobject.co_freevars` attribute (which, despite its name, only + includes the names of closure variables rather than listing all referenced free + variables), the more general :term:`free variable` term is sometimes used even + when the intended meaning is to refer specifically to closure variables. + complex number An extension of the familiar real number system in which all numbers are expressed as a sum of a real part and an imaginary part. Imaginary @@ -237,19 +287,33 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + context + This term has different meanings depending on where and how it is used. + Some common meanings: + + * The temporary state or environment established by a :term:`context + manager` via a :keyword:`with` statement. + * The collection of keyvalue bindings associated with a particular + :class:`contextvars.Context` object and accessed via + :class:`~contextvars.ContextVar` objects. Also see :term:`context + variable`. + * A :class:`contextvars.Context` object. Also see :term:`current + context`. + + context management protocol + The :meth:`~object.__enter__` and :meth:`~object.__exit__` methods called + by the :keyword:`with` statement. See :pep:`343`. + context manager - An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`__enter__` and :meth:`__exit__` methods. - See :pep:`343`. + An object which implements the :term:`context management protocol` and + controls the environment seen in a :keyword:`with` statement. See + :pep:`343`. context variable - A variable which can have different values depending on its context. - This is similar to Thread-Local Storage in which each execution - thread may have a different value for a variable. However, with context - variables, there may be several contexts in one execution thread and the - main usage for context variables is to keep track of variables in + A variable whose value depends on which context is the :term:`current + context`. Values are accessed via :class:`contextvars.ContextVar` + objects. Context variables are primarily used to isolate state between concurrent asynchronous tasks. - See :mod:`contextvars`. contiguous .. index:: C-contiguous, Fortran contiguous @@ -283,6 +347,20 @@ Glossary is used when necessary to distinguish this implementation from others such as Jython or IronPython. + current context + The :term:`context` (:class:`contextvars.Context` object) that is + currently used by :class:`~contextvars.ContextVar` objects to access (get + or set) the values of :term:`context variables <context variable>`. Each + thread has its own current context. Frameworks for executing asynchronous + tasks (see :mod:`asyncio`) associate each task with a context which + becomes the current context whenever the task starts or resumes execution. + + cyclic isolate + A subgroup of one or more objects that reference each other in a reference + cycle, but are not referenced by objects outside the group. The goal of + the :term:`cyclic garbage collector <garbage collection>` is to identify these groups and break the reference + cycles so that the memory can be reclaimed. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -304,8 +382,9 @@ Glossary :ref:`class definitions <class>` for more about decorators. descriptor - Any object which defines the methods :meth:`__get__`, :meth:`__set__`, or - :meth:`__delete__`. When a class attribute is a descriptor, its special + Any object which defines the methods :meth:`~object.__get__`, + :meth:`~object.__set__`, or :meth:`~object.__delete__`. + When a class attribute is a descriptor, its special binding behavior is triggered upon attribute lookup. Normally, using *a.b* to get, set or delete an attribute looks up the object named *b* in the class dictionary for *a*, but if *b* is a descriptor, the respective @@ -319,7 +398,8 @@ Glossary dictionary An associative array, where arbitrary keys are mapped to values. The - keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. + keys can be any object with :meth:`~object.__hash__` and + :meth:`~object.__eq__` methods. Called a hash in Perl. dictionary comprehension @@ -339,7 +419,7 @@ Glossary docstring A string literal which appears as the first expression in a class, function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute + recognized by the compiler and put into the :attr:`~definition.__doc__` attribute of the enclosing class, function or module. Since it is available via introspection, it is the canonical place for documentation of the object. @@ -363,6 +443,11 @@ Glossary statements. The technique contrasts with the :term:`LBYL` style common to many other languages such as C. + evaluate function + A function that can be called to evaluate a lazily evaluated attribute + of an object, such as the value of type aliases created with the :keyword:`type` + statement. + expression A piece of syntax which can be evaluated to some value. In other words, an expression is an accumulation of expression elements like literals, @@ -383,7 +468,7 @@ Glossary file object An object exposing a file-oriented API (with methods such as - :meth:`read()` or :meth:`write()`) to an underlying resource. Depending + :meth:`!read` or :meth:`!write`) to an underlying resource. Depending on the way it was created, a file object can mediate access to a real on-disk file or to another type of storage or communication device (for example standard input/output, in-memory buffers, sockets, pipes, @@ -422,11 +507,11 @@ Glossary An object that tries to find the :term:`loader` for a module that is being imported. - Since Python 3.3, there are two types of finder: :term:`meta path finders + There are two types of finder: :term:`meta path finders <meta path finder>` for use with :data:`sys.meta_path`, and :term:`path entry finders <path entry finder>` for use with :data:`sys.path_hooks`. - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. + See :ref:`finders-and-loaders` and :mod:`importlib` for much more detail. floor division Mathematical division that rounds down to nearest integer. The floor @@ -435,6 +520,19 @@ Glossary division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` rounded *downward*. See :pep:`238`. + free threading + A threading model where multiple threads can run Python bytecode + simultaneously within the same interpreter. This is in contrast to + the :term:`global interpreter lock` which allows only one thread to + execute Python bytecode at a time. See :pep:`703`. + + free variable + Formally, as defined in the :ref:`language execution model <bind_names>`, a free + variable is any variable used in a namespace which is not a local variable in that + namespace. See :term:`closure variable` for an example. + Pragmatically, due to the name of the :attr:`codeobject.co_freevars` attribute, + the term is also sometimes used as a synonym for :term:`closure variable`. + function A series of statements which returns some value to a caller. It can also be passed zero or more :term:`arguments <argument>` which may be used in @@ -494,7 +592,7 @@ Glossary An object created by a :term:`generator` function. Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending + execution state (including local variables and pending try-statements). When the *generator iterator* resumes, it picks up where it left off (in contrast to functions which start fresh on every invocation). @@ -502,7 +600,7 @@ Glossary .. index:: single: generator expression generator expression - An expression that returns an iterator. It looks like a normal expression + An :term:`expression` that returns an :term:`iterator`. It looks like a normal expression followed by a :keyword:`!for` clause defining a loop variable, range, and an optional :keyword:`!if` clause. The combined expression generates values for an enclosing function:: @@ -545,12 +643,16 @@ Glossary tasks such as compression or hashing. Also, the GIL is always released when doing I/O. - Past efforts to create a "free-threaded" interpreter (one which locks - shared data at a much finer granularity) have not been successful - because performance suffered in the common single-processor case. It - is believed that overcoming this performance issue would make the - implementation much more complicated and therefore costlier to maintain. + As of Python 3.13, the GIL can be disabled using the :option:`--disable-gil` + build configuration. After building Python with this option, code must be + run with :option:`-X gil=0 <-X>` or after setting the :envvar:`PYTHON_GIL=0 <PYTHON_GIL>` + environment variable. This feature enables improved performance for + multi-threaded applications and makes it easier to use multi-core CPUs + efficiently. For more details, see :pep:`703`. + In prior versions of Python's C API, a function might declare that it + requires the GIL to be held in order to use it. This refers to having an + :term:`attached thread state`. hash-based pyc A bytecode cache file that uses the hash rather than the last-modified @@ -559,8 +661,9 @@ Glossary hashable An object is *hashable* if it has a hash value which never changes during - its lifetime (it needs a :meth:`__hash__` method), and can be compared to - other objects (it needs an :meth:`__eq__` method). Hashable objects which + its lifetime (it needs a :meth:`~object.__hash__` method), and can be + compared to other objects (it needs an :meth:`~object.__eq__` method). + Hashable objects which compare equal must have the same hash value. Hashability makes an object usable as a dictionary key and a set member, @@ -579,6 +682,17 @@ Glossary :ref:`idle` is a basic editor and interpreter environment which ships with the standard distribution of Python. + immortal + *Immortal objects* are a CPython implementation detail introduced + in :pep:`683`. + + If an object is immortal, its :term:`reference count` is never modified, + and therefore it is never deallocated while the interpreter is running. + For example, :const:`True` and :const:`None` are immortal in CPython. + + Immortal objects can be identified via :func:`sys._is_immortal`, or + via :c:func:`PyUnstable_IsImmortal` in the C API. + immutable An object with a fixed value. Immutable objects include numbers, strings and tuples. Such an object cannot be altered. A new object has to @@ -607,7 +721,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, @@ -636,7 +751,8 @@ Glossary iterables include all sequence types (such as :class:`list`, :class:`str`, and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects <file object>`, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`__getitem__` method + with an :meth:`~object.__iter__` method or with a + :meth:`~object.__getitem__` method that implements :term:`sequence` semantics. Iterables can be @@ -645,7 +761,7 @@ Glossary as an argument to the built-in function :func:`iter`, it returns an iterator for the object. This iterator is good for one pass over the set of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` + :func:`iter` or deal with iterator objects yourself. The :keyword:`for` statement does that automatically for you, creating a temporary unnamed variable to hold the iterator for the duration of the loop. See also :term:`iterator`, :term:`sequence`, and :term:`generator`. @@ -656,8 +772,8 @@ Glossary :func:`next`) return successive items in the stream. When no more data are available a :exc:`StopIteration` exception is raised instead. At this point, the iterator object is exhausted and any further calls to its - :meth:`__next__` method just raise :exc:`StopIteration` again. Iterators - are required to have an :meth:`__iter__` method that returns the iterator + :meth:`!__next__` method just raise :exc:`StopIteration` again. Iterators + are required to have an :meth:`~iterator.__iter__` method that returns the iterator object itself so every iterator is also iterable and may be used in most places where other iterables are accepted. One notable exception is code which attempts multiple iteration passes. A container object (such as a @@ -671,7 +787,10 @@ Glossary .. impl-detail:: CPython does not consistently apply the requirement that an iterator - define :meth:`__iter__`. + define :meth:`~iterator.__iter__`. + And also please note that the free-threading CPython does not guarantee + the thread-safety of iterator operations. + key function A key function or collation function is a callable that returns a value @@ -713,22 +832,14 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. - locale encoding - On Unix, it is the encoding of the LC_CTYPE locale. It can be set with - :func:`locale.setlocale(locale.LC_CTYPE, new_locale) <locale.setlocale>`. - - On Windows, it is the ANSI code page (ex: ``"cp1252"``). - - On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. - - ``locale.getencoding()`` can be used to get the locale encoding. + lexical analyzer - See also the :term:`filesystem encoding and error handler`. + Formal name for the *tokenizer*; see :term:`token`. list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to - elements is O(1). + elements is *O*\ (1). list comprehension A compact way to process all or part of the elements in a sequence and @@ -739,10 +850,27 @@ Glossary processed. loader - An object that loads a module. It must define a method named - :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See :pep:`302` for details and - :class:`importlib.abc.Loader` for an :term:`abstract base class`. + An object that loads a module. + It must define the :meth:`!exec_module` and :meth:`!create_module` methods + to implement the :class:`~importlib.abc.Loader` interface. + A loader is typically returned by a :term:`finder`. + See also: + + * :ref:`finders-and-loaders` + * :class:`importlib.abc.Loader` + * :pep:`302` + + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + :func:`locale.setlocale(locale.LC_CTYPE, new_locale) <locale.setlocale>`. + + On Windows, it is the ANSI code page (ex: ``"cp1252"``). + + On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. + + :func:`locale.getencoding` can be used to get the locale encoding. + + See also the :term:`filesystem encoding and error handler`. magic method .. index:: pair: magic; method @@ -786,8 +914,7 @@ Glossary method resolution order Method Resolution Order is the order in which base classes are searched - for a member during lookup. See `The Python 2.3 Method Resolution Order - <https://www.python.org/download/releases/2.3/mro/>`_ for details of the + for a member during lookup. See :ref:`python_2.3_mro` for details of the algorithm used by the Python interpreter since the 2.3 release. module @@ -801,6 +928,8 @@ Glossary A namespace containing the import-related information used to load a module. An instance of :class:`importlib.machinery.ModuleSpec`. + See also :ref:`module-specs`. + MRO See :term:`method resolution order`. @@ -827,10 +956,11 @@ Glossary Some named tuples are built-in types (such as the above examples). Alternatively, a named tuple can be created from a regular class definition that inherits from :class:`tuple` and that defines named - fields. Such a class can be written by hand or it can be created with - the factory function :func:`collections.namedtuple`. The latter - technique also adds some extra methods that may not be found in - hand-written or built-in named tuples. + fields. Such a class can be written by hand, or it can be created by + inheriting :class:`typing.NamedTuple`, or with the factory function + :func:`collections.namedtuple`. The latter techniques also add some + extra methods that may not be found in hand-written or built-in named + tuples. namespace The place where a variable is stored. Namespaces are implemented as @@ -845,11 +975,16 @@ Glossary modules, respectively. namespace package - A :pep:`420` :term:`package` which serves only as a container for - subpackages. Namespace packages may have no physical representation, + A :term:`package` which serves only as a container for subpackages. + Namespace packages may have no physical representation, and specifically are not like a :term:`regular package` because they have no ``__init__.py`` file. + Namespace packages allow several individually installable packages to have a common parent package. + Otherwise, it is recommended to use a :term:`regular package`. + + For more information, see :pep:`420` and :ref:`reference-namespace-package`. + See also :term:`module`. nested scope @@ -865,13 +1000,23 @@ Glossary Old name for the flavor of classes now used for all class objects. In earlier Python versions, only new-style classes could use Python's newer, versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. + properties, :meth:`~object.__getattribute__`, class methods, and static + methods. object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style class`. + optimized scope + A scope where target local variable names are reliably known to the + compiler when the code is compiled, allowing optimization of read and + write access to these names. The local namespaces for functions, + generators, coroutines, comprehensions, and generator expressions are + optimized in this fashion. Note: most interpreter optimizations are + applied to all scopes, only those relying on a known set of local + and nonlocal variable names are restricted to optimized scopes. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -945,7 +1090,7 @@ Glossary finders implement. path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path + A callable on the :data:`sys.path_hooks` list which returns a :term:`path entry finder` if it knows how to find modules on a specific :term:`path entry`. @@ -1056,7 +1201,7 @@ Glossary reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are - "immortal" and have reference counts that are never modified, and + :term:`immortal` and have reference counts that are never modified, and therefore the objects are never deallocated. Reference counting is generally not visible to Python code, but it is a key element of the :term:`CPython` implementation. Programmers can call the @@ -1069,6 +1214,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though @@ -1078,21 +1227,23 @@ Glossary sequence An :term:`iterable` which supports efficient element access using integer - indices via the :meth:`__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. + indices via the :meth:`~object.__getitem__` special method and defines a + :meth:`~object.__len__` method that returns the length of the sequence. Some built-in sequence types are :class:`list`, :class:`str`, :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`__getitem__` and :meth:`__len__`, but is considered a + supports :meth:`~object.__getitem__` and :meth:`!__len__`, but is considered a mapping rather than a sequence because the lookups use arbitrary - :term:`immutable` keys rather than integers. + :term:`hashable` keys rather than integers. The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just - :meth:`__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded + :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding + :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and + :meth:`~object.__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.ABCMeta.register`. + :func:`~abc.ABCMeta.register`. For more documentation on sequence + methods generally, see + :ref:`Common Sequence Operations <typesseq-common>`. set comprehension A compact way to process all or part of the elements in an iterable and @@ -1111,16 +1262,12 @@ Glossary (subscript) notation uses :class:`slice` objects internally. soft deprecated - A soft deprecation can be used when using an API which should no longer - be used to write new code, but it remains safe to continue using it in - existing code. The API remains documented and tested, but will not be - developed further (no enhancement). + A soft deprecated API should not be used in new code, + but it is safe for already existing code to use it. + The API remains documented and tested, but will not be enhanced further. - The main difference between a "soft" and a (regular) "hard" deprecation - is that the soft deprecation does not imply scheduling the removal of the - deprecated API. - - Another difference is that a soft deprecation does not issue a warning. + Soft deprecation, unlike normal deprecation, does not plan on removing the API + and will not emit warnings. See `PEP 387: Soft Deprecation <https://peps.python.org/pep-0387/#soft-deprecation>`_. @@ -1138,6 +1285,11 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. + static type checker + An external tool that reads Python code and analyzes it, looking for + issues such as incorrect types. See also :term:`type hints <type hint>` + and the :mod:`typing` module. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1175,6 +1327,40 @@ Glossary See also :term:`binary file` for a file object able to read and write :term:`bytes-like objects <bytes-like object>`. + thread state + + The information used by the :term:`CPython` runtime to run in an OS thread. + For example, this includes the current exception, if any, and the + state of the bytecode interpreter. + + Each thread state is bound to a single OS thread, but threads may have + many thread states available. At most, one of them may be + :term:`attached <attached thread state>` at once. + + An :term:`attached thread state` is required to call most + of Python's C API, unless a function explicitly documents otherwise. + The bytecode interpreter only runs under an attached thread state. + + Each thread state belongs to a single interpreter, but each interpreter + may have many thread states, including multiple for the same OS thread. + Thread states from multiple interpreters may be bound to the same + thread, but only one can be :term:`attached <attached thread state>` in + that thread at any given moment. + + See :ref:`Thread State and the Global Interpreter Lock <threads>` for more + information. + + token + + A small unit of source code, generated by the + :ref:`lexical analyzer <lexical>` (also called the *tokenizer*). + Names, numbers, strings, operators, + newlines and similar are represented by tokens. + + The :mod:`tokenize` module exposes Python's lexical analyzer. + The :mod:`token` module contains information on the various types + of tokens. + triple-quoted string A string which is bound by three instances of either a quotation mark (") or an apostrophe ('). While they don't provide any functionality @@ -1187,7 +1373,7 @@ Glossary type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with + :attr:`~object.__class__` attribute or can be retrieved with ``type(obj)``. type alias @@ -1214,8 +1400,8 @@ Glossary attribute, or a function parameter or return value. Type hints are optional and are not enforced by Python but - they are useful to static type analysis tools, and aid IDEs with code - completion and refactoring. + they are useful to :term:`static type checkers <static type checker>`. + They can also aid IDEs with code completion and refactoring. Type hints of global variables, class attributes, and functions, but not local variables, can be accessed using diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 1134686c947d66..d7deb6c6bc1768 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -34,11 +34,16 @@ Accessing The Annotations Dict Of An Object In Python 3.10 And Newer Python 3.10 adds a new function to the standard library: :func:`inspect.get_annotations`. In Python versions 3.10 -and newer, calling this function is the best practice for +through 3.13, calling this function is the best practice for accessing the annotations dict of any object that supports annotations. This function can also "un-stringize" stringized annotations for you. +In Python 3.14, there is a new :mod:`annotationlib` module +with functionality for working with annotations. This +includes a :func:`annotationlib.get_annotations` function, +which supersedes :func:`inspect.get_annotations`. + If for some reason :func:`inspect.get_annotations` isn't viable for your use case, you may access the ``__annotations__`` data member manually. Best practice @@ -102,9 +107,9 @@ Your code will have to have a separate code path if the object you're examining is a class (``isinstance(o, type)``). In that case, best practice relies on an implementation detail of Python 3.9 and before: if a class has annotations defined, -they are stored in the class's ``__dict__`` dictionary. Since +they are stored in the class's :attr:`~type.__dict__` dictionary. Since the class may or may not have annotations defined, best practice -is to call the ``get`` method on the class dict. +is to call the :meth:`~dict.get` method on the class dict. To put it all together, here is some sample code that safely accesses the ``__annotations__`` attribute on an arbitrary @@ -121,8 +126,8 @@ the type of ``ann`` using :func:`isinstance` before further examination. Note that some exotic or malformed type objects may not have -a ``__dict__`` attribute, so for extra safety you may also wish -to use :func:`getattr` to access ``__dict__``. +a :attr:`~type.__dict__` attribute, so for extra safety you may also wish +to use :func:`getattr` to access :attr:`!__dict__`. Manually Un-Stringizing Stringized Annotations @@ -153,7 +158,8 @@ on an arbitrary object ``o``: unwrap it by accessing either ``o.__wrapped__`` or ``o.func`` as appropriate, until you have found the root unwrapped function. * If ``o`` is a callable (but not a class), use - ``o.__globals__`` as the globals when calling :func:`eval`. + :attr:`o.__globals__ <function.__globals__>` as the globals when calling + :func:`eval`. However, not all string values used as annotations can be successfully turned into Python values by :func:`eval`. @@ -183,7 +189,11 @@ Best Practices For ``__annotations__`` In Any Python Version * If you do assign directly to the ``__annotations__`` member of an object, you should always set it to a ``dict`` object. -* If you directly access the ``__annotations__`` member +* You should avoid accessing ``__annotations__`` directly on any object. + Instead, use :func:`annotationlib.get_annotations` (Python 3.14+) + or :func:`inspect.get_annotations` (Python 3.10+). + +* If you do directly access the ``__annotations__`` member of an object, you should ensure that it's a dictionary before attempting to examine its contents. @@ -230,3 +240,17 @@ itself be quoted. In effect the annotation is quoted This prints ``{'a': "'str'"}``. This shouldn't really be considered a "quirk"; it's mentioned here simply because it might be surprising. + +If you use a class with a custom metaclass and access ``__annotations__`` +on the class, you may observe unexpected behavior; see +:pep:`749 <749#pep749-metaclasses>` for some examples. You can avoid these +quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or +:func:`inspect.get_annotations` on Python 3.10+. On earlier versions of +Python, you can avoid these bugs by accessing the annotations from the +class's :attr:`~type.__dict__` +(for example, ``cls.__dict__.get('__annotations__', None)``). + +In some versions of Python, instances of classes may have an ``__annotations__`` +attribute. However, this is not supported functionality. If you need the +annotations of an instance, you can use :func:`type` to access its class +(for example, ``annotationlib.get_annotations(type(myinstance))`` on Python 3.14+). diff --git a/Doc/howto/argparse-optparse.rst b/Doc/howto/argparse-optparse.rst new file mode 100644 index 00000000000000..b684619885b4c7 --- /dev/null +++ b/Doc/howto/argparse-optparse.rst @@ -0,0 +1,65 @@ +.. currentmodule:: argparse + +.. _upgrading-optparse-code: +.. _migrating-optparse-code: + +============================================ +Migrating ``optparse`` code to ``argparse`` +============================================ + +The :mod:`argparse` module offers several higher level features not natively +provided by the :mod:`optparse` module, including: + +* Handling positional arguments. +* Supporting subcommands. +* Allowing alternative option prefixes like ``+`` and ``/``. +* Handling zero-or-more and one-or-more style arguments. +* Producing more informative usage messages. +* Providing a much simpler interface for custom ``type`` and ``action``. + +Originally, the :mod:`argparse` module attempted to maintain compatibility +with :mod:`optparse`. However, the fundamental design differences between +supporting declarative command line option processing (while leaving positional +argument processing to application code), and supporting both named options +and positional arguments in the declarative