From 2ba1aed596ff9e7c5e193cf990f1f20f08bbf116 Mon Sep 17 00:00:00 2001 From: Alex Turner Date: Mon, 29 Apr 2024 21:26:26 +0100 Subject: [PATCH 001/220] gh-117657: TSAN fix race on `gstate->young.count` (#118313) --- Python/gc_free_threading.c | 25 +++++++++++----------- Tools/tsan/suppressions_free_threading.txt | 1 - 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 8c0940d8f066a7a..ef6aaad12523116 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -1044,9 +1044,20 @@ record_deallocation(PyThreadState *tstate) } static void -gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) +gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, int generation) { _PyEval_StopTheWorld(interp); + + // update collection and allocation counters + if (generation+1 < NUM_GENERATIONS) { + state->gcstate->old[generation].count += 1; + } + + state->gcstate->young.count = 0; + for (int i = 1; i <= generation; ++i) { + state->gcstate->old[i-1].count = 0; + } + // merge refcounts for all queued objects merge_all_queued_objects(interp, state); process_delayed_frees(interp); @@ -1115,7 +1126,6 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state) static Py_ssize_t gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - int i; Py_ssize_t m = 0; /* # objects collected */ Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ @@ -1161,15 +1171,6 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) PyDTrace_GC_START(generation); } - /* update collection and allocation counters */ - if (generation+1 < NUM_GENERATIONS) { - gcstate->old[generation].count += 1; - } - gcstate->young.count = 0; - for (i = 1; i <= generation; i++) { - gcstate->old[i-1].count = 0; - } - PyInterpreterState *interp = tstate->interp; struct collection_state state = { @@ -1177,7 +1178,7 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) .gcstate = gcstate, }; - gc_collect_internal(interp, &state); + gc_collect_internal(interp, &state, generation); m = state.collected; n = state.uncollectable; diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 4b1a2fdf6dd43aa..4f6648a75731840 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -27,7 +27,6 @@ race:_PyObject_GC_TRACK race:_PyParkingLot_Park race:_PyType_HasFeature race:assign_version_tag -race:gc_collect_main race:gc_restore_tid race:initialize_new_array race:insertdict From 96d8ca7ad6c4f23f023dd6ed0abd042b29dff4a2 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 29 Apr 2024 21:46:25 +0100 Subject: [PATCH 002/220] gh-118347: Fix Windows installer not updating launcher (GH-118386) --- .../2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst | 1 + .../bundle/bootstrap/PythonBootstrapperApplication.cpp | 8 ++++---- Tools/msi/bundle/packagegroups/launcher.wxs | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst diff --git a/Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst b/Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst new file mode 100644 index 000000000000000..8f02ed94f100fe8 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-04-29-13-53-25.gh-issue-118347.U5ZRm_.rst @@ -0,0 +1 @@ +Fixes launcher updates not being installed. diff --git a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp index e0e179e3aede6d9..7cddda9b06555d8 100644 --- a/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp +++ b/Tools/msi/bundle/bootstrap/PythonBootstrapperApplication.cpp @@ -464,11 +464,11 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { LOC_STRING *pLocString = nullptr; LPCWSTR locKey = L"#(loc.Include_launcherHelp)"; - LONGLONG detectedLauncher; + LONGLONG blockedLauncher; - if (SUCCEEDED(BalGetNumericVariable(L"DetectedLauncher", &detectedLauncher)) && detectedLauncher) { + if (SUCCEEDED(BalGetNumericVariable(L"BlockedLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherRemove)"; - } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &detectedLauncher)) && detectedLauncher) { + } else if (SUCCEEDED(BalGetNumericVariable(L"DetectedOldLauncher", &blockedLauncher)) && blockedLauncher) { locKey = L"#(loc.Include_launcherUpgrade)"; } @@ -2671,7 +2671,7 @@ class PythonBootstrapperApplication : public CBalBaseBootstrapperApplication { /*Elevate when installing for all users*/ L"InstallAllUsers or " /*Elevate when installing the launcher for all users and it was not detected*/ - L"(Include_launcher and InstallLauncherAllUsers and not DetectedLauncher)" + L"(Include_launcher and InstallLauncherAllUsers and not BlockedLauncher)" L")", L"" }; diff --git a/Tools/msi/bundle/packagegroups/launcher.wxs b/Tools/msi/bundle/packagegroups/launcher.wxs index a6922758f31f14f..080598a0a486ef4 100644 --- a/Tools/msi/bundle/packagegroups/launcher.wxs +++ b/Tools/msi/bundle/packagegroups/launcher.wxs @@ -11,7 +11,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="(InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> @@ -25,7 +25,7 @@ EnableFeatureSelection="yes" Permanent="yes" Visible="yes" - InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not DetectedLauncher"> + InstallCondition="not (InstallAllUsers or InstallLauncherAllUsers) and Include_launcher and not BlockedLauncher"> From 8e4fb5d260e529c9d4ca60980225fbd00dd5c3c8 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 29 Apr 2024 13:57:49 -0700 Subject: [PATCH 003/220] gh-118359: Improve docs for Bdb.user_call (#118368) The `argument_list` parameter of bdb.Bdb.user_call has been useless for 25 years. It is retained for backwards compatibility, but it will always be None. --- Doc/library/bdb.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 7bf4308a96d0f55..b050560cd9038cb 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -240,6 +240,9 @@ The :mod:`bdb` module also defines two classes: Called from :meth:`dispatch_call` if a break might stop inside the called function. + *argument_list* is not used anymore and will always be ``None``. + The argument is kept for backwards compatibility. + .. method:: user_line(frame) Called from :meth:`dispatch_line` when either :meth:`stop_here` or From 8b56d82c59c2983b4292a7f506982f2cab352bb2 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 29 Apr 2024 14:09:16 -0700 Subject: [PATCH 004/220] GH-118306: Update JIT to use LLVM 18 (GH-118307) --- .github/workflows/jit.yml | 11 ++-- ...-04-26-05-38-18.gh-issue-118306.vRUEOU.rst | 1 + Python/jit.c | 3 ++ Tools/jit/README.md | 10 ++-- Tools/jit/_llvm.py | 2 +- Tools/jit/_schema.py | 9 ++-- Tools/jit/_stencils.py | 4 +- Tools/jit/_targets.py | 52 +++++++++---------- 8 files changed, 55 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b37eb7279559099..b969e6d13e61a3e 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -41,7 +41,7 @@ jobs: - true - false llvm: - - 16 + - 18 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 @@ -94,6 +94,7 @@ jobs: - name: Native Windows if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -102,27 +103,31 @@ jobs: - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | + choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - name: Native macOS if: runner.os == 'macOS' run: | + brew update brew install llvm@${{ matrix.llvm }} SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux if: runner.os == 'Linux' && matrix.architecture == 'x86_64' run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }} make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' run: | @@ -139,6 +144,6 @@ jobs: CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ CPP="$CC --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst new file mode 100644 index 000000000000000..051295541ab7e2f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-05-38-18.gh-issue-118306.vRUEOU.rst @@ -0,0 +1 @@ +Update JIT compilation to use LLVM 18 diff --git a/Python/jit.c b/Python/jit.c index 8782adb847cfd60..75ec4fb9756eb73 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -234,6 +234,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) assert((int64_t)value < (1LL << 31)); *loc32 = (uint32_t)value; continue; + case HoleKind_ARM64_RELOC_BRANCH26: case HoleKind_IMAGE_REL_ARM64_BRANCH26: case HoleKind_R_AARCH64_CALL26: case HoleKind_R_AARCH64_JUMP26: @@ -278,6 +279,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: case HoleKind_R_AARCH64_ADR_GOT_PAGE: + case HoleKind_R_AARCH64_ADR_PREL_PG_HI21: // 21-bit count of pages between this page and an absolute address's // page... I know, I know, it's weird. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). @@ -341,6 +343,7 @@ patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) case HoleKind_ARM64_RELOC_PAGEOFF12: case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: + case HoleKind_R_AARCH64_ADD_ABS_LO12_NC: case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: // 12-bit low part of an absolute address. Pairs nicely with // ARM64_RELOC_GOT_LOAD_PAGE21 (above). diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 04a6c0780bf9727..7b33f99d23f75d0 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -23,17 +23,21 @@ sudo ./llvm.sh 16 ### macOS -Install LLVM 16 with [Homebrew](https://brew.sh): +Install LLVM 18 with [Homebrew](https://brew.sh): ```sh -brew install llvm@16 +brew install llvm@18 ``` Homebrew won't add any of the tools to your `$PATH`. That's okay; the build script knows how to find them. ### Windows -Install LLVM 16 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=16), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** +Install LLVM 18 [by searching for it on LLVM's GitHub releases page](https://github.com/llvm/llvm-project/releases?q=18), clicking on "Assets", downloading the appropriate Windows installer for your platform (likely the file ending with `-win64.exe`), and running it. **When installing, be sure to select the option labeled "Add LLVM to the system PATH".** + +### Dev Containers + +If you are working CPython in a [Codespaces instance](https://devguide.python.org/getting-started/setup-building/#using-codespaces), there's no need to install LLVM as the Fedora 40 base image includes LLVM 18 out of the box. ## Building diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 603bbef59ba2e63..74a048ccbfcc586 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -7,7 +7,7 @@ import subprocess import typing -_LLVM_VERSION = 16 +_LLVM_VERSION = 18 _LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\s+") _P = typing.ParamSpec("_P") diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 045fd502a03c12b..6aef887e12475b5 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -2,6 +2,7 @@ import typing HoleKind: typing.TypeAlias = typing.Literal[ + "ARM64_RELOC_BRANCH26", "ARM64_RELOC_GOT_LOAD_PAGE21", "ARM64_RELOC_GOT_LOAD_PAGEOFF12", "ARM64_RELOC_PAGE21", @@ -16,8 +17,10 @@ "IMAGE_REL_I386_REL32", "R_AARCH64_ABS64", "R_AARCH64_ADR_GOT_PAGE", + "R_AARCH64_ADR_PREL_PG_HI21", "R_AARCH64_CALL26", "R_AARCH64_JUMP26", + "R_AARCH64_ADD_ABS_LO12_NC", "R_AARCH64_LD64_GOT_LO12_NC", "R_AARCH64_MOVW_UABS_G0_NC", "R_AARCH64_MOVW_UABS_G1_NC", @@ -68,12 +71,12 @@ class _COFFSymbol(typing.TypedDict): class _ELFSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int class _MachOSymbol(typing.TypedDict): - Name: dict[typing.Literal["Value"], str] + Name: dict[typing.Literal["Name"], str] Value: int @@ -99,7 +102,7 @@ class ELFSection(typing.TypedDict): Relocations: list[dict[typing.Literal["Relocation"], ELFRelocation]] SectionData: dict[typing.Literal["Bytes"], list[int]] Symbols: list[dict[typing.Literal["Symbol"], _ELFSymbol]] - Type: dict[typing.Literal["Value"], str] + Type: dict[typing.Literal["Name"], str] class MachOSection(typing.TypedDict): diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 243bb3dd134f704..f8ecffcf3ddda25 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -203,7 +203,9 @@ def process_relocations(self, *, alignment: int = 1) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( - hole.kind in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26"} + hole.kind in { + "R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26" + } and hole.value is HoleValue.ZERO ): self.code.pad(alignment) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 66db358679239e8..91734b36b4ab1bc 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -221,7 +221,7 @@ def _handle_relocation( case { "Offset": offset, "Symbol": s, - "Type": {"Value": "IMAGE_REL_I386_DIR32" as kind}, + "Type": {"Name": "IMAGE_REL_I386_DIR32" as kind}, }: offset += base value, symbol = self._unwrap_dllimport(s) @@ -230,7 +230,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind + "Name": "IMAGE_REL_AMD64_REL32" | "IMAGE_REL_I386_REL32" as kind }, }: offset += base @@ -242,7 +242,7 @@ def _handle_relocation( "Offset": offset, "Symbol": s, "Type": { - "Value": "IMAGE_REL_ARM64_BRANCH26" + "Name": "IMAGE_REL_ARM64_BRANCH26" | "IMAGE_REL_ARM64_PAGEBASE_REL21" | "IMAGE_REL_ARM64_PAGEOFFSET_12A" | "IMAGE_REL_ARM64_PAGEOFFSET_12L" as kind @@ -262,7 +262,7 @@ class _ELF( def _handle_section( self, section: _schema.ELFSection, group: _stencils.StencilGroup ) -> None: - section_type = section["Type"]["Value"] + section_type = section["Type"]["Name"] flags = {flag["Name"] for flag in section["Flags"]["Flags"]} if section_type == "SHT_RELA": assert "SHF_INFO_LINK" in flags, flags @@ -290,7 +290,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = len(stencil.body) + symbol["Value"] - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset stencil.body.extend(section["SectionData"]["Bytes"]) @@ -312,9 +312,9 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "R_AARCH64_ADR_GOT_PAGE" + "Name": "R_AARCH64_ADR_GOT_PAGE" | "R_AARCH64_LD64_GOT_LO12_NC" | "R_X86_64_GOTPCREL" | "R_X86_64_GOTPCRELX" @@ -327,8 +327,8 @@ def _handle_relocation( case { "Addend": addend, "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -371,7 +371,7 @@ def _handle_section( for wrapped_symbol in section["Symbols"]: symbol = wrapped_symbol["Symbol"] offset = symbol["Value"] - start_address - name = symbol["Name"]["Value"] + name = symbol["Name"]["Name"] name = name.removeprefix(self.prefix) group.symbols[name] = value, offset assert "Relocations" in section @@ -387,9 +387,9 @@ def _handle_relocation( match relocation: case { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "ARM64_RELOC_GOT_LOAD_PAGE21" + "Name": "ARM64_RELOC_GOT_LOAD_PAGE21" | "ARM64_RELOC_GOT_LOAD_PAGEOFF12" as kind }, }: @@ -399,8 +399,8 @@ def _handle_relocation( addend = 0 case { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, + "Symbol": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_GOT" | "X86_64_RELOC_GOT_LOAD" as kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -410,13 +410,13 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": "X86_64_RELOC_SIGNED" as kind}, + "Section": {"Name": s}, + "Type": {"Name": "X86_64_RELOC_SIGNED" as kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, + "Symbol": {"Name": s}, "Type": { - "Value": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind + "Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind }, }: offset += base @@ -427,12 +427,12 @@ def _handle_relocation( ) case { "Offset": offset, - "Section": {"Value": s}, - "Type": {"Value": kind}, + "Section": {"Name": s}, + "Type": {"Name": kind}, } | { "Offset": offset, - "Symbol": {"Value": s}, - "Type": {"Value": kind}, + "Symbol": {"Name": s}, + "Type": {"Name": kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -446,13 +446,12 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" if re.fullmatch(r"aarch64-apple-darwin.*", host): - args = ["-mcmodel=large"] - return _MachO(host, alignment=8, args=args, prefix="_") + return _MachO(host, alignment=8, prefix="_") if re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] return _COFF(host, alignment=8, args=args) if re.fullmatch(r"aarch64-.*-linux-gnu", host): - args = ["-mcmodel=large"] + args = ["-fpic"] return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] @@ -463,5 +462,6 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: args = ["-fms-runtime-lib=dll"] return _COFF(host, args=args) if re.fullmatch(r"x86_64-.*-linux-gnu", host): - return _ELF(host) + args = ["-fpic"] + return _ELF(host, args=args) raise ValueError(host) From 11cbf77f9799e86603bca927a98959c7b94fff80 Mon Sep 17 00:00:00 2001 From: Donghee Na Date: Tue, 30 Apr 2024 13:42:13 +0900 Subject: [PATCH 005/220] gh-118392: Add note about random.random for multi thread app (gh-118396) --- Doc/library/random.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 8fbce18c56f17c9..61798263d61195c 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -59,6 +59,12 @@ from sources provided by the operating system. random number generator with a long period and comparatively simple update operations. +.. note:: + The global random number generator and instances of :class:`Random` are thread-safe. + However, in the free-threaded build, concurrent calls to the global generator or + to the same instance of :class:`Random` may encounter contention and poor performance. + Consider using separate instances of :class:`Random` per thread instead. + Bookkeeping functions --------------------- From 0f797402bc77192c76a952410ca8e17359feab3c Mon Sep 17 00:00:00 2001 From: edson duarte Date: Tue, 30 Apr 2024 05:34:15 -0300 Subject: [PATCH 006/220] gh-85453: Consistent backquotes on None occurences across datetime.rst (#118282) --- Doc/library/datetime.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index b1b5e04c74b0666..f40c6cea83820c4 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1735,7 +1735,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. * ``fold in [0, 1]``. If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to 0 except *tzinfo*, which defaults to :const:`None`. + default to 0 except *tzinfo*, which defaults to ``None``. Class attributes: From 5b05d452cd20d9f0cfecdeec90adad3af5e4dfff Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 30 Apr 2024 11:33:13 +0100 Subject: [PATCH 007/220] GH-118095: Add tier 2 support for YIELD_VALUE (GH-118380) --- Include/internal/pycore_opcode_metadata.h | 1 + Include/internal/pycore_uop_ids.h | 1 + Include/internal/pycore_uop_metadata.h | 4 +++ Python/bytecodes.c | 16 ++++++--- Python/executor_cases.c.h | 41 +++++++++++++++++++++++ Python/generated_cases.c.h | 18 +++++++--- Python/optimizer.c | 4 +-- Python/optimizer_bytecodes.c | 9 +++++ Python/optimizer_cases.c.h | 14 +++++--- 9 files changed, 92 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 4b1f43cf2af06e7..2ccc548ca6c5fd9 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1347,6 +1347,7 @@ _PyOpcode_macro_expansion[256] = { [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { _UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [WITH_EXCEPT_START] = { .nuops = 1, .uops = { { _WITH_EXCEPT_START, 0, 0 } } }, + [YIELD_VALUE] = { .nuops = 1, .uops = { { _YIELD_VALUE, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 030321ef4fcb23a..0a2231e88488c90 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -282,6 +282,7 @@ extern "C" { #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START +#define _YIELD_VALUE YIELD_VALUE #define MAX_UOP_ID 447 #ifdef __cplusplus diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 59e690f3aace356..a84212c1ec0b69e 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -91,6 +91,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_AITER] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_ASSERTION_ERROR] = 0, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -500,6 +501,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE_TUPLE] = "_UNPACK_SEQUENCE_TUPLE", [_UNPACK_SEQUENCE_TWO_TUPLE] = "_UNPACK_SEQUENCE_TWO_TUPLE", [_WITH_EXCEPT_START] = "_WITH_EXCEPT_START", + [_YIELD_VALUE] = "_YIELD_VALUE", }; int _PyUop_num_popped(int opcode, int oparg) { @@ -648,6 +650,8 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _GET_AWAITABLE: return 1; + case _YIELD_VALUE: + return 1; case _POP_EXCEPT: return 1; case _LOAD_ASSERTION_ERROR: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f688856d6909caf..eee8b328a65b8c8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1089,32 +1089,38 @@ dummy_func( goto resume_frame; } - tier1 inst(YIELD_VALUE, (retval -- unused)) { + inst(YIELD_VALUE, (retval -- value)) { // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + #if TIER_ONE assert(frame != &entry_frame); - frame->instr_ptr = next_instr; + #endif + frame->instr_ptr++; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); } inst(POP_EXCEPT, (exc_value -- )) { diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 2d9acfeea432bcc..b17f3762714c72c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1110,6 +1110,47 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 because it is instrumented */ + case _YIELD_VALUE: { + PyObject *retval; + PyObject *value; + oparg = CURRENT_OPARG(); + retval = stack_pointer[-1]; + // NOTE: It's important that YIELD_VALUE never raises an exception! + // The compiler treats any exception raised here as a failed close() + // or throw() call. + #if TIER_ONE + assert(frame != &entry_frame); + #endif + frame->instr_ptr++; + PyGenObject *gen = _PyFrame_GetGenerator(frame); + assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); + assert(oparg == 0 || oparg == 1); + gen->gi_frame_state = FRAME_SUSPENDED + oparg; + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + tstate->exc_info = gen->gi_exc_state.previous_item; + gen->gi_exc_state.previous_item = NULL; + _Py_LeaveRecursiveCallPy(tstate); + _PyInterpreterFrame *gen_frame = frame; + frame = tstate->current_frame = frame->previous; + gen_frame->previous = NULL; + /* We don't know which of these is relevant here, so keep them equal */ + assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE + assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif + LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + break; + } + case _POP_EXCEPT: { PyObject *exc_value; exc_value = stack_pointer[-1]; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c27505fde3d9fa6..7c1cc147de4e1a6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6012,31 +6012,41 @@ next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); PyObject *retval; + PyObject *value; retval = stack_pointer[-1]; // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. + #if TIER_ONE assert(frame != &entry_frame); - frame->instr_ptr = next_instr; + #endif + frame->instr_ptr++; PyGenObject *gen = _PyFrame_GetGenerator(frame); assert(FRAME_SUSPENDED_YIELD_FROM == FRAME_SUSPENDED + 1); assert(oparg == 0 || oparg == 1); gen->gi_frame_state = FRAME_SUSPENDED + oparg; - _PyFrame_SetStackPointer(frame, stack_pointer - 1); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; _Py_LeaveRecursiveCallPy(tstate); _PyInterpreterFrame *gen_frame = frame; frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; - _PyFrame_StackPush(frame, retval); /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); + #if TIER_ONE assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #endif LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); - goto resume_frame; + LOAD_SP(); + value = retval; + LLTRACE_RESUME_FRAME(); + stack_pointer[0] = value; + stack_pointer += 1; + DISPATCH(); } #undef TIER_ONE diff --git a/Python/optimizer.c b/Python/optimizer.c index fcd7d18f2c2e228..a9a35fc902018ab 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -704,7 +704,7 @@ translate_bytecode_to_trace( int nuops = expansion->nuops; RESERVE(nuops + 1); /* One extra for exit */ int16_t last_op = expansion->uops[nuops-1].uop; - if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR) { + if (last_op == _POP_FRAME || last_op == _RETURN_GENERATOR || last_op == _YIELD_VALUE) { // Check for trace stack underflow now: // We can't bail e.g. in the middle of // LOAD_CONST + _POP_FRAME. @@ -763,7 +763,7 @@ translate_bytecode_to_trace( Py_FatalError("garbled expansion"); } - if (uop == _POP_FRAME || uop == _RETURN_GENERATOR) { + if (uop == _POP_FRAME || uop == _RETURN_GENERATOR || uop == _YIELD_VALUE) { TRACE_STACK_POP(); /* Set the operand to the function or code object returned to, * to assist optimization passes. (See _PUSH_FRAME below.) diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 8bc563427747905..60763286178c71d 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -673,6 +673,15 @@ dummy_func(void) { } } + op(_YIELD_VALUE, (unused -- res)) { + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + } + + op(_FOR_ITER_GEN_FRAME, ( -- )) { + /* We are about to hit the end of the trace */ + goto done; + } + op(_CHECK_STACK_SPACE, ( --)) { assert(corresponding_check_stack == NULL); corresponding_check_stack = this_instr; diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4102d00171fbaff..0b696921ebfc9f4 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -684,6 +684,13 @@ /* _INSTRUMENTED_YIELD_VALUE is not a viable micro-op for tier 2 */ + case _YIELD_VALUE: { + _Py_UopsSymbol *res; + OUT_OF_SPACE_IF_NULL(res = sym_new_unknown(ctx)); + stack_pointer[-1] = res; + break; + } + case _POP_EXCEPT: { stack_pointer += -1; break; @@ -1440,11 +1447,8 @@ } case _FOR_ITER_GEN_FRAME: { - _PyInterpreterFrame *gen_frame; - gen_frame = sym_new_not_null(ctx); - if (gen_frame == NULL) goto out_of_space; - stack_pointer[0] = (_Py_UopsSymbol *)gen_frame; - stack_pointer += 1; + /* We are about to hit the end of the trace */ + goto done; break; } From 11f8348d78c22f85694d7a424541b34d6054a8ee Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 30 Apr 2024 15:04:16 +0300 Subject: [PATCH 008/220] gh-118404: Fix inspect.signature() for non-comparable callables (GH-118405) --- Lib/inspect.py | 6 ++++-- Lib/test/test_inspect/test_inspect.py | 10 ++++++++++ .../2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst | 1 + 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 1f4216f0389d28f..d46514f4b104677 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2179,8 +2179,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 6b577090bdff687..fbef34eddacb3a4 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4690,6 +4690,16 @@ class D2(D1): self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + class TestParameterObject(unittest.TestCase): def test_signature_parameter_kinds(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst b/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst new file mode 100644 index 000000000000000..b8f9ee061ac1646 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-29-22-11-54.gh-issue-118404.GYfMaD.rst @@ -0,0 +1 @@ +Fix :func:`inspect.signature` for non-comparable callables. From 3b268f4edc02b22257d745363b5cae199b6e5720 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Tue, 30 Apr 2024 15:00:31 +0100 Subject: [PATCH 009/220] gh-116622: Redirect stdout and stderr to system log when embedded in an Android app (#118063) --- Lib/_android_support.py | 94 +++++ Lib/test/test_android.py | 332 ++++++++++++++++++ ...-04-17-22-49-15.gh-issue-116622.tthNUF.rst | 1 + Python/pylifecycle.c | 77 ++++ Python/stdlib_module_names.h | 1 + configure | 3 + configure.ac | 3 + 7 files changed, 511 insertions(+) create mode 100644 Lib/_android_support.py create mode 100644 Lib/test/test_android.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst diff --git a/Lib/_android_support.py b/Lib/_android_support.py new file mode 100644 index 000000000000000..590e85ea8c2db11 --- /dev/null +++ b/Lib/_android_support.py @@ -0,0 +1,94 @@ +import io +import sys + + +# The maximum length of a log message in bytes, including the level marker and +# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +# platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +# longer than this will be be truncated by logcat. This limit has already been +# reduced at least once in the history of Android (from 4076 to 4068 between API +# level 23 and 26), so leave some headroom. +MAX_BYTES_PER_WRITE = 4000 + +# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this +# size ensures that TextIOWrapper can always avoid exceeding MAX_BYTES_PER_WRITE. +# However, if the actual number of bytes per character is smaller than that, +# then TextIOWrapper may still join multiple consecutive text writes into binary +# writes containing a larger number of characters. +MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4 + + +# When embedded in an app on current versions of Android, there's no easy way to +# monitor the C-level stdout and stderr. The testbed comes with a .c file to +# redirect them to the system log using a pipe, but that wouldn't be convenient +# or appropriate for all apps. So we redirect at the Python level instead. +def init_streams(android_log_write, stdout_prio, stderr_prio): + if sys.executable: + return # Not embedded in an app. + + sys.stdout = TextLogStream( + android_log_write, stdout_prio, "python.stdout", errors=sys.stdout.errors) + sys.stderr = TextLogStream( + android_log_write, stderr_prio, "python.stderr", errors=sys.stderr.errors) + + +class TextLogStream(io.TextIOWrapper): + def __init__(self, android_log_write, prio, tag, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(BinaryLogStream(android_log_write, prio, tag), **kwargs) + self._CHUNK_SIZE = MAX_BYTES_PER_WRITE + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line wherever possible, so split + # the string before sending it to the superclass. Note that + # "".splitlines() == [], so nothing will be logged for an empty string. + for line in s.splitlines(keepends=True): + while line: + super().write(line[:MAX_CHARS_PER_WRITE]) + line = line[MAX_CHARS_PER_WRITE:] + + return len(s) + + +class BinaryLogStream(io.RawIOBase): + def __init__(self, android_log_write, prio, tag): + self.android_log_write = android_log_write + self.prio = prio + self.tag = tag + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.android_log_write(self.prio, self.tag, + b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py new file mode 100644 index 000000000000000..115882a4c281f60 --- /dev/null +++ b/Lib/test/test_android.py @@ -0,0 +1,332 @@ +import platform +import queue +import re +import subprocess +import sys +import unittest +from array import array +from contextlib import contextmanager +from threading import Thread +from test.support import LOOPBACK_TIMEOUT +from time import time + + +if sys.platform != "android": + raise unittest.SkipTest("Android-specific") + +api_level = platform.android_ver().api_level + + +# Test redirection of stdout and stderr to the Android log. +@unittest.skipIf( + api_level < 23 and platform.machine() == "aarch64", + "SELinux blocks reading logs on older ARM64 emulators" +) +class TestAndroidOutput(unittest.TestCase): + maxDiff = None + + def setUp(self): + self.logcat_process = subprocess.Popen( + ["logcat", "-v", "tag"], stdout=subprocess.PIPE, + errors="backslashreplace" + ) + self.logcat_queue = queue.Queue() + + def logcat_thread(): + for line in self.logcat_process.stdout: + self.logcat_queue.put(line.rstrip("\n")) + self.logcat_process.stdout.close() + Thread(target=logcat_thread).start() + + from ctypes import CDLL, c_char_p, c_int + android_log_write = getattr(CDLL("liblog.so"), "__android_log_write") + android_log_write.argtypes = (c_int, c_char_p, c_char_p) + ANDROID_LOG_INFO = 4 + + # Separate tests using a marker line with a different tag. + tag, message = "python.test", f"{self.id()} {time()}" + android_log_write( + ANDROID_LOG_INFO, tag.encode("UTF-8"), message.encode("UTF-8")) + self.assert_log("I", tag, message, skip=True, timeout=5) + + def assert_logs(self, level, tag, expected, **kwargs): + for line in expected: + self.assert_log(level, tag, line, **kwargs) + + def assert_log(self, level, tag, expected, *, skip=False, timeout=0.5): + deadline = time() + timeout + while True: + try: + line = self.logcat_queue.get(timeout=(deadline - time())) + except queue.Empty: + self.fail(f"line not found: {expected!r}") + if match := re.fullmatch(fr"(.)/{tag}: (.*)", line): + try: + self.assertEqual(level, match[1]) + self.assertEqual(expected, match[2]) + break + except AssertionError: + if not skip: + raise + + def tearDown(self): + self.logcat_process.terminate() + self.logcat_process.wait(LOOPBACK_TIMEOUT) + + @contextmanager + def unbuffered(self, stream): + stream.reconfigure(write_through=True) + try: + yield + finally: + stream.reconfigure(write_through=False) + + def test_str(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name) + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + self.assertEqual("UTF-8", stream.encoding) + self.assertTrue(stream.line_buffering) + self.assertFalse(stream.write_through) + + # stderr is backslashreplace by default; stdout is configured + # that way by libregrtest.main. + self.assertEqual("backslashreplace", stream.errors) + + def write(s, lines=None, *, write_len=None): + if write_len is None: + write_len = len(s) + self.assertEqual(write_len, stream.write(s)) + if lines is None: + lines = [s] + self.assert_logs(level, tag, lines) + + # Single-line messages, + with self.unbuffered(stream): + write("", []) + + write("a") + write("Hello") + write("Hello world") + write(" ") + write(" ") + + # Non-ASCII text + write("ol\u00e9") # Spanish + write("\u4e2d\u6587") # Chinese + + # Non-BMP emoji + write("\U0001f600") + + # Non-encodable surrogates + write("\ud800\udc00", [r"\ud800\udc00"]) + + # Code used by surrogateescape (which isn't enabled here) + write("\udc80", [r"\udc80"]) + + # Null characters are logged using "modified UTF-8". + write("\u0000", [r"\xc0\x80"]) + write("a\u0000", [r"a\xc0\x80"]) + write("\u0000b", [r"\xc0\x80b"]) + write("a\u0000b", [r"a\xc0\x80b"]) + + # Multi-line messages. Avoid identical consecutive lines, as + # they may activate "chatty" filtering and break the tests. + write("\nx", [""]) + write("\na\n", ["x", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d"]) + write("xx", []) + write("f\n\ng", ["exxf", ""]) + write("\n", ["g"]) + + with self.unbuffered(stream): + write("\nx", ["", "x"]) + write("\na\n", ["", "a"]) + write("\n", [""]) + write("b\n", ["b"]) + write("c\n\n", ["c", ""]) + write("d\ne", ["d", "e"]) + write("xx", ["xx"]) + write("f\n\ng", ["f", "", "g"]) + write("\n", [""]) + + # "\r\n" should be translated into "\n". + write("hello\r\n", ["hello"]) + write("hello\r\nworld\r\n", ["hello", "world"]) + write("\r\n", [""]) + + # Non-standard line separators should be preserved. + write("before form feed\x0cafter form feed\n", + ["before form feed\x0cafter form feed"]) + write("before line separator\u2028after line separator\n", + ["before line separator\u2028after line separator"]) + + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + write(CustomStr("custom\n"), ["custom"], write_len=7) + + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) + + # Manual flushing is supported. + write("hello", []) + stream.flush() + self.assert_log(level, tag, "hello") + write("hello", []) + write("world", []) + stream.flush() + self.assert_log(level, tag, "helloworld") + + # Long lines are split into blocks of 1000 characters + # (MAX_CHARS_PER_WRITE in _android_support.py), but + # TextIOWrapper should then join them back together as much as + # possible without exceeding 4000 UTF-8 bytes + # (MAX_BYTES_PER_WRITE). + # + # ASCII (1 byte per character) + write(("foobar" * 700) + "\n", + [("foobar" * 666) + "foob", # 4000 bytes + "ar" + ("foobar" * 33)]) # 200 bytes + + # "Full-width" digits 0-9 (3 bytes per character) + s = "\uff10\uff11\uff12\uff13\uff14\uff15\uff16\uff17\uff18\uff19" + write((s * 150) + "\n", + [s * 100, # 3000 bytes + s * 50]) # 1500 bytes + + s = "0123456789" + write(s * 200, []) + write(s * 150, []) + write(s * 51, [s * 350]) # 3500 bytes + write("\n", [s * 51]) # 510 bytes + + def test_bytes(self): + for stream_name, level in [("stdout", "I"), ("stderr", "W")]: + with self.subTest(stream=stream_name): + stream = getattr(sys, stream_name).buffer + tag = f"python.{stream_name}" + self.assertEqual(f"", repr(stream)) + self.assertTrue(stream.writable()) + self.assertFalse(stream.readable()) + + def write(b, lines=None, *, write_len=None): + if write_len is None: + write_len = len(b) + self.assertEqual(write_len, stream.write(b)) + if lines is None: + lines = [b.decode()] + self.assert_logs(level, tag, lines) + + # Single-line messages, + write(b"", []) + + write(b"a") + write(b"Hello") + write(b"Hello world") + write(b" ") + write(b" ") + + # Non-ASCII text + write(b"ol\xc3\xa9") # Spanish + write(b"\xe4\xb8\xad\xe6\x96\x87") # Chinese + + # Non-BMP emoji + write(b"\xf0\x9f\x98\x80") + + # Null bytes are logged using "modified UTF-8". + write(b"\x00", [r"\xc0\x80"]) + write(b"a\x00", [r"a\xc0\x80"]) + write(b"\x00b", [r"\xc0\x80b"]) + write(b"a\x00b", [r"a\xc0\x80b"]) + + # Invalid UTF-8 + write(b"\xff", [r"\xff"]) + write(b"a\xff", [r"a\xff"]) + write(b"\xffb", [r"\xffb"]) + write(b"a\xffb", [r"a\xffb"]) + + # Log entries containing newlines are shown differently by + # `logcat -v tag`, `logcat -v long`, and Android Studio. We + # currently use `logcat -v tag`, which shows each line as if it + # was a separate log entry, but strips a single trailing + # newline. + # + # On newer versions of Android, all three of the above tools (or + # maybe Logcat itself) will also strip any number of leading + # newlines. + write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) + write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) + write(b"\n", [""]) + write(b"b\n", ["b"]) + write(b"c\n\n", ["c", ""]) + write(b"d\ne", ["d", "e"]) + write(b"xx", ["xx"]) + write(b"f\n\ng", ["f", "", "g"]) + write(b"\n", [""]) + + # "\r\n" should be translated into "\n". + write(b"hello\r\n", ["hello"]) + write(b"hello\r\nworld\r\n", ["hello", "world"]) + write(b"\r\n", [""]) + + # Other bytes-like objects are accepted. + write(bytearray(b"bytearray")) + + mv = memoryview(b"memoryview") + write(mv, ["memoryview"]) # Continuous + write(mv[::2], ["mmrve"]) # Discontinuous + + write( + # Android only supports little-endian architectures, so the + # bytes representation is as follows: + array("H", [ + 0, # 00 00 + 1, # 01 00 + 65534, # FE FF + 65535, # FF FF + ]), + + # After encoding null bytes with modified UTF-8, the only + # valid UTF-8 sequence is \x01. All other bytes are handled + # by backslashreplace. + ["\\xc0\\x80\\xc0\\x80" + "\x01\\xc0\\x80" + "\\xfe\\xff" + "\\xff\\xff"], + write_len=8, + ) + + # Non-bytes-like classes are not accepted. + for obj in ["", "hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + stream.write(obj) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst new file mode 100644 index 000000000000000..04f84792f667a06 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-17-22-49-15.gh-issue-116622.tthNUF.rst @@ -0,0 +1 @@ +Redirect stdout and stderr to system log when embedded in an Android app. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0f3ca4a66875144..a672d8c7915ccd7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -71,6 +71,9 @@ static PyStatus add_main_module(PyInterpreterState *interp); static PyStatus init_import_site(void); static PyStatus init_set_builtins_open(void); static PyStatus init_sys_streams(PyThreadState *tstate); +#ifdef __ANDROID__ +static PyStatus init_android_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1223,6 +1226,13 @@ init_interp_main(PyThreadState *tstate) return status; } +#ifdef __ANDROID__ + status = init_android_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } +#endif + #ifdef Py_DEBUG run_presite(tstate); #endif @@ -2719,6 +2729,73 @@ init_sys_streams(PyThreadState *tstate) } +#ifdef __ANDROID__ +#include + +static PyObject * +android_log_write_impl(PyObject *self, PyObject *args) +{ + int prio = 0; + const char *tag = NULL; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "isy", &prio, &tag, &text)) { + return NULL; + } + + // Despite its name, this function is part of the public API + // (https://developer.android.com/ndk/reference/group/logging). + __android_log_write(prio, tag, text); + Py_RETURN_NONE; +} + + +static PyMethodDef android_log_write_method = { + "android_log_write", android_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_android_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_android_support = NULL; + PyObject *android_log_write = NULL; + PyObject *result = NULL; + + _android_support = PyImport_ImportModule("_android_support"); + if (_android_support == NULL) { + goto error; + } + + android_log_write = PyCFunction_New(&android_log_write_method, NULL); + if (android_log_write == NULL) { + goto error; + } + + // These log priorities match those used by Java's System.out and System.err. + result = PyObject_CallMethod( + _android_support, "init_streams", "Oii", + android_log_write, ANDROID_LOG_INFO, ANDROID_LOG_WARN); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Android streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(android_log_write); + Py_XDECREF(_android_support); + return status; +} + +#endif // __ANDROID__ + + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, PyThreadState *tstate) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index 08a66f447e22580..f44abf1f9c3121d 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -5,6 +5,7 @@ static const char* _Py_stdlib_module_names[] = { "__future__", "_abc", "_aix_support", +"_android_support", "_ast", "_asyncio", "_bisect", diff --git a/configure b/configure index 571ab8c882aa685..e1d0f36463c44b0 100755 --- a/configure +++ b/configure @@ -7103,6 +7103,9 @@ printf "%s\n" "$ANDROID_API_LEVEL" >&6; } printf "%s\n" "#define ANDROID_API_LEVEL $ANDROID_API_LEVEL" >>confdefs.h + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for the Android arm ABI" >&5 printf %s "checking for the Android arm ABI... " >&6; } { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $_arm_arch" >&5 diff --git a/configure.ac b/configure.ac index a2d6b1357efbc9d..7681ea3d3fe3c69 100644 --- a/configure.ac +++ b/configure.ac @@ -1192,6 +1192,9 @@ if $CPP $CPPFLAGS conftest.c >conftest.out 2>/dev/null; then AC_DEFINE_UNQUOTED([ANDROID_API_LEVEL], [$ANDROID_API_LEVEL], [The Android API level.]) + # For __android_log_write() in Python/pylifecycle.c. + LIBS="$LIBS -llog" + AC_MSG_CHECKING([for the Android arm ABI]) AC_MSG_RESULT([$_arm_arch]) if test "$_arm_arch" = 7; then From c0eaa232f63a62e0e0408911ab5f118dca2af607 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 30 Apr 2024 17:23:44 +0300 Subject: [PATCH 010/220] gh-117860: Add tests for resolving names when import rebind names (GH-118176) Add tests for "import", pkgutil.resolve_name() and unittest.mock.path() for cases when "import a.b as x" and "from a import b as x" give different results. --- Lib/test/test_import/__init__.py | 28 ++++++++ .../test_import/data/package3/__init__.py | 2 + .../test_import/data/package3/submodule.py | 7 ++ .../test_import/data/package4/__init__.py | 5 ++ .../test_import/data/package4/submodule.py | 3 + Lib/test/test_pkgutil.py | 35 ++++++++++ Lib/test/test_unittest/testmock/testpatch.py | 67 +++++++++++++++++++ Makefile.pre.in | 2 + 8 files changed, 149 insertions(+) create mode 100644 Lib/test/test_import/data/package3/__init__.py create mode 100644 Lib/test/test_import/data/package3/submodule.py create mode 100644 Lib/test/test_import/data/package4/__init__.py create mode 100644 Lib/test/test_import/data/package4/submodule.py diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 40c3023a827067b..f88b51f916cc818 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1327,6 +1327,34 @@ def test_import_from_unloaded_package(self): import package2.submodule1 package2.submodule1.submodule2 + def test_rebinding(self): + # The same data is also used for testing pkgutil.resolve_name() + # in test_pkgutil and mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + import package3.submodule as submodule + self.assertEqual(submodule.attr, 'rebound') + from package3 import submodule + self.assertEqual(submodule.attr, 'rebound') + + def test_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + from package4 import submodule + self.assertEqual(submodule.attr, 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + from package4 import submodule + self.assertEqual(submodule.attr, 'origin') + import package4.submodule as submodule + self.assertEqual(submodule.attr, 'submodule') + class OverridingImportBuiltinTests(unittest.TestCase): def test_override_builtin(self): diff --git a/Lib/test/test_import/data/package3/__init__.py b/Lib/test/test_import/data/package3/__init__.py new file mode 100644 index 000000000000000..7033c22a719ec45 --- /dev/null +++ b/Lib/test/test_import/data/package3/__init__.py @@ -0,0 +1,2 @@ +"""Rebinding the package attribute after importing the module.""" +from .submodule import submodule diff --git a/Lib/test/test_import/data/package3/submodule.py b/Lib/test/test_import/data/package3/submodule.py new file mode 100644 index 000000000000000..cd7b30db15e449a --- /dev/null +++ b/Lib/test/test_import/data/package3/submodule.py @@ -0,0 +1,7 @@ +attr = 'submodule' +class A: + attr = 'submodule' +class submodule: + attr = 'rebound' + class B: + attr = 'rebound' diff --git a/Lib/test/test_import/data/package4/__init__.py b/Lib/test/test_import/data/package4/__init__.py new file mode 100644 index 000000000000000..d8af60ab38a319d --- /dev/null +++ b/Lib/test/test_import/data/package4/__init__.py @@ -0,0 +1,5 @@ +"""Binding the package attribute without importing the module.""" +class submodule: + attr = 'origin' + class B: + attr = 'origin' diff --git a/Lib/test/test_import/data/package4/submodule.py b/Lib/test/test_import/data/package4/submodule.py new file mode 100644 index 000000000000000..c861417aece9c35 --- /dev/null +++ b/Lib/test/test_import/data/package4/submodule.py @@ -0,0 +1,3 @@ +attr = 'submodule' +class A: + attr = 'submodule' diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index 6fcd726345eeac4..e19dce1dbd25834 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -12,6 +12,9 @@ import shutil import zipfile +from test.support.import_helper import DirsOnSysPath +from test.test_importlib.util import uncache + # Note: pkgutil.walk_packages is currently tested in test_runpy. This is # a hack to get a major issue resolved for 3.3b2. Longer term, it should # be moved back here, perhaps by factoring out the helper code for @@ -318,6 +321,38 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_import_rebinding(self): + # The same data is also used for testing import in test_import and + # mock.patch in test_unittest. + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + self.assertEqual(pkgutil.resolve_name('package3.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package3:submodule.attr'), 'rebound') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(__file__), 'test_import', 'data') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule.attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'origin') + self.assertEqual(pkgutil.resolve_name('package4.submodule:attr'), 'submodule') + self.assertEqual(pkgutil.resolve_name('package4:submodule.attr'), 'submodule') + class PkgutilPEP302Tests(unittest.TestCase): diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index d0046d702a53f42..be75fda7826af17 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest +import test from test.test_unittest.testmock import support from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -1728,6 +1730,71 @@ def test(mock): 'exception traceback not propagated') + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, diff --git a/Makefile.pre.in b/Makefile.pre.in index 0e52e10602cf857..4e4d01d14ee3cbc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2380,6 +2380,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/test_import/data/circular_imports/subpkg2/parent \ test/test_import/data/package \ test/test_import/data/package2 \ + test/test_import/data/package3 \ + test/test_import/data/package4 \ test/test_import/data/unwritable \ test/test_importlib \ test/test_importlib/builtin \ From 02887c6428042832f17fd71ed90619f25d8f951d Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 30 Apr 2024 17:35:19 +0300 Subject: [PATCH 011/220] gh-102402: Make test_relativeCreated_has_higher_precision less implementation dependent (GH-118062) --- Lib/test/test_logging.py | 60 ++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index d8ae607d1d8d014..e651d96f100a837 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4598,25 +4598,55 @@ def test_msecs_has_no_floating_point_precision_loss(self): self.assertEqual(record.msecs, want) self.assertEqual(record.created, ns / 1e9) - # The test overrides a private attribute - @support.cpython_only def test_relativeCreated_has_higher_precision(self): - # See issue gh-102402 - ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging - with (patch("time.time_ns") as time_ns_mock, - support.swap_attr(logging, '_startTime', ns)): - for offset_ns in offsets_ns: - # mock for log record creation - new_ns = ns + offset_ns - time_ns_mock.return_value = new_ns - - record = logging.makeLogRecord({'msg': 'test'}) - self.assertAlmostEqual(record.created, new_ns / 1e9, places=6) - + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) # After PR gh-102412, precision (places) increases from 3 to 7 - self.assertAlmostEqual(record.relativeCreated, offset_ns / 1e6, places=7) + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) class TestBufferingFormatter(logging.BufferingFormatter): From 17a8af9508d6001e1666de984e2fb73f02167306 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 30 Apr 2024 17:49:28 +0300 Subject: [PATCH 012/220] gh-118402: Fix inspect.signature() for functools.cmp_to_key() result (GH-118427) --- Lib/test/test_functools.py | 12 +++++++++--- .../2024-04-29-21-51-28.gh-issue-118402.Z_06Th.rst | 2 ++ Modules/_functoolsmodule.c | 12 ++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-29-21-51-28.gh-issue-118402.Z_06Th.rst diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index bb4c7cc8701fb42..4a9a7313712f602 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -947,8 +947,13 @@ def mycmp(x, y): @unittest.skipIf(support.MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_cmp_to_signature(self): - self.assertEqual(str(Signature.from_callable(self.cmp_to_key)), - '(mycmp)') + sig = Signature.from_callable(self.cmp_to_key) + self.assertEqual(str(sig), '(mycmp)') + def mycmp(x, y): + return y - x + sig = Signature.from_callable(self.cmp_to_key(mycmp)) + self.assertEqual(str(sig), '(obj)') + @unittest.skipUnless(c_functools, 'requires the C _functools module') @@ -1864,9 +1869,10 @@ def test_staticmethod(x): self.assertIsNone(ref()) def test_common_signatures(self): - def orig(): ... + def orig(a, /, b, c=True): ... lru = self.module.lru_cache(1)(orig) + self.assertEqual(str(Signature.from_callable(lru)), '(a, /, b, c=True)') self.assertEqual(str(Signature.from_callable(lru.cache_info)), '()') self.assertEqual(str(Signature.from_callable(lru.cache_clear)), '()') diff --git a/Misc/NEWS.d/next/Library/2024-04-29-21-51-28.gh-issue-118402.Z_06Th.rst b/Misc/NEWS.d/next/Library/2024-04-29-21-51-28.gh-issue-118402.Z_06Th.rst new file mode 100644 index 000000000000000..25d7b45b8726c93 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-29-21-51-28.gh-issue-118402.Z_06Th.rst @@ -0,0 +1,2 @@ +Fix :func:`inspect.signature` for the result of the +:func:`functools.cmp_to_key` call. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 406fcf0da2f7e42..e37473a582b55fb 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -571,6 +571,17 @@ static PyMemberDef keyobject_members[] = { {NULL} }; +static PyObject * +keyobject_text_signature(PyObject *self, void *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("(obj)"); +} + +static PyGetSetDef keyobject_getset[] = { + {"__text_signature__", keyobject_text_signature, (setter)NULL}, + {NULL} +}; + static PyObject * keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); @@ -585,6 +596,7 @@ static PyType_Slot keyobject_type_slots[] = { {Py_tp_clear, keyobject_clear}, {Py_tp_richcompare, keyobject_richcompare}, {Py_tp_members, keyobject_members}, + {Py_tp_getset, keyobject_getset}, {0, 0} }; From 9a75d56d5d9fdffb6ce9d83ede98486df238102d Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 30 Apr 2024 16:55:15 +0200 Subject: [PATCH 013/220] gh-118379: Use PyTuple_Pack instead of Py_BuildValue if possible (GH-118381) --- Modules/_decimal/_decimal.c | 4 ++-- Modules/_testbuffer.c | 2 +- Modules/_testcapimodule.c | 4 ++-- Modules/_testclinic.c | 2 +- Modules/posixmodule.c | 2 +- Python/errors.c | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index c105367a2704589..fe6711143b845b1 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -4287,7 +4287,7 @@ nm_mpd_qdivmod(PyObject *v, PyObject *w) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; @@ -5312,7 +5312,7 @@ ctx_mpd_qdivmod(PyObject *context, PyObject *args) return NULL; } - ret = Py_BuildValue("(OO)", q, r); + ret = PyTuple_Pack(2, q, r); Py_DECREF(r); Py_DECREF(q); return ret; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index cad21bdb4d85ed3..9e77744efad728a 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -1217,7 +1217,7 @@ init_ndbuf(PyObject *items, PyObject *shape, PyObject *strides, /* convert scalar to list */ if (ndim == 0) { - items = Py_BuildValue("(O)", items); + items = PyTuple_Pack(1, items); if (items == NULL) return NULL; } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0bdd252efdabc75..3448291e401e357 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2475,7 +2475,7 @@ get_basic_static_type(PyObject *self, PyObject *args) PyTypeObject *cls = &BasicStaticTypes[num_basic_static_types_used++]; if (base != NULL) { - cls->tp_bases = Py_BuildValue("(O)", base); + cls->tp_bases = PyTuple_Pack(1, base); if (cls->tp_bases == NULL) { return NULL; } @@ -3474,7 +3474,7 @@ typedef struct { static PyObject * ipowType_ipow(PyObject *self, PyObject *other, PyObject *mod) { - return Py_BuildValue("OO", other, mod); + return PyTuple_Pack(2, other, mod); } static PyNumberMethods ipowType_as_number = { diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 454173b434fb6bb..2e9d00ac2189eb7 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1243,7 +1243,7 @@ _testclinic_TestClass_get_defining_class_arg_impl(PyObject *self, PyObject *arg) /*[clinic end generated code: output=fe7e49d96cbb7718 input=d1b83d3b853af6d9]*/ { - return Py_BuildValue("(OO)", cls, arg); + return PyTuple_Pack(2, cls, arg); } static struct PyMethodDef test_class_methods[] = { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c9d67ccbb8c9089..722159a39d098f0 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5500,7 +5500,7 @@ os__path_splitroot_ex_impl(PyObject *module, PyObject *path) if (tail == NULL) { goto exit; } - result = Py_BuildValue("(OOO)", drv, root, tail); + result = PyTuple_Pack(3, drv, root, tail); exit: PyMem_Free(buffer); Py_XDECREF(drv); diff --git a/Python/errors.c b/Python/errors.c index e5f176a5dd208e4..433253b8f9aadaf 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -632,8 +632,8 @@ _PyErr_StackItemToExcInfoTuple(_PyErr_StackItem *err_info) PyObject *exc_type = get_exc_type(exc_value); PyObject *exc_traceback = get_exc_traceback(exc_value); - return Py_BuildValue( - "(OOO)", + return PyTuple_Pack( + 3, exc_type ? exc_type : Py_None, exc_value ? exc_value : Py_None, exc_traceback ? exc_traceback : Py_None); From 72dae53e09a5344bf4922d934a34a2fa48a11c86 Mon Sep 17 00:00:00 2001 From: Seth Michael Larson Date: Tue, 30 Apr 2024 10:05:05 -0500 Subject: [PATCH 014/220] gh-116122: Add SBOM generation to PCbuild/build.bat (GH-116138) --- PCbuild/regen.targets | 15 +++++++++++++-- Tools/build/generate_sbom.py | 21 ++++++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index b72ef5b5c6055c6..4aa14ed1fad9ebd 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -33,11 +33,15 @@ <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> + <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> + <_SbomOutputs Include="$(PySourcePath)Misc\externals.spdx.json;$(PySourcePath)Misc\sbom.spdx.json"> + json + - @@ -126,7 +130,14 @@ DependsOnTargets="_TouchRegenSources;_RegenPegen;_RegenAST_H;_RegenTokens;_RegenKeywords;_RegenGlobalObjects"> - + + + + + diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 5c1851f09338a06..258b58c03c68003 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,13 +4,13 @@ import hashlib import json import glob -import pathlib +from pathlib import Path, PurePosixPath, PureWindowsPath import subprocess import sys import urllib.request import typing -CPYTHON_ROOT_DIR = pathlib.Path(__file__).parent.parent.parent +CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent # Before adding a new entry to this list, double check that # the license expression is a valid SPDX license expression: @@ -119,9 +119,16 @@ def filter_gitignored_paths(paths: list[str]) -> list[str]: # 1 means matches, 0 means no matches. assert git_check_ignore_proc.returncode in (0, 1) + # Paths may or may not be quoted, Windows quotes paths. + git_check_ignore_re = re.compile(r"^::\s+(\"([^\"]+)\"|(.+))\Z") + # Return the list of paths sorted git_check_ignore_lines = git_check_ignore_proc.stdout.decode().splitlines() - return sorted([line.split()[-1] for line in git_check_ignore_lines if line.startswith("::")]) + git_check_not_ignored = [] + for line in git_check_ignore_lines: + if match := git_check_ignore_re.fullmatch(line): + git_check_not_ignored.append(match.group(2) or match.group(3)) + return sorted(git_check_not_ignored) def get_externals() -> list[str]: @@ -238,12 +245,20 @@ def create_source_sbom() -> None: ) for path in paths: + + # Normalize the filename from any combination of slashes. + path = str(PurePosixPath(PureWindowsPath(path))) + # Skip directories and excluded files if not (CPYTHON_ROOT_DIR / path).is_file() or path in exclude: continue # SPDX requires SHA1 to be used for files, but we provide SHA256 too. data = (CPYTHON_ROOT_DIR / path).read_bytes() + # We normalize line-endings for consistent checksums. + # This is a rudimentary check for binary files. + if b"\x00" not in data: + data = data.replace(b"\r\n", b"\n") checksum_sha1 = hashlib.sha1(data).hexdigest() checksum_sha256 = hashlib.sha256(data).hexdigest() From 4a5ad8469af9a6fc0ec1355eb203cc22bb4321d5 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 30 Apr 2024 18:44:37 +0300 Subject: [PATCH 015/220] gh-118418: Use a default value for `type_params` in `typing._eval_type` (#118431) --- Lib/typing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/typing.py b/Lib/typing.py index b3f4ba99f9ec219..eff65cfb68b8666 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -400,7 +400,7 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, type_params, *, recursive_guard=frozenset()): +def _eval_type(t, globalns, localns, type_params=None, *, recursive_guard=frozenset()): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). @@ -981,7 +981,7 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): self.__forward_is_class__ = is_class self.__forward_module__ = module - def _evaluate(self, globalns, localns, type_params, *, recursive_guard): + def _evaluate(self, globalns, localns, type_params=None, *, recursive_guard): if self.__forward_arg__ in recursive_guard: return self if not self.__forward_evaluated__ or localns is not globalns: From d7ac427a796a3f869b813dac37b030889b56cd3b Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 30 Apr 2024 11:18:01 -0700 Subject: [PATCH 016/220] gh-117618: Make package.module searchable for breakpoints and clean up docs (#117619) --- Doc/library/pdb.rst | 19 ++++++--- Lib/pdb.py | 24 ++++++----- Lib/test/test_pdb.py | 40 +++++++++++++++++++ ...-04-08-03-23-22.gh-issue-117618.-4DCUw.rst | 1 + 4 files changed, 69 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-08-03-23-22.gh-issue-117618.-4DCUw.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index ac3007f70c3534d..a640976493caba8 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -328,12 +328,16 @@ can be overridden by the local file. .. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. + With a *lineno* argument, set a break at line *lineno* in the current file. + The line number may be prefixed with a *filename* and a colon, + to specify a breakpoint in another file (possibly one that hasn't been loaded + yet). The file is searched on :data:`sys.path`. Accepatable forms of *filename* + are ``/abspath/to/file.py``, ``relpath/file.py``, ``module`` and + ``package.module``. + + With a *function* argument, set a break at the first executable statement within + that function. *function* can be any expression that evaluates to a function + in the current namespace. If a second argument is present, it is an expression which must evaluate to true before the breakpoint is honored. @@ -342,6 +346,9 @@ can be overridden by the local file. of times that breakpoint has been hit, the current ignore count, and the associated condition if any. + Each breakpoint is assigned a number to which all the other + breakpoint commands refer. + .. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] Temporary breakpoint, which is removed automatically when it is first hit. diff --git a/Lib/pdb.py b/Lib/pdb.py index d4138b95d3c3322..fa2e1ec94be2358 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -2007,17 +2007,23 @@ def lookupmodule(self, filename): lookupmodule() translates (possibly incomplete) file or module name into an absolute file name. + + filename could be in format of: + * an absolute path like '/path/to/file.py' + * a relative path like 'file.py' or 'dir/file.py' + * a module name like 'module' or 'package.module' + + files and modules will be searched in sys.path. """ - if os.path.isabs(filename) and os.path.exists(filename): - return filename - f = os.path.join(sys.path[0], filename) - if os.path.exists(f) and self.canonic(f) == self.mainpyfile: - return f - root, ext = os.path.splitext(filename) - if ext == '': - filename = filename + '.py' + if not filename.endswith('.py'): + # A module is passed in so convert it to equivalent file + filename = filename.replace('.', os.sep) + '.py' + if os.path.isabs(filename): - return filename + if os.path.exists(filename): + return filename + return None + for dirname in sys.path: while os.path.islink(dirname): dirname = os.readlink(dirname) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 2d057e2647f13c4..5635d17c99d2a32 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -354,6 +354,46 @@ def test_pdb_breakpoint_commands(): 4 """ +def test_pdb_breakpoint_with_filename(): + """Breakpoints with filename:lineno + + >>> def test_function(): + ... # inspect_fodder2 is a great module as the line number is stable + ... from test.test_inspect import inspect_fodder2 as mod2 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... mod2.func88() + ... mod2.func114() + ... # Be a good citizen and clean up the mess + ... reset_Breakpoint() + + First, need to clear bdb state that might be left over from previous tests. + Otherwise, the new breakpoints might get assigned different numbers. + + >>> reset_Breakpoint() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE +ELLIPSIS + ... 'break test.test_inspect.inspect_fodder2:90', + ... 'continue', # will stop at func88 + ... 'break test/test_inspect/inspect_fodder2.py:115', + ... 'continue', # will stop at func114 + ... 'continue', + ... ]): + ... test_function() + > (5)test_function() + -> mod2.func88() + (Pdb) break test.test_inspect.inspect_fodder2:90 + Breakpoint 1 at ...inspect_fodder2.py:90 + (Pdb) continue + > ...inspect_fodder2.py(90)func88() + -> return 90 + (Pdb) break test/test_inspect/inspect_fodder2.py:115 + Breakpoint 2 at ...inspect_fodder2.py:115 + (Pdb) continue + > ...inspect_fodder2.py(115)func114() + -> return 115 + (Pdb) continue + """ + def test_pdb_breakpoints_preserved_across_interactive_sessions(): """Breakpoints are remembered between interactive sessions diff --git a/Misc/NEWS.d/next/Library/2024-04-08-03-23-22.gh-issue-117618.-4DCUw.rst b/Misc/NEWS.d/next/Library/2024-04-08-03-23-22.gh-issue-117618.-4DCUw.rst new file mode 100644 index 000000000000000..569c5d57effdcb9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-08-03-23-22.gh-issue-117618.-4DCUw.rst @@ -0,0 +1 @@ +Support ``package.module`` as ``filename`` for ``break`` command of :mod:`pdb` From f7747f73a9d9b9b1661c1a69cd8d934d56bbd3b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 30 Apr 2024 20:25:25 +0200 Subject: [PATCH 017/220] gh-109975: Document crypt_r as a possible replacement of crypt (#118439) --- Doc/whatsnew/3.13.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 98349a5984bb7ea..a9a7e485dd2bac4 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1437,7 +1437,10 @@ PEP 594: dead batteries * :pypi:`argon2-cffi`: The secure Argon2 password hashing algorithm. * :pypi:`legacycrypt`: - Wrapper to the POSIX crypt library call and associated functionality. + :mod:`ctypes` wrapper to the POSIX crypt library call and associated functionality. + * :pypi:`crypt_r`: + Fork of the :mod:`!crypt` module, wrapper to the :manpage:`crypt_r(3)` library + call and associated functionality. (Contributed by Victor Stinner in :gh:`104773`.) From 1f16b4ce569f222af74fcbb7b2ef98eee2398d20 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Tue, 30 Apr 2024 19:32:25 +0100 Subject: [PATCH 018/220] gh-118272: Clear generator frame's locals when the generator is closed (#118277) Co-authored-by: Thomas Grainger --- Include/internal/pycore_frame.h | 3 +++ Lib/test/test_generators.py | 20 +++++++++++++++++++ ...-04-25-12-55-47.gh-issue-118272.5ptjk_.rst | 2 ++ Objects/genobject.c | 1 + Python/frame.c | 17 +++++++++++----- 5 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-25-12-55-47.gh-issue-118272.5ptjk_.rst diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index f913928f38bd058..37ae5ae850389b3 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -227,6 +227,9 @@ _PyFrame_GetFrameObject(_PyInterpreterFrame *frame) return _PyFrame_MakeAndSetFrameObject(frame); } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame); + /* Clears all references in the frame. * If take is non-zero, then the _PyInterpreterFrame frame * may be transferred to the frame object it references diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index d48f0d47ba1962b..6d36df2c7413e05 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -532,6 +532,26 @@ def f(): with self.assertRaises(RuntimeError): gen.close() + def test_close_releases_frame_locals(self): + # See gh-118272 + + class Foo: + pass + + f = Foo() + f_wr = weakref.ref(f) + + def genfn(): + a = f + yield + + g = genfn() + next(g) + del f + g.close() + support.gc_collect() + self.assertIsNone(f_wr()) + class GeneratorThrowTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-25-12-55-47.gh-issue-118272.5ptjk_.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-12-55-47.gh-issue-118272.5ptjk_.rst new file mode 100644 index 000000000000000..32043440fd03651 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-12-55-47.gh-issue-118272.5ptjk_.rst @@ -0,0 +1,2 @@ +Fix bug where ``generator.close`` does not free the generator frame's +locals. diff --git a/Objects/genobject.c b/Objects/genobject.c index 5d7da49cfca1668..04736a04562e149 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -380,6 +380,7 @@ gen_close(PyGenObject *gen, PyObject *args) // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); gen->gi_frame_state = FRAME_COMPLETED; + _PyFrame_ClearLocals((_PyInterpreterFrame *)gen->gi_iframe); Py_RETURN_NONE; } } diff --git a/Python/frame.c b/Python/frame.c index db9d13359a23ca2..ec390e7426ad470 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -94,6 +94,17 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) } } +void +_PyFrame_ClearLocals(_PyInterpreterFrame *frame) +{ + assert(frame->stacktop >= 0); + for (int i = 0; i < frame->stacktop; i++) { + Py_XDECREF(frame->localsplus[i]); + } + frame->stacktop = 0; + Py_CLEAR(frame->f_locals); +} + void _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) { @@ -114,11 +125,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) } Py_DECREF(f); } - assert(frame->stacktop >= 0); - for (int i = 0; i < frame->stacktop; i++) { - Py_XDECREF(frame->localsplus[i]); - } - Py_XDECREF(frame->f_locals); + _PyFrame_ClearLocals(frame); Py_DECREF(frame->f_funcobj); } From 4a1cf66c5c0afa36d7a51d5f9d3874cda10df79c Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 30 Apr 2024 11:38:05 -0700 Subject: [PATCH 019/220] gh-117657: Fix small issues with instrumentation and TSAN (#118064) Small TSAN fixups for instrumentation --- Include/internal/pycore_pyatomic_ft_wrappers.h | 11 +++++++++-- Objects/genobject.c | 8 +++++--- Python/bytecodes.c | 4 ++-- Python/ceval.c | 2 +- Python/ceval_macros.h | 2 +- Python/generated_cases.c.h | 2 +- Python/instrumentation.c | 9 ++++++--- 7 files changed, 25 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index bbfc462a733d0e3..83f02c92495b3f1 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -39,6 +39,10 @@ extern "C" { _Py_atomic_load_uint8(&value) #define FT_ATOMIC_STORE_UINT8(value, new_value) \ _Py_atomic_store_uint8(&value, new_value) +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) \ + _Py_atomic_load_uint8_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ + _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -49,7 +53,8 @@ extern "C" { _Py_atomic_store_ssize_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \ _Py_atomic_store_uint8_relaxed(&value, new_value) - +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ + _Py_atomic_store_uint16_relaxed(&value, new_value) #else #define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value @@ -62,12 +67,14 @@ extern "C" { #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value - +#define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #endif #ifdef __cplusplus diff --git a/Objects/genobject.c b/Objects/genobject.c index 04736a04562e149..a1ed1cbd2bfb580 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -11,6 +11,7 @@ #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_opcode_utils.h" // RESUME_AFTER_YIELD_FROM +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() @@ -329,10 +330,11 @@ gen_close_iter(PyObject *yf) static inline bool is_resume(_Py_CODEUNIT *instr) { + uint8_t code = FT_ATOMIC_LOAD_UINT8_RELAXED(instr->op.code); return ( - instr->op.code == RESUME || - instr->op.code == RESUME_CHECK || - instr->op.code == INSTRUMENTED_RESUME + code == RESUME || + code == RESUME_CHECK || + code == INSTRUMENTED_RESUME ); } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index eee8b328a65b8c8..5bb7e1211385a5e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -20,7 +20,7 @@ #include "pycore_object.h" // _PyObject_GC_TRACK() #include "pycore_opcode_metadata.h" // uop names #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject @@ -163,7 +163,7 @@ dummy_func( if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - this_instr->op.code = RESUME_CHECK; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } } diff --git a/Python/ceval.c b/Python/ceval.c index d130c734a67144e..8b23bc6bcbeb396 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -20,7 +20,7 @@ #include "pycore_opcode_metadata.h" // EXTRA_CASES #include "pycore_optimizer.h" // _PyUOpExecutor_Type #include "pycore_opcode_utils.h" // MAKE_FUNCTION_* -#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_range.h" // _PyRangeIterObject diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 1a8554ab72269fe..c88a07c1f5e9514 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -162,7 +162,7 @@ GETITEM(PyObject *v, Py_ssize_t i) { /* The integer overflow is checked by an assertion below. */ #define INSTR_OFFSET() ((int)(next_instr - _PyCode_CODE(_PyFrame_GetCode(frame)))) #define NEXTOPARG() do { \ - _Py_CODEUNIT word = *next_instr; \ + _Py_CODEUNIT word = {.cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t*)next_instr)}; \ opcode = word.op.code; \ oparg = word.op.arg; \ } while (0) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7c1cc147de4e1a6..a5bb29385844d8c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4955,7 +4955,7 @@ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - this_instr->op.code = RESUME_CHECK; + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); } DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 328a3b1733d6042..ce97c3add7d0ea3 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -626,9 +626,10 @@ de_instrument(PyCodeObject *code, int i, int event) return; } CHECK(_PyOpcode_Deopt[deinstrumented] == deinstrumented); - *opcode_ptr = deinstrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, deinstrumented); if (_PyOpcode_Caches[deinstrumented]) { - instr[1].counter = adaptive_counter_warmup(); + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); } } @@ -703,8 +704,10 @@ instrument(PyCodeObject *code, int i) int deopt = _PyOpcode_Deopt[opcode]; int instrumented = INSTRUMENTED_OPCODES[deopt]; assert(instrumented); - *opcode_ptr = instrumented; + FT_ATOMIC_STORE_UINT8_RELAXED(*opcode_ptr, instrumented); if (_PyOpcode_Caches[deopt]) { + FT_ATOMIC_STORE_UINT16_RELAXED(instr[1].counter.as_counter, + adaptive_counter_warmup().as_counter); instr[1].counter = adaptive_counter_warmup(); } } From b2c3b70c7102197e4505e6cd69722dc508527d22 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 30 Apr 2024 15:01:28 -0400 Subject: [PATCH 020/220] gh-118332: Fix deadlock involving stop the world (#118412) Avoid detaching thread state when stopping the world. When re-attaching the thread state, the thread would attempt to resume the top-most critical section, which might now be held by a thread paused for our stop-the-world request. --- Include/internal/pycore_lock.h | 6 +- .../test_critical_sections.c | 85 +++++++++++++++++++ Modules/_threadmodule.c | 3 +- Python/lock.c | 6 +- Python/pystate.c | 3 +- 5 files changed, 96 insertions(+), 7 deletions(-) diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index f993c95ecbf75ae..a5b28e4bd4744e8 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -150,8 +150,10 @@ PyAPI_FUNC(void) PyEvent_Wait(PyEvent *evt); // Wait for the event to be set, or until the timeout expires. If the event is // already set, then this returns immediately. Returns 1 if the event was set, -// and 0 if the timeout expired or thread was interrupted. -PyAPI_FUNC(int) PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns); +// and 0 if the timeout expired or thread was interrupted. If `detach` is +// true, then the thread will detach/release the GIL while waiting. +PyAPI_FUNC(int) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach); // _PyRawMutex implements a word-sized mutex that that does not depend on the // parking lot API, and therefore can be used in the parking lot diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index 94da0468fcf149a..cdf8a70fc79ff3f 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -204,6 +204,90 @@ test_critical_sections_threads(PyObject *self, PyObject *Py_UNUSED(args)) Py_DECREF(test_data.obj1); Py_RETURN_NONE; } + +static void +pysleep(int ms) +{ +#ifdef MS_WINDOWS + Sleep(ms); +#else + usleep(ms * 1000); +#endif +} + +struct test_data_gc { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t id; + Py_ssize_t countdown; + PyEvent done_event; + PyEvent ready; +}; + +static void +thread_gc(void *arg) +{ + struct test_data_gc *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + Py_ssize_t id = _Py_atomic_add_ssize(&test_data->id, 1); + if (id == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + else { + // wait for all test threads to more reliably reproduce the issue. + PyEvent_Wait(&test_data->ready); + } + + if (id == 0) { + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + // pause long enough that the lock would be handed off directly to + // a waiting thread. + pysleep(5); + PyGC_Collect(); + Py_END_CRITICAL_SECTION(); + } + else if (id == 1) { + pysleep(1); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + else if (id == 2) { + // sleep long enough so that thread 0 is waiting to stop the world + pysleep(6); + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(1); + Py_END_CRITICAL_SECTION(); + } + + PyGILState_Release(gil); + if (_Py_atomic_add_ssize(&test_data->countdown, -1) == 1) { + // last thread to finish sets done_event + _PyEvent_Notify(&test_data->done_event); + } +} + +static PyObject * +test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-118332: Contended critical sections should not deadlock with GC + const Py_ssize_t NUM_THREADS = 3; + struct test_data_gc test_data = { + .obj = PyDict_New(), + .countdown = NUM_THREADS, + .num_threads = NUM_THREADS, + }; + assert(test_data.obj != NULL); + + for (int i = 0; i < NUM_THREADS; i++) { + PyThread_start_new_thread(&thread_gc, &test_data); + } + PyEvent_Wait(&test_data.done_event); + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + #endif static PyMethodDef test_methods[] = { @@ -212,6 +296,7 @@ static PyMethodDef test_methods[] = { {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, + {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 5aa719c3834e614..f5e3b42600675e4 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -501,7 +501,8 @@ ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns) // Wait until the deadline for the thread to exit. PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0; - while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns)) { + int detach = 1; + while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) { if (deadline) { // _PyDeadline_Get will return a negative value if the deadline has // been exceeded. diff --git a/Python/lock.c b/Python/lock.c index 91c66df8fd90935..5ed95fcaf4188c3 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -277,12 +277,12 @@ _PyEvent_Notify(PyEvent *evt) void PyEvent_Wait(PyEvent *evt) { - while (!PyEvent_WaitTimed(evt, -1)) + while (!PyEvent_WaitTimed(evt, -1, /*detach=*/1)) ; } int -PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) +PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns, int detach) { for (;;) { uint8_t v = _Py_atomic_load_uint8(&evt->v); @@ -298,7 +298,7 @@ PyEvent_WaitTimed(PyEvent *evt, PyTime_t timeout_ns) uint8_t expected = _Py_HAS_PARKED; (void) _PyParkingLot_Park(&evt->v, &expected, sizeof(evt->v), - timeout_ns, NULL, 1); + timeout_ns, NULL, detach); return _Py_atomic_load_uint8(&evt->v) == _Py_LOCKED; } diff --git a/Python/pystate.c b/Python/pystate.c index 78b39c9a5774046..9d7b73b3a071e12 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2238,7 +2238,8 @@ stop_the_world(struct _stoptheworld_state *stw) } PyTime_t wait_ns = 1000*1000; // 1ms (arbitrary, may need tuning) - if (PyEvent_WaitTimed(&stw->stop_event, wait_ns)) { + int detach = 0; + if (PyEvent_WaitTimed(&stw->stop_event, wait_ns, detach)) { assert(stw->thread_countdown == 0); break; } From 19d468a3ab1d57c07a4283d67d439908574aa0cc Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Tue, 30 Apr 2024 12:37:38 -0700 Subject: [PATCH 021/220] [gh-117657] Fix some issues with TSAN in typeobject (#118249) Fix some racing reads in typebobject.c --- Include/internal/pycore_pyatomic_ft_wrappers.h | 8 ++++++++ Objects/typeobject.c | 17 +++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 83f02c92495b3f1..bc6aba56cf9fc7b 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -43,6 +43,8 @@ extern "C" { _Py_atomic_load_uint8_relaxed(&value) #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) \ _Py_atomic_load_uint16_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ + _Py_atomic_load_uint32_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ _Py_atomic_store_ptr_relaxed(&value, new_value) #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \ @@ -55,6 +57,9 @@ extern "C" { _Py_atomic_store_uint8_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) \ _Py_atomic_store_uint16_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ + _Py_atomic_store_uint32_relaxed(&value, new_value) + #else #define FT_ATOMIC_LOAD_PTR(value) value #define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value @@ -69,12 +74,15 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value + #endif #ifdef __cplusplus diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 50efbb6e182df05..ec19a3d461f6239 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -43,7 +43,8 @@ class object "PyObject *" "&PyBaseObject_Type" & ((1 << MCACHE_SIZE_EXP) - 1)) #define MCACHE_HASH_METHOD(type, name) \ - MCACHE_HASH((type)->tp_version_tag, ((Py_ssize_t)(name)) >> 3) + MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ + ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ PyUnicode_CheckExact(name) && \ PyUnicode_IS_READY(name) && \ @@ -907,7 +908,7 @@ type_modified_unlocked(PyTypeObject *type) } type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -984,7 +985,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; /* 0 is not a valid version tag */ + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1020,7 +1021,8 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, + NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1029,7 +1031,8 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - type->tp_version_tag = NEXT_VERSION_TAG(interp)++; + FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, + NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } @@ -5085,7 +5088,9 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) // synchronize-with other writing threads by doing an acquire load on the sequence while (1) { int sequence = _PySeqLock_BeginRead(&entry->sequence); - if (_Py_atomic_load_uint32_relaxed(&entry->version) == type->tp_version_tag && + uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); + uint32_t type_version = _Py_atomic_load_uint32_relaxed(&type->tp_version_tag); + if (entry_version == type_version && _Py_atomic_load_ptr_relaxed(&entry->name) == name) { assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); From b568c2c1ff5c0b1922a6402dc95c588d7f9aa914 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 30 Apr 2024 21:58:22 +0200 Subject: [PATCH 022/220] gh-118406: Add signature for sqlite3.Connection objects (#118428) --- Lib/test/test_sqlite3/test_dbapi.py | 5 +++++ .../Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst | 1 + Modules/_sqlite/connection.c | 7 +++++++ 3 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 6d8744ca5f79699..51ce095df41fc1e 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -590,6 +590,11 @@ def test_connection_resource_warning(self): del cx gc_collect() + def test_connection_signature(self): + from inspect import signature + sig = signature(self.cx) + self.assertEqual(str(sig), "(sql, /)") + class UninitialisedConnectionTests(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst b/Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst new file mode 100644 index 000000000000000..c60ddf9e00498e5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-30-15-18-19.gh-issue-118406.y-GnMo.rst @@ -0,0 +1 @@ +Add signature for :class:`sqlite3.Connection` objects. diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 74984ca53657435..fc03e4a085c179d 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -2561,6 +2561,12 @@ set_autocommit(pysqlite_Connection *self, PyObject *val, void *Py_UNUSED(ctx)) return 0; } +static PyObject * +get_sig(PyObject *self, void *Py_UNUSED(ctx)) +{ + return PyUnicode_FromString("(sql, /)"); +} + static const char connection_doc[] = PyDoc_STR("SQLite database connection object."); @@ -2570,6 +2576,7 @@ static PyGetSetDef connection_getset[] = { {"total_changes", (getter)pysqlite_connection_get_total_changes, (setter)0}, {"in_transaction", (getter)pysqlite_connection_get_in_transaction, (setter)0}, {"autocommit", (getter)get_autocommit, (setter)set_autocommit}, + {"__text_signature__", get_sig, (setter)0}, {NULL} }; From 6999d68d2878871493d85dc63599f3d44eada104 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Tue, 30 Apr 2024 13:16:52 -0700 Subject: [PATCH 023/220] gh-118218: Reuse return tuple in itertools.pairwise (GH-118219) --- Lib/test/test_itertools.py | 7 ++++ ...-04-24-07-45-08.gh-issue-118218.m1OHbN.rst | 1 + Modules/itertoolsmodule.c | 34 +++++++++++++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 95e67911db6a7f2..e243da309f03d8b 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1821,6 +1821,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject diff --git a/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst new file mode 100644 index 000000000000000..6d3c54cb485655d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-24-07-45-08.gh-issue-118218.m1OHbN.rst @@ -0,0 +1 @@ +Speed up :func:`itertools.pairwise` in the common case by up to 1.8x. diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 6ee447ef6a8cd6c..21ce3ecfad03547 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -270,6 +270,7 @@ typedef struct { PyObject_HEAD PyObject *it; PyObject *old; + PyObject *result; } pairwiseobject; /*[clinic input] @@ -301,6 +302,11 @@ pairwise_new_impl(PyTypeObject *type, PyObject *iterable) } po->it = it; po->old = NULL; + po->result = PyTuple_Pack(2, Py_None, Py_None); + if (po->result == NULL) { + Py_DECREF(po); + return NULL; + } return (PyObject *)po; } @@ -311,6 +317,7 @@ pairwise_dealloc(pairwiseobject *po) PyObject_GC_UnTrack(po); Py_XDECREF(po->it); Py_XDECREF(po->old); + Py_XDECREF(po->result); tp->tp_free(po); Py_DECREF(tp); } @@ -321,6 +328,7 @@ pairwise_traverse(pairwiseobject *po, visitproc visit, void *arg) Py_VISIT(Py_TYPE(po)); Py_VISIT(po->it); Py_VISIT(po->old); + Py_VISIT(po->result); return 0; } @@ -355,8 +363,30 @@ pairwise_next(pairwiseobject *po) Py_DECREF(old); return NULL; } - /* Future optimization: Reuse the result tuple as we do in enumerate() */ - result = PyTuple_Pack(2, old, new); + + result = po->result; + if (Py_REFCNT(result) == 1) { + Py_INCREF(result); + PyObject *last_old = PyTuple_GET_ITEM(result, 0); + PyObject *last_new = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + Py_DECREF(last_old); + Py_DECREF(last_new); + // bpo-42536: The GC may have untracked this result tuple. Since we're + // recycling it, make sure it's tracked again: + if (!_PyObject_GC_IS_TRACKED(result)) { + _PyObject_GC_TRACK(result); + } + } + else { + result = PyTuple_New(2); + if (result != NULL) { + PyTuple_SET_ITEM(result, 0, Py_NewRef(old)); + PyTuple_SET_ITEM(result, 1, Py_NewRef(new)); + } + } + Py_XSETREF(po->old, new); Py_DECREF(old); return result; From 587388ff22dc7cfa4b66722daf0d33cd804af9f2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 30 Apr 2024 22:29:48 +0200 Subject: [PATCH 024/220] gh-118124: Use static_assert() in Py_BUILD_ASSERT() on C11 (#118398) Use static_assert() in Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() on C11 and newer and C++11 and newer. Add tests to test_cext and test_cppext. --- Include/pymacro.h | 49 +++++++++++++------ Lib/test/test_cext/extension.c | 5 ++ Lib/test/test_cppext/extension.cpp | 4 ++ ...-04-29-17-44-15.gh-issue-118124.czQQ9G.rst | 3 ++ 4 files changed, 45 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst diff --git a/Include/pymacro.h b/Include/pymacro.h index cd6fc4eba9c2ed8..b388c2a4a663ce2 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -46,24 +46,41 @@ /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) -/* Assert a build-time dependency, as an expression. - - Your compile will fail if the condition isn't true, or can't be evaluated - by the compiler. This can be used in an expression: its value is 0. - - Example: - - #define foo_to_char(foo) \ - ((char *)(foo) \ - + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) - - Written by Rusty Russell, public domain, http://ccodearchive.net/ */ -#define Py_BUILD_ASSERT_EXPR(cond) \ +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L +# define Py_BUILD_ASSERT_EXPR(cond) \ + ((void)sizeof(struct { int dummy; _Static_assert(cond, #cond); }), \ + 0) +#else + /* Assert a build-time dependency, as an expression. + * + * Your compile will fail if the condition isn't true, or can't be evaluated + * by the compiler. This can be used in an expression: its value is 0. + * + * Example: + * + * #define foo_to_char(foo) \ + * ((char *)(foo) \ + * + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + * + * Written by Rusty Russell, public domain, http://ccodearchive.net/ + */ +# define Py_BUILD_ASSERT_EXPR(cond) \ (sizeof(char [1 - 2*!(cond)]) - 1) +#endif -#define Py_BUILD_ASSERT(cond) do { \ - (void)Py_BUILD_ASSERT_EXPR(cond); \ - } while(0) +#if ((defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L) \ + || (defined(__cplusplus) && __cplusplus >= 201103L)) + // Use static_assert() on C11 and newer +# define Py_BUILD_ASSERT(cond) \ + do { \ + static_assert((cond), #cond); \ + } while (0) +#else +# define Py_BUILD_ASSERT(cond) \ + do { \ + (void)Py_BUILD_ASSERT_EXPR(cond); \ + } while(0) +#endif /* Get the number of elements in a visible array diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 662abd4005e8d2b..eb23dbe20353bae 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -44,6 +44,11 @@ _testcext_exec(PyObject *module) return -1; } #endif + + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index a569c2251d1ad75..ab485b629b77884 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -225,6 +225,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() + Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); + assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); + return 0; } diff --git a/Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst b/Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst new file mode 100644 index 000000000000000..3deeb517eb9fcee --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-04-29-17-44-15.gh-issue-118124.czQQ9G.rst @@ -0,0 +1,3 @@ +Fix :c:macro:`Py_BUILD_ASSERT` and :c:macro:`Py_BUILD_ASSERT_EXPR` for +non-constant expressions: use ``static_assert()`` on C11 and newer. +Patch by Victor Stinner. From e93c39b47ea623dfaf61f80775ad4747b163efe5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 30 Apr 2024 22:32:55 +0200 Subject: [PATCH 025/220] gh-118422: Fix run_fileexflags() test (#118429) Don't test the undefined behavior of fileno() on a closed file, but use fstat() as a reliable test if the file was closed or not. --- Include/internal/pycore_fileutils.h | 3 ++ Modules/_testcapi/run.c | 13 ++++--- Python/fileutils.c | 49 +++++++++++++++++++++++++ Python/pylifecycle.c | 55 +++-------------------------- 4 files changed, 62 insertions(+), 58 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index bc8100b58e8ea32..13f86b01bbfe8ff 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -326,6 +326,9 @@ extern int _PyFile_Flush(PyObject *); extern int _Py_GetTicksPerSecond(long *ticks_per_second); #endif +// Export for '_testcapi' shared extension +PyAPI_FUNC(int) _Py_IsValidFD(int fd); + #ifdef __cplusplus } #endif diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index 4fd98b82d762ff4..21244d02967ebf2 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -1,5 +1,7 @@ +#define PYTESTCAPI_NEED_INTERNAL_API #include "parts.h" #include "util.h" +#include "pycore_fileutils.h" // _Py_IsValidFD() #include #include @@ -71,21 +73,18 @@ run_fileexflags(PyObject *mod, PyObject *pos_args) PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); return NULL; } + int fd = fileno(fp); result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); -#if defined(__linux__) || defined(MS_WINDOWS) || defined(__APPLE__) - /* The behavior of fileno() after fclose() is undefined, but it is - * the only practical way to check whether the file was closed. - * Only test this on the known platforms. */ - if (closeit && result && fileno(fp) >= 0) { + if (closeit && result && _Py_IsValidFD(fd)) { PyErr_SetString(PyExc_AssertionError, "File was not closed after excution"); Py_DECREF(result); fclose(fp); return NULL; } -#endif - if (!closeit && fileno(fp) < 0) { + + if (!closeit && !_Py_IsValidFD(fd)) { PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after excution"); Py_XDECREF(result); return NULL; diff --git a/Python/fileutils.c b/Python/fileutils.c index 54853ba2f75d9d0..e6a5391a3a28b56 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -3050,3 +3050,52 @@ _Py_GetTicksPerSecond(long *ticks_per_second) return 0; } #endif + + +/* Check if a file descriptor is valid or not. + Return 0 if the file descriptor is invalid, return non-zero otherwise. */ +int +_Py_IsValidFD(int fd) +{ +/* dup() is faster than fstat(): fstat() can require input/output operations, + whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python + startup. Problem: dup() doesn't check if the file descriptor is valid on + some platforms. + + fcntl(fd, F_GETFD) is even faster, because it only checks the process table. + It is preferred over dup() when available, since it cannot fail with the + "too many open files" error (EMFILE). + + bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other + side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with + EBADF. FreeBSD has similar issue (bpo-32849). + + Only use dup() on Linux where dup() is enough to detect invalid FD + (bpo-32849). +*/ + if (fd < 0) { + return 0; + } +#if defined(F_GETFD) && ( \ + defined(__linux__) || \ + defined(__APPLE__) || \ + (defined(__wasm__) && !defined(__wasi__))) + return fcntl(fd, F_GETFD) >= 0; +#elif defined(__linux__) + int fd2 = dup(fd); + if (fd2 >= 0) { + close(fd2); + } + return (fd2 >= 0); +#elif defined(MS_WINDOWS) + HANDLE hfile; + _Py_BEGIN_SUPPRESS_IPH + hfile = (HANDLE)_get_osfhandle(fd); + _Py_END_SUPPRESS_IPH + return (hfile != INVALID_HANDLE_VALUE + && GetFileType(hfile) != FILE_TYPE_UNKNOWN); +#else + struct stat st; + return (fstat(fd, &st) == 0); +#endif +} diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a672d8c7915ccd7..790398e944f58e8 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2410,54 +2410,6 @@ init_import_site(void) return _PyStatus_OK(); } -/* Check if a file descriptor is valid or not. - Return 0 if the file descriptor is invalid, return non-zero otherwise. */ -static int -is_valid_fd(int fd) -{ -/* dup() is faster than fstat(): fstat() can require input/output operations, - whereas dup() doesn't. There is a low risk of EMFILE/ENFILE at Python - startup. Problem: dup() doesn't check if the file descriptor is valid on - some platforms. - - fcntl(fd, F_GETFD) is even faster, because it only checks the process table. - It is preferred over dup() when available, since it cannot fail with the - "too many open files" error (EMFILE). - - bpo-30225: On macOS Tiger, when stdout is redirected to a pipe and the other - side of the pipe is closed, dup(1) succeed, whereas fstat(1, &st) fails with - EBADF. FreeBSD has similar issue (bpo-32849). - - Only use dup() on Linux where dup() is enough to detect invalid FD - (bpo-32849). -*/ - if (fd < 0) { - return 0; - } -#if defined(F_GETFD) && ( \ - defined(__linux__) || \ - defined(__APPLE__) || \ - defined(__wasm__)) - return fcntl(fd, F_GETFD) >= 0; -#elif defined(__linux__) - int fd2 = dup(fd); - if (fd2 >= 0) { - close(fd2); - } - return (fd2 >= 0); -#elif defined(MS_WINDOWS) - HANDLE hfile; - _Py_BEGIN_SUPPRESS_IPH - hfile = (HANDLE)_get_osfhandle(fd); - _Py_END_SUPPRESS_IPH - return (hfile != INVALID_HANDLE_VALUE - && GetFileType(hfile) != FILE_TYPE_UNKNOWN); -#else - struct stat st; - return (fstat(fd, &st) == 0); -#endif -} - /* returns Py_None if the fd is not valid */ static PyObject* create_stdio(const PyConfig *config, PyObject* io, @@ -2471,8 +2423,9 @@ create_stdio(const PyConfig *config, PyObject* io, int buffering, isatty; const int buffered_stdio = config->buffered_stdio; - if (!is_valid_fd(fd)) + if (!_Py_IsValidFD(fd)) { Py_RETURN_NONE; + } /* stdin is always opened in buffered mode, first because it shouldn't make a difference in common use cases, second because TextIOWrapper @@ -2588,9 +2541,9 @@ create_stdio(const PyConfig *config, PyObject* io, Py_XDECREF(text); Py_XDECREF(raw); - if (PyErr_ExceptionMatches(PyExc_OSError) && !is_valid_fd(fd)) { + if (PyErr_ExceptionMatches(PyExc_OSError) && !_Py_IsValidFD(fd)) { /* Issue #24891: the file descriptor was closed after the first - is_valid_fd() check was called. Ignore the OSError and set the + _Py_IsValidFD() check was called. Ignore the OSError and set the stream to None. */ PyErr_Clear(); Py_RETURN_NONE; From dc6b12d1b2ea26bb0323d932fa7b1a337eaca562 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 1 May 2024 04:46:13 +0800 Subject: [PATCH 026/220] gh-117139: Add header for tagged pointers (GH-118330) --------- Co-authored-by: Sam Gross <655866+colesbury@users.noreply.github.com> --- Include/internal/pycore_stackref.h | 195 +++++++++++++++++++++++++++++ Makefile.pre.in | 1 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + 4 files changed, 200 insertions(+) create mode 100644 Include/internal/pycore_stackref.h diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h new file mode 100644 index 000000000000000..fd929cd4873a8bd --- /dev/null +++ b/Include/internal/pycore_stackref.h @@ -0,0 +1,195 @@ +#ifndef Py_INTERNAL_STACKREF_H +#define Py_INTERNAL_STACKREF_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +#include + +typedef union { + uintptr_t bits; +} _PyStackRef; + +static const _PyStackRef Py_STACKREF_NULL = { .bits = 0 }; + +#define Py_TAG_DEFERRED (1) + +// Gets a PyObject * from a _PyStackRef +#if defined(Py_GIL_DISABLED) +static inline PyObject * +PyStackRef_Get(_PyStackRef tagged) +{ + PyObject *cleared = ((PyObject *)((tagged).bits & (~Py_TAG_DEFERRED))); + return cleared; +} +#else +# define PyStackRef_Get(tagged) ((PyObject *)((tagged).bits)) +#endif + +// Converts a PyObject * to a PyStackRef, stealing the reference. +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_StealRef(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + return ((_PyStackRef){.bits = ((uintptr_t)(obj))}); +} +# define PyStackRef_StealRef(obj) _PyStackRef_StealRef(_PyObject_CAST(obj)) +#else +# define PyStackRef_StealRef(obj) ((_PyStackRef){.bits = ((uintptr_t)(obj))}) +#endif + +// Converts a PyObject * to a PyStackRef, with a new reference +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_NewRefDeferred(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + assert(obj != NULL); + if (_PyObject_HasDeferredRefcount(obj)) { + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; + } + else { + return (_PyStackRef){ .bits = (uintptr_t)Py_NewRef(obj) }; + } +} +# define PyStackRef_NewRefDeferred(obj) _PyStackRef_NewRefDeferred(_PyObject_CAST(obj)) +#else +# define PyStackRef_NewRefDeferred(obj) PyStackRef_NewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) +#endif + +#if defined(Py_GIL_DISABLED) +static inline _PyStackRef +_PyStackRef_XNewRefDeferred(PyObject *obj) +{ + // Make sure we don't take an already tagged value. + assert(((uintptr_t)obj & Py_TAG_DEFERRED) == 0); + if (obj == NULL) { + return Py_STACKREF_NULL; + } + return _PyStackRef_NewRefDeferred(obj); +} +# define PyStackRef_XNewRefDeferred(obj) _PyStackRef_XNewRefDeferred(_PyObject_CAST(obj)) +#else +# define PyStackRef_XNewRefDeferred(obj) PyStackRef_XNewRef(((_PyStackRef){.bits = ((uintptr_t)(obj))})) +#endif + +// Converts a PyStackRef back to a PyObject *. +#if defined(Py_GIL_DISABLED) +static inline PyObject * +PyStackRef_StealObject(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); + return Py_NewRef(PyStackRef_Get(tagged)); + } + return PyStackRef_Get(tagged); +} +#else +# define PyStackRef_StealObject(tagged) PyStackRef_Get(tagged) +#endif + +static inline void +_Py_untag_stack_borrowed(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_Get(src[i]); + } +} + +static inline void +_Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length) +{ + for (size_t i = 0; i < length; i++) { + dst[i] = PyStackRef_StealObject(src[i]); + } +} + + +#define PyStackRef_XSETREF(dst, src) \ + do { \ + _PyStackRef *_tmp_dst_ptr = &(dst) \ + _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + PyStackRef_XDECREF(_tmp_old_dst); \ + } while (0) + +#define PyStackRef_SETREF(dst, src) \ + do { \ + _PyStackRef *_tmp_dst_ptr = &(dst); \ + _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ + *_tmp_dst_ptr = (src); \ + PyStackRef_DECREF(_tmp_old_dst); \ + } while (0) + +#define PyStackRef_CLEAR(op) \ + do { \ + _PyStackRef *_tmp_op_ptr = &(op); \ + _PyStackRef _tmp_old_op = (*_tmp_op_ptr); \ + if (_tmp_old_op.bits != Py_STACKREF_NULL.bits) { \ + *_tmp_op_ptr = Py_STACKREF_NULL; \ + PyStackRef_DECREF(_tmp_old_op); \ + } \ + } while (0) + +#if defined(Py_GIL_DISABLED) +static inline void +PyStackRef_DECREF(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + return; + } + Py_DECREF(PyStackRef_Get(tagged)); +} +#else +# define PyStackRef_DECREF(op) Py_DECREF(PyStackRef_Get(op)) +#endif + +#if defined(Py_GIL_DISABLED) +static inline void +PyStackRef_INCREF(_PyStackRef tagged) +{ + if ((tagged.bits & Py_TAG_DEFERRED) == Py_TAG_DEFERRED) { + assert(_PyObject_HasDeferredRefcount(PyStackRef_Get(tagged))); + return; + } + Py_INCREF(PyStackRef_Get(tagged)); +} +#else +# define PyStackRef_INCREF(op) Py_INCREF(PyStackRef_Get(op)) +#endif + +static inline void +PyStackRef_XDECREF(_PyStackRef op) +{ + if (op.bits != Py_STACKREF_NULL.bits) { + PyStackRef_DECREF(op); + } +} + +static inline _PyStackRef +PyStackRef_NewRef(_PyStackRef obj) +{ + PyStackRef_INCREF(obj); + return obj; +} + +static inline _PyStackRef +PyStackRef_XNewRef(_PyStackRef obj) +{ + if (obj.bits == Py_STACKREF_NULL.bits) { + return obj; + } + return PyStackRef_NewRef(obj); +} + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_STACKREF_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 4e4d01d14ee3cbc..e69d1fe6e2dd14e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1225,6 +1225,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/internal/pycore_structseq.h \ $(srcdir)/Include/internal/pycore_symtable.h \ $(srcdir)/Include/internal/pycore_sysmodule.h \ + $(srcdir)/Include/internal/pycore_stackref.h \ $(srcdir)/Include/internal/pycore_time.h \ $(srcdir)/Include/internal/pycore_token.h \ $(srcdir)/Include/internal/pycore_traceback.h \ diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 25d52945c1c3300..4cb3e0d32270927 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -291,6 +291,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 4b1f9aa6538562e..0b858cf1eb46a8a 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -789,6 +789,9 @@ Include\internal + + Include\internal + Include\internal From 7fabcc727dee52a3e0dfe4f903ad414e93cf2dc9 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Wed, 1 May 2024 05:51:59 +0800 Subject: [PATCH 027/220] gh-117657: Don't specialize RESUME_CHECK when specialization is disabled (GH-118349) --- Python/bytecodes.c | 2 ++ Python/generated_cases.c.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5bb7e1211385a5e..18837aef74d78e2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -163,7 +163,9 @@ dummy_func( if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } + #if ENABLE_SPECIALIZATION FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index a5bb29385844d8c..1444f5cdebba4b6 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4955,7 +4955,9 @@ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } + #if ENABLE_SPECIALIZATION FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ } DISPATCH(); } From 9c468e2c5dffb6fa9811fd16e70fa0463bdfce5f Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 1 May 2024 07:32:37 +0800 Subject: [PATCH 028/220] gh-118201 - Disable the flaky POSIX test_confstr test on iOS (GH-118452) --- Lib/test/test_posix.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 1d22869046fd124..7e5f04c22bd6d31 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -564,6 +564,7 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_confstr(self): self.assertRaises(ValueError, posix.confstr, "CS_garbage") self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) From 7d83f7bcc484145596bae1ff015fed0762da345d Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 30 Apr 2024 18:26:34 -0700 Subject: [PATCH 029/220] gh-118335: Configure Tier 2 interpreter at build time (#118339) The code for Tier 2 is now only compiled when configured with `--enable-experimental-jit[=yes|interpreter]`. We drop support for `PYTHON_UOPS` and -`Xuops`, but you can disable the interpreter or JIT at runtime by setting `PYTHON_JIT=0`. You can also build it without enabling it by default using `--enable-experimental-jit=yes-off`; enable with `PYTHON_JIT=1`. On Windows, the `build.bat` script supports `--experimental-jit`, `--experimental-jit-off`, `--experimental-interpreter`. In the C code, `_Py_JIT` is defined as before when the JIT is enabled; the new variable `_Py_TIER2` is defined when the JIT *or* the interpreter is enabled. It is actually a bitmask: 1: JIT; 2: default-off; 4: interpreter. --- Doc/whatsnew/3.13.rst | 30 ++++++++++++++----- Include/internal/pycore_opcode_metadata.h | 4 +-- Include/internal/pycore_uop_metadata.h | 2 +- Lib/dis.py | 2 +- Lib/test/support/__init__.py | 8 ++--- Lib/test/test_capi/test_opt.py | 8 +++++ Lib/test/test_monitoring.py | 10 ++++--- Lib/test/test_opcache.py | 2 ++ Lib/test/test_optimizer.py | 2 ++ Lib/test/test_weakref.py | 2 ++ ...-04-26-14-06-18.gh-issue-118335.SRFsxO.rst | 4 +++ Modules/_opcode.c | 6 ++++ Modules/_testinternalcapi.c | 13 +++++++- Objects/codeobject.c | 6 ++++ Objects/object.c | 4 +++ PCbuild/_testinternalcapi.vcxproj | 6 ++++ PCbuild/build.bat | 11 +++++-- PCbuild/pythoncore.vcxproj | 1 + Python/bytecodes.c | 6 ++++ Python/ceval.c | 5 +++- Python/generated_cases.c.h | 6 ++++ Python/instrumentation.c | 6 ++++ Python/optimizer.c | 4 +++ Python/optimizer_analysis.c | 4 +++ Python/optimizer_symbols.c | 3 ++ Python/pylifecycle.c | 28 +++++++++-------- Python/pystate.c | 6 ++++ Python/sysmodule.c | 2 ++ Tools/cases_generator/analyzer.py | 1 + Tools/jit/README.md | 6 ++-- configure | 12 ++++++-- configure.ac | 13 ++++++-- 32 files changed, 181 insertions(+), 42 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a9a7e485dd2bac4..ee50effd662f12e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -888,7 +888,7 @@ Experimental JIT Compiler ========================= When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which can speed up some Python programs. +a just-in-time compiler is added which may speed up some Python programs. The internal architecture is roughly as follows. @@ -905,19 +905,35 @@ The internal architecture is roughly as follows. before it is interpreted or translated to machine code. * There is a Tier 2 interpreter, but it is mostly intended for debugging - the earlier stages of the optimization pipeline. If the JIT is not - enabled, the Tier 2 interpreter can be invoked by passing Python the - ``-X uops`` option or by setting the ``PYTHON_UOPS`` environment - variable to ``1``. + the earlier stages of the optimization pipeline. + The Tier 2 interpreter can be enabled by configuring Python + with ``--enable-experimental-jit=interpreter``. -* When the ``--enable-experimental-jit`` option is used, the optimized +* When the JIT is enabled, the optimized Tier 2 IR is translated to machine code, which is then executed. - This does not require additional runtime options. * The machine code translation process uses an architecture called *copy-and-patch*. It has no runtime dependencies, but there is a new build-time dependency on LLVM. +The ``--enable-experimental-jit`` flag has the following optional values: + +* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. + +* ``yes`` (default if the flag is present without optional value) + -- Enable the JIT. To disable the JIT at runtime, + pass the environment variable ``PYTHON_JIT=0``. + +* ``yes-off`` -- Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable + ``PYTHON_JIT=1``. + +* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with + ``PYTHON_JIT=0``. + +(On Windows, use ``PCbuild/build.bat --enable-jit`` to enable the JIT.) + See :pep:`744` for more details. (JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 2ccc548ca6c5fd9..808badea90d8eda 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -981,7 +981,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [BUILD_SLICE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_STRING] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BUILD_TUPLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, - [CACHE] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [CACHE] = { true, INSTR_FMT_IX, 0 }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, @@ -1121,7 +1121,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [PUSH_NULL] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [RAISE_VARARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [RERAISE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, - [RESERVED] = { true, INSTR_FMT_IX, HAS_ESCAPES_FLAG }, + [RESERVED] = { true, INSTR_FMT_IX, 0 }, [RESUME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [RESUME_CHECK] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, [RETURN_CONST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_CONST_FLAG }, diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index a84212c1ec0b69e..5697b13d1fc51d2 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -249,7 +249,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_COLD_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_DYNAMIC_EXIT] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, - [_FATAL_ERROR] = HAS_ESCAPES_FLAG, + [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, [_SIDE_EXIT] = 0, diff --git a/Lib/dis.py b/Lib/dis.py index 111d624fc259c55..76934eb00e63f0a 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -216,7 +216,7 @@ def _get_code_array(co, adaptive): if op == ENTER_EXECUTOR: try: ex = get_executor(co, i) - except ValueError: + except (ValueError, RuntimeError): ex = None if ex: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 15f654302b17935..52573e665a1273f 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2539,17 +2539,17 @@ def exceeds_recursion_limit(): # Decorator to disable optimizer while a function run def without_optimizer(func): try: - import _testinternalcapi + from _testinternalcapi import get_optimizer, set_optimizer except ImportError: return func @functools.wraps(func) def wrapper(*args, **kwargs): - save_opt = _testinternalcapi.get_optimizer() + save_opt = get_optimizer() try: - _testinternalcapi.set_optimizer(None) + set_optimizer(None) return func(*args, **kwargs) finally: - _testinternalcapi.set_optimizer(save_opt) + set_optimizer(save_opt) return wrapper diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index c798b3436266777..6e5b626e93291ae 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -34,6 +34,8 @@ def clear_executors(func): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestOptimizerAPI(unittest.TestCase): def test_new_counter_optimizer_dealloc(self): @@ -136,6 +138,8 @@ def get_opnames(ex): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): def setUp(self): @@ -215,6 +219,8 @@ def f(): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUops(unittest.TestCase): @@ -579,6 +585,8 @@ def testfunc(n): @requires_specialization +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 11c61bc2e0688d3..a9140d4d3dd7434 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -1831,15 +1831,17 @@ class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): _testinternalcapi = import_module("_testinternalcapi") - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(opt) + if hasattr(_testinternalcapi, "get_optimizer"): + self.old_opt = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(opt) super(TestOptimizer, self).setUp() def tearDown(self): super(TestOptimizer, self).tearDown() import _testinternalcapi - _testinternalcapi.set_optimizer(self.old_opt) + if hasattr(_testinternalcapi, "get_optimizer"): + _testinternalcapi.set_optimizer(self.old_opt) def test_for_loop(self): def test_func(x): diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index f4e954fd02148dd..92a34113bc03833 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -16,6 +16,8 @@ def disabling_optimizer(func): def wrapper(*args, **kwargs): + if not hasattr(_testinternalcapi, "get_optimizer"): + return func(*args, **kwargs) old_opt = _testinternalcapi.get_optimizer() _testinternalcapi.set_optimizer(None) try: diff --git a/Lib/test/test_optimizer.py b/Lib/test/test_optimizer.py index 899a45073173347..fac4d1a4ab44c47 100644 --- a/Lib/test/test_optimizer.py +++ b/Lib/test/test_optimizer.py @@ -80,6 +80,8 @@ def func(x=0): class TestOptimizerSymbols(unittest.TestCase): + @unittest.skipUnless(hasattr(_testinternalcapi, "uop_symbols_test"), + "requires _testinternalcapi.uop_symbols_test") def test_optimizer_symbols(self): _testinternalcapi.uop_symbols_test() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index df90647baee31e6..16da24d7805b566 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -17,6 +17,7 @@ from test.support import gc_collect from test.support import import_helper from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -960,6 +961,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst new file mode 100644 index 000000000000000..e13edbbbe528eb2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst @@ -0,0 +1,4 @@ +Change how to use the tier 2 interpreter. Instead of running Python with +``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this +choice is now made at build time by configuring with +``--enable-experimental-jit=interpreter``. diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 5350adb456b8595..85e0ffec900e89f 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -367,7 +367,13 @@ _opcode_get_executor_impl(PyObject *module, PyObject *code, int offset) Py_TYPE(code)->tp_name); return NULL; } +#ifdef _Py_TIER2 return (PyObject *)PyUnstable_GetExecutor((PyCodeObject *)code, offset); +#else + PyErr_Format(PyExc_RuntimeError, + "Executors are not available in this build"); + return NULL; +#endif } static PyMethodDef diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 99e80baa223e8f7..f7952a119f0bd9c 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -985,6 +985,8 @@ get_co_framesize(PyObject *self, PyObject *arg) return PyLong_FromLong(code->co_framesize); } +#ifdef _Py_TIER2 + static PyObject * new_counter_optimizer(PyObject *self, PyObject *arg) { @@ -1012,7 +1014,10 @@ set_optimizer(PyObject *self, PyObject *opt) static PyObject * get_optimizer(PyObject *self, PyObject *Py_UNUSED(ignored)) { - PyObject *opt = (PyObject *)PyUnstable_GetOptimizer(); + PyObject *opt = NULL; +#ifdef _Py_TIER2 + opt = (PyObject *)PyUnstable_GetOptimizer(); +#endif if (opt == NULL) { Py_RETURN_NONE; } @@ -1045,6 +1050,8 @@ invalidate_executors(PyObject *self, PyObject *obj) Py_RETURN_NONE; } +#endif + static int _pending_callback(void *arg) { /* we assume the argument is callable object to which we own a reference */ @@ -2020,12 +2027,14 @@ static PyMethodDef module_functions[] = { {"iframe_getline", iframe_getline, METH_O, NULL}, {"iframe_getlasti", iframe_getlasti, METH_O, NULL}, {"get_co_framesize", get_co_framesize, METH_O, NULL}, +#ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, +#endif {"pending_threadfunc", _PyCFunction_CAST(pending_threadfunc), METH_VARARGS | METH_KEYWORDS}, {"pending_identify", pending_identify, METH_VARARGS, NULL}, @@ -2072,7 +2081,9 @@ static PyMethodDef module_functions[] = { {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS}, +#ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 014632962bfcf36..605167c5c0bd3aa 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1496,6 +1496,8 @@ PyCode_GetFreevars(PyCodeObject *code) return _PyCode_GetFreevars(code); } +#ifdef _Py_TIER2 + static void clear_executors(PyCodeObject *co) { @@ -1515,6 +1517,8 @@ _PyCode_Clear_Executors(PyCodeObject *code) clear_executors(code); } +#endif + static void deopt_code(PyCodeObject *code, _Py_CODEUNIT *instructions) { @@ -1739,9 +1743,11 @@ code_dealloc(PyCodeObject *co) PyMem_Free(co_extra); } +#ifdef _Py_TIER2 if (co->co_executors != NULL) { clear_executors(co); } +#endif Py_XDECREF(co->co_consts); Py_XDECREF(co->co_names); diff --git a/Objects/object.c b/Objects/object.c index 8d856939254080f..45310a6c22d6774 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2281,9 +2281,11 @@ static PyTypeObject* static_types[] = { &_PyBufferWrapper_Type, &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, +#ifdef _Py_TIER2 &_PyCounterExecutor_Type, &_PyCounterOptimizer_Type, &_PyDefaultOptimizer_Type, +#endif &_Py_GenericAliasIterType, &_PyHamtItems_Type, &_PyHamtKeys_Type, @@ -2304,8 +2306,10 @@ static PyTypeObject* static_types[] = { &_PyPositionsIterator, &_PyUnicodeASCIIIter_Type, &_PyUnion_Type, +#ifdef _Py_TIER2 &_PyUOpExecutor_Type, &_PyUOpOptimizer_Type, +#endif &_PyWeakref_CallableProxyType, &_PyWeakref_ProxyType, &_PyWeakref_RefType, diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index a825cac9138674a..d4cd8ad1a46f24a 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -108,6 +108,12 @@ false + + + _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + + diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 83b50db44670337..43a99aa684483f3 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -36,7 +36,9 @@ echo. overrides -c and -d echo. --disable-gil Enable experimental support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. -echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit Enable the experimental just-in-time compiler. +echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). +echo. --experimental-interpreter Enable the experimental Tier 2 interpreter. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -66,6 +68,7 @@ set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo +set UseTIER2=0 :CheckOpts if "%~1"=="-h" goto Usage @@ -86,7 +89,10 @@ if "%~1"=="--disable-gil" (set UseDisableGil=true) & shift & goto CheckOpts if "%~1"=="--test-marker" (set UseTestMarker=true) & shift & goto CheckOpts if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts -if "%~1"=="--experimental-jit" (set UseJIT=true) & shift & goto CheckOpts +if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts +if "%~1"=="--experimental-interpreter" (set UseTIER2=4) & shift & goto CheckOpts +if "%~1"=="--experimental-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this @@ -179,6 +185,7 @@ echo on /p:DisableGil=%UseDisableGil%^ /p:UseTestMarker=%UseTestMarker% %GITProperty%^ /p:UseJIT=%UseJIT%^ + /p:UseTIER2=%UseTIER2%^ %1 %2 %3 %4 %5 %6 %7 %8 %9 @echo off diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 4cb3e0d32270927..a24667dc74064aa 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -105,6 +105,7 @@ _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) _Py_JIT;%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 18837aef74d78e2..28766d6d97cc5f9 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2363,6 +2363,7 @@ dummy_func( CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); + #ifdef _Py_TIER2 #if ENABLE_SPECIALIZATION _Py_BackoffCounter counter = this_instr[1].counter; if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { @@ -2388,6 +2389,7 @@ dummy_func( ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ } pseudo(JUMP) = { @@ -2401,6 +2403,7 @@ dummy_func( }; tier1 inst(ENTER_EXECUTOR, (--)) { + #ifdef _Py_TIER2 int prevoparg = oparg; CHECK_EVAL_BREAKER(); if (this_instr->op.code != ENTER_EXECUTOR || @@ -2418,6 +2421,9 @@ dummy_func( tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ } replaced op(_POP_JUMP_IF_FALSE, (cond -- )) { diff --git a/Python/ceval.c b/Python/ceval.c index 8b23bc6bcbeb396..59498bc826e9412 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -755,7 +755,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int _Py_CODEUNIT *next_instr; PyObject **stack_pointer; -#ifndef _Py_JIT +#if defined(_Py_TIER2) && !defined(_Py_JIT) /* Tier 2 interpreter state */ _PyExecutorObject *current_executor = NULL; const _PyUOpInstruction *next_uop = NULL; @@ -959,6 +959,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int goto error; +#ifdef _Py_TIER2 // Tier 2 is also here! enter_tier_two: @@ -1113,6 +1114,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #endif // _Py_JIT +#endif // _Py_TIER2 + } #if defined(__GNUC__) diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 1444f5cdebba4b6..602cce4beb1e28a 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2492,6 +2492,7 @@ (void)this_instr; next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); + #ifdef _Py_TIER2 int prevoparg = oparg; CHECK_EVAL_BREAKER(); if (this_instr->op.code != ENTER_EXECUTOR || @@ -2508,6 +2509,9 @@ tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); + #else + Py_FatalError("ENTER_EXECUTOR is not supported in this build"); + #endif /* _Py_TIER2 */ DISPATCH(); } @@ -3432,6 +3436,7 @@ CHECK_EVAL_BREAKER(); assert(oparg <= INSTR_OFFSET()); JUMPBY(-oparg); + #ifdef _Py_TIER2 #if ENABLE_SPECIALIZATION _Py_BackoffCounter counter = this_instr[1].counter; if (backoff_counter_triggers(counter) && this_instr->op.code == JUMP_BACKWARD) { @@ -3457,6 +3462,7 @@ ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); } #endif /* ENABLE_SPECIALIZATION */ + #endif /* _Py_TIER2 */ DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index ce97c3add7d0ea3..5614f4867a390d4 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1705,10 +1705,12 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) ); return 0; } +#ifdef _Py_TIER2 if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); } _Py_Executors_InvalidateDependency(interp, code, 1); +#endif int code_len = (int)Py_SIZE(code); /* Exit early to avoid creating instrumentation * data for potential statically allocated code @@ -1946,7 +1948,9 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) goto done; } set_global_version(tstate, new_version); +#ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 1); +#endif res = instrument_all_executing_code_objects(interp); done: _PyEval_StartTheWorld(interp); @@ -1986,7 +1990,9 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; } +#ifdef _Py_TIER2 _Py_Executors_InvalidateDependency(interp, code, 1); +#endif res = instrument_lock_held(code, interp); diff --git a/Python/optimizer.c b/Python/optimizer.c index a9a35fc902018ab..a5e7430c46450ef 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1,3 +1,5 @@ +#ifdef _Py_TIER2 + #include "Python.h" #include "opcode.h" #include "pycore_interp.h" @@ -1622,3 +1624,5 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) } } } + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 9315d7228b57323..842b2e489239af0 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -1,3 +1,5 @@ +#ifdef _Py_TIER2 + /* * This file contains the support code for CPython's uops optimizer. * It also performs some simple optimizations. @@ -603,3 +605,5 @@ _Py_uop_analyze_and_optimize( OPT_STAT_INC(optimizer_successes); return length; } + +#endif /* _Py_TIER2 */ diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 204599b08766c3a..d52f490853c006a 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -1,3 +1,4 @@ +#ifdef _Py_TIER2 #include "Python.h" @@ -506,3 +507,5 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) Py_XDECREF(val_43); return NULL; } + +#endif /* _Py_TIER2 */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 790398e944f58e8..7726ccc979dbcc0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -624,9 +624,11 @@ static int builtins_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef _Py_TIER2 if (interp->rare_events.builtin_dict < _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS) { _Py_Executors_InvalidateAll(interp, 1); } +#endif RARE_EVENT_INTERP_INC(interp, builtin_dict); return 0; } @@ -1272,30 +1274,30 @@ init_interp_main(PyThreadState *tstate) } // Turn on experimental tier 2 (uops-based) optimizer + // This is also needed when the JIT is enabled +#ifdef _Py_TIER2 if (is_main_interp) { -#ifndef _Py_JIT - // No JIT, maybe use the tier two interpreter: - char *envvar = Py_GETENV("PYTHON_UOPS"); - int enabled = envvar != NULL && *envvar > '0'; - if (_Py_get_xoption(&config->xoptions, L"uops") != NULL) { - enabled = 1; + int enabled = 1; +#if _Py_TIER2 & 2 + enabled = 0; +#endif + char *env = Py_GETENV("PYTHON_JIT"); + if (env && *env != '\0') { + // PYTHON_JIT=0|1 overrides the default + enabled = *env != '0'; } if (enabled) { -#else - // Always enable tier two for JIT builds (ignoring the environment - // variable and command-line option above): - if (true) { -#endif PyObject *opt = PyUnstable_Optimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } if (PyUnstable_SetOptimizer((_PyOptimizerObject *)opt)) { - return _PyStatus_ERR("can't initialize optimizer"); + return _PyStatus_ERR("can't install optimizer"); } Py_DECREF(opt); } } +#endif if (!is_main_interp) { // The main interpreter is handled in Py_Main(), for now. @@ -1655,10 +1657,12 @@ finalize_modules(PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; +#ifdef _Py_TIER2 // Invalidate all executors and turn off tier 2 optimizer _Py_Executors_InvalidateAll(interp, 0); _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); Py_XDECREF(old); +#endif // Stop watching __builtin__ modifications PyDict_Unwatch(0, interp->builtins); diff --git a/Python/pystate.c b/Python/pystate.c index 9d7b73b3a071e12..3d6e76e88bd7310 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -653,8 +653,10 @@ init_interpreter(PyInterpreterState *interp, } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; +#ifdef _Py_TIER2 (void)_Py_SetOptimizer(interp, NULL); interp->executor_list_head = NULL; +#endif if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); @@ -806,9 +808,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) tstate->_status.cleared = 0; } +#ifdef _Py_TIER2 _PyOptimizerObject *old = _Py_SetOptimizer(interp, NULL); assert(old != NULL); Py_DECREF(old); +#endif /* It is possible that any of the objects below have a finalizer that runs Python code or otherwise relies on a thread state @@ -2821,9 +2825,11 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, if (eval_frame == interp->eval_frame) { return; } +#ifdef _Py_TIER2 if (eval_frame != NULL) { _Py_Executors_InvalidateAll(interp, 1); } +#endif RARE_EVENT_INC(set_eval_frame_func); interp->eval_frame = eval_frame; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 7af363678e8e867..726051521cf574e 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2165,8 +2165,10 @@ static PyObject * sys__clear_internal_caches_impl(PyObject *module) /*[clinic end generated code: output=0ee128670a4966d6 input=253e741ca744f6e8]*/ { +#ifdef _Py_TIER2 PyInterpreterState *interp = _PyInterpreterState_GET(); _Py_Executors_InvalidateAll(interp, 0); +#endif PyType_ClearCache(); Py_RETURN_NONE; } diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 18cefa083288042..fdb635486b95317 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -411,6 +411,7 @@ def has_error_without_pop(op: parser.InstDef) -> bool: "PyCell_New", "PyFloat_AS_DOUBLE", "_PyFrame_PushUnchecked", + "Py_FatalError", ) ESCAPING_FUNCTIONS = ( diff --git a/Tools/jit/README.md b/Tools/jit/README.md index 7b33f99d23f75d0..0f5aa9ce656bc98 100644 --- a/Tools/jit/README.md +++ b/Tools/jit/README.md @@ -7,18 +7,18 @@ This version of CPython can be built with an experimental just-in-time compiler. The JIT compiler does not require end users to install any third-party dependencies, but part of it must be *built* using LLVM[^why-llvm]. You are *not* required to build the rest of CPython using LLVM, or even the same version of LLVM (in fact, this is uncommon). -LLVM version 16 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-16`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. +LLVM version 18 is required. Both `clang` and `llvm-readobj` need to be installed and discoverable (version suffixes, like `clang-18`, are okay). It's highly recommended that you also have `llvm-objdump` available, since this allows the build script to dump human-readable assembly for the generated code. It's easy to install all of the required tools: ### Linux -Install LLVM 16 on Ubuntu/Debian: +Install LLVM 18 on Ubuntu/Debian: ```sh wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh -sudo ./llvm.sh 16 +sudo ./llvm.sh 18 ``` ### macOS diff --git a/configure b/configure index e1d0f36463c44b0..01c00d2198548f5 100755 --- a/configure +++ b/configure @@ -1818,7 +1818,7 @@ Optional Features: --disable-gil enable experimental support for running without the GIL (default is no) --enable-pystats enable internal statistics gathering (default is no) - --enable-experimental-jit + --enable-experimental-jit[=no|yes|yes-off|interpreter] build the experimental just-in-time compiler (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) @@ -8229,11 +8229,19 @@ else $as_nop enable_experimental_jit=no fi +case $enable_experimental_jit in + no) enable_experimental_jit=no ;; + yes) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=1" ;; + yes-off) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=3" ;; + interpreter) enable_experimental_jit="-D_Py_TIER2=4" ;; + interpreter-off) enable_experimental_jit="-D_Py_TIER2=6" ;; # Secret option + *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; +esac if test "x$enable_experimental_jit" = xno then : else $as_nop - as_fn_append CFLAGS_NODIST " -D_Py_JIT" + as_fn_append CFLAGS_NODIST " $enable_experimental_jit" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue diff --git a/configure.ac b/configure.ac index 7681ea3d3fe3c69..ae65efeac55366a 100644 --- a/configure.ac +++ b/configure.ac @@ -1768,14 +1768,23 @@ fi # Check for --enable-experimental-jit: AC_MSG_CHECKING([for --enable-experimental-jit]) AC_ARG_ENABLE([experimental-jit], - [AS_HELP_STRING([--enable-experimental-jit], + [AS_HELP_STRING([--enable-experimental-jit@<:@=no|yes|yes-off|interpreter@:>@], [build the experimental just-in-time compiler (default is no)])], [], [enable_experimental_jit=no]) +case $enable_experimental_jit in + no) enable_experimental_jit=no ;; + yes) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=1" ;; + yes-off) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=3" ;; + interpreter) enable_experimental_jit="-D_Py_TIER2=4" ;; + interpreter-off) enable_experimental_jit="-D_Py_TIER2=6" ;; # Secret option + *) AC_MSG_ERROR( + [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; +esac AS_VAR_IF([enable_experimental_jit], [no], [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" -D_Py_JIT"]) + [AS_VAR_APPEND([CFLAGS_NODIST], [" $enable_experimental_jit"]) AS_VAR_SET([REGEN_JIT_COMMAND], ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) From 21336aa12762ffe33bac83c7bd7992fcf42eee10 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 1 May 2024 10:31:00 +0800 Subject: [PATCH 030/220] gh-118201: Accomodate flaky behavior of `os.sysconf` on iOS (GH-118453) --- Lib/test/support/os_helper.py | 3 ++- Lib/test/test_os.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 8071c248b9b67ef..891405943b78c50 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -632,7 +632,8 @@ def fd_count(): if hasattr(os, 'sysconf'): try: MAXFD = os.sysconf("SC_OPEN_MAX") - except OSError: + except (OSError, ValueError): + # gh-118201: ValueError is raised intermittently on iOS pass old_modes = None diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 6a34f48f7873eeb..eaa676673f8af07 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2365,6 +2365,7 @@ def test_fchown(self): support.is_emscripten or support.is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(support.is_apple_mobile, "gh-118201: Test is flaky on iOS") def test_fpathconf(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") From 2520eed0a529be3815f70c43e1a5006deeee5596 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 1 May 2024 07:36:45 +0100 Subject: [PATCH 031/220] gh-116622: Add Android testbed (GH-117878) Add code and config for a minimal Android app, and instructions to build and run it. Improve Android build instructions in general. Add a tool subcommand to download the Gradle wrapper (with its binary blob). Android studio must be downloaded manually (due to the license). --- .github/CODEOWNERS | 12 ++ Android/README.md | 43 ++++- Android/android.py | 45 +++++- Android/testbed/.gitignore | 21 +++ Android/testbed/app/.gitignore | 1 + Android/testbed/app/build.gradle.kts | 129 +++++++++++++++ .../testbed/app/src/main/AndroidManifest.xml | 20 +++ Android/testbed/app/src/main/c/CMakeLists.txt | 9 ++ .../testbed/app/src/main/c/main_activity.c | 147 ++++++++++++++++++ .../java/org/python/testbed/MainActivity.kt | 61 ++++++++ Android/testbed/app/src/main/python/main.py | 17 ++ .../main/res/drawable-xxhdpi/ic_launcher.png | Bin 0 -> 3110 bytes .../app/src/main/res/layout/activity_main.xml | 19 +++ .../app/src/main/res/values/strings.xml | 3 + Android/testbed/build.gradle.kts | 5 + Android/testbed/gradle.properties | 23 +++ .../gradle/wrapper/gradle-wrapper.properties | 6 + Android/testbed/settings.gradle.kts | 18 +++ ...-04-14-19-35-35.gh-issue-116622.8lpX-7.rst | 1 + 19 files changed, 570 insertions(+), 10 deletions(-) create mode 100644 Android/testbed/.gitignore create mode 100644 Android/testbed/app/.gitignore create mode 100644 Android/testbed/app/build.gradle.kts create mode 100644 Android/testbed/app/src/main/AndroidManifest.xml create mode 100644 Android/testbed/app/src/main/c/CMakeLists.txt create mode 100644 Android/testbed/app/src/main/c/main_activity.c create mode 100644 Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt create mode 100644 Android/testbed/app/src/main/python/main.py create mode 100644 Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png create mode 100644 Android/testbed/app/src/main/res/layout/activity_main.xml create mode 100644 Android/testbed/app/src/main/res/values/strings.xml create mode 100644 Android/testbed/build.gradle.kts create mode 100644 Android/testbed/gradle.properties create mode 100644 Android/testbed/gradle/wrapper/gradle-wrapper.properties create mode 100644 Android/testbed/settings.gradle.kts create mode 100644 Misc/NEWS.d/next/Build/2024-04-14-19-35-35.gh-issue-116622.8lpX-7.rst diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 235bc78599400e4..1f5f7e57dc4859f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -243,6 +243,18 @@ Lib/test/support/interpreters/ @ericsnowcurrently Modules/_xx*interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +# Android +**/*Android* @mhsmith +**/*android* @mhsmith + +# iOS (but not termios) +**/iOS* @freakboy3742 +**/ios* @freakboy3742 +**/*_iOS* @freakboy3742 +**/*_ios* @freakboy3742 +**/*-iOS* @freakboy3742 +**/*-ios* @freakboy3742 + # WebAssembly /Tools/wasm/ @brettcannon diff --git a/Android/README.md b/Android/README.md index 5ed186e06e3951c..f5f463ca116589c 100644 --- a/Android/README.md +++ b/Android/README.md @@ -22,12 +22,25 @@ you don't already have the SDK, here's how to install it: `android-sdk/cmdline-tools/latest`. * `export ANDROID_HOME=/path/to/android-sdk` +The `android.py` script also requires the following commands to be on the `PATH`: + +* `curl` +* `java` +* `tar` +* `unzip` + ## Building -Building for Android requires doing a cross-build where you have a "build" -Python to help produce an Android build of CPython. This procedure has been -tested on Linux and macOS. +Python can be built for Android on any POSIX platform supported by the Android +development tools, which currently means Linux or macOS. This involves doing a +cross-build where you use a "build" Python (for your development machine) to +help produce a "host" Python for Android. + +First, make sure you have all the usual tools and libraries needed to build +Python for your development machine. The only Android tool you need to install +is the command line tools package above: the build script will download the +rest. The easiest way to do a build is to use the `android.py` script. You can either have it perform the entire build process from start to finish in one step, or @@ -43,9 +56,10 @@ The discrete steps for building via `android.py` are: ./android.py make-host HOST ``` -To see the possible values of HOST, run `./android.py configure-host --help`. +`HOST` identifies which architecture to build. To see the possible values, run +`./android.py configure-host --help`. -Or to do it all in a single command, run: +To do all steps in a single command, run: ```sh ./android.py build HOST @@ -62,3 +76,22 @@ call. For example, if you want a pydebug build that also caches the results from ```sh ./android.py build HOST -- -C --with-pydebug ``` + + +## Testing + +To run the Python test suite on Android: + +* Install Android Studio, if you don't already have it. +* Follow the instructions in the previous section to build all supported + architectures. +* Run `./android.py setup-testbed` to download the Gradle wrapper. +* Open the `testbed` directory in Android Studio. +* In the *Device Manager* dock, connect a device or start an emulator. + Then select it from the drop-down list in the toolbar. +* Click the "Run" button in the toolbar. +* The testbed app displays nothing on screen while running. To see its output, + open the [Logcat window](https://developer.android.com/studio/debug/logcat). + +To run specific tests, or pass any other arguments to the test suite, edit the +command line in testbed/app/src/main/python/main.py. diff --git a/Android/android.py b/Android/android.py index 5c57e53c415d2b7..0a1393e61ddb0e6 100755 --- a/Android/android.py +++ b/Android/android.py @@ -7,8 +7,9 @@ import subprocess import sys import sysconfig -from os.path import relpath +from os.path import basename, relpath from pathlib import Path +from tempfile import TemporaryDirectory SCRIPT_NAME = Path(__file__).name CHECKOUT = Path(__file__).resolve().parent.parent @@ -102,11 +103,17 @@ def unpack_deps(host): for name_ver in ["bzip2-1.0.8-1", "libffi-3.4.4-2", "openssl-3.0.13-1", "sqlite-3.45.1-0", "xz-5.4.6-0"]: filename = f"{name_ver}-{host}.tar.gz" - run(["wget", f"{deps_url}/{name_ver}/{filename}"]) + download(f"{deps_url}/{name_ver}/{filename}") run(["tar", "-xf", filename]) os.remove(filename) +def download(url, target_dir="."): + out_path = f"{target_dir}/{basename(url)}" + run(["curl", "-Lf", "-o", out_path, url]) + return out_path + + def configure_host_python(context): host_dir = subdir(context.host, clean=context.clean) @@ -160,6 +167,30 @@ def clean_all(context): delete_if_exists(CROSS_BUILD_DIR) +# To avoid distributing compiled artifacts without corresponding source code, +# the Gradle wrapper is not included in the CPython repository. Instead, we +# extract it from the Gradle release. +def setup_testbed(context): + ver_long = "8.7.0" + ver_short = ver_long.removesuffix(".0") + testbed_dir = CHECKOUT / "Android/testbed" + + for filename in ["gradlew", "gradlew.bat"]: + out_path = download( + f"https://raw.githubusercontent.com/gradle/gradle/v{ver_long}/{filename}", + testbed_dir) + os.chmod(out_path, 0o755) + + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + os.chdir(temp_dir) + bin_zip = download( + f"https://services.gradle.org/distributions/gradle-{ver_short}-bin.zip") + outer_jar = f"gradle-{ver_short}/lib/plugins/gradle-wrapper-{ver_short}.jar" + run(["unzip", bin_zip, outer_jar]) + run(["unzip", "-o", "-d", f"{testbed_dir}/gradle/wrapper", outer_jar, + "gradle-wrapper.jar"]) + + def main(): parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -173,8 +204,11 @@ def main(): help="Run `configure` for Android") make_host = subcommands.add_parser("make-host", help="Run `make` for Android") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") + subcommands.add_parser( + "clean", help="Delete the cross-build directory") + subcommands.add_parser( + "setup-testbed", help="Download the testbed Gradle wrapper") + for subcommand in build, configure_build, configure_host: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", @@ -194,7 +228,8 @@ def main(): "configure-host": configure_host_python, "make-host": make_host_python, "build": build_all, - "clean": clean_all} + "clean": clean_all, + "setup-testbed": setup_testbed} dispatch[context.subcommand](context) diff --git a/Android/testbed/.gitignore b/Android/testbed/.gitignore new file mode 100644 index 000000000000000..b9a7d611c943cf6 --- /dev/null +++ b/Android/testbed/.gitignore @@ -0,0 +1,21 @@ +# The Gradle wrapper should be downloaded by running `../android.py setup-testbed`. +/gradlew +/gradlew.bat +/gradle/wrapper/gradle-wrapper.jar + +*.iml +.gradle +/local.properties +/.idea/caches +/.idea/deploymentTargetDropdown.xml +/.idea/libraries +/.idea/modules.xml +/.idea/workspace.xml +/.idea/navEditor.xml +/.idea/assetWizardSettings.xml +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties diff --git a/Android/testbed/app/.gitignore b/Android/testbed/app/.gitignore new file mode 100644 index 000000000000000..42afabfd2abebf3 --- /dev/null +++ b/Android/testbed/app/.gitignore @@ -0,0 +1 @@ +/build \ No newline at end of file diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts new file mode 100644 index 000000000000000..7690d3fd86b2fdf --- /dev/null +++ b/Android/testbed/app/build.gradle.kts @@ -0,0 +1,129 @@ +import com.android.build.api.variant.* + +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") +} + +val PYTHON_DIR = File(projectDir, "../../..").canonicalPath +val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build" +val ABIS = mapOf( + "arm64-v8a" to "aarch64-linux-android", + "x86_64" to "x86_64-linux-android", +) + +val PYTHON_VERSION = File("$PYTHON_DIR/Include/patchlevel.h").useLines { + for (line in it) { + val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line) + if (match != null) { + return@useLines match.groupValues[1] + } + } + throw GradleException("Failed to find Python version") +} + + +android { + namespace = "org.python.testbed" + compileSdk = 34 + + defaultConfig { + applicationId = "org.python.testbed" + minSdk = 21 + targetSdk = 34 + versionCode = 1 + versionName = "1.0" + + ndk.abiFilters.addAll(ABIS.keys) + externalNativeBuild.cmake.arguments( + "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR", + "-DPYTHON_VERSION=$PYTHON_VERSION") + } + + externalNativeBuild.cmake { + path("src/main/c/CMakeLists.txt") + } + + // Set this property to something non-empty, otherwise it'll use the default + // list, which ignores asset directories beginning with an underscore. + aaptOptions.ignoreAssetsPattern = ".git" + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + kotlinOptions { + jvmTarget = "1.8" + } +} + +dependencies { + implementation("androidx.appcompat:appcompat:1.6.1") + implementation("com.google.android.material:material:1.11.0") + implementation("androidx.constraintlayout:constraintlayout:2.1.4") +} + + +// Create some custom tasks to copy Python and its standard library from +// elsewhere in the repository. +androidComponents.onVariants { variant -> + generateTask(variant, variant.sources.assets!!) { + into("python") { + for (triplet in ABIS.values) { + for (subDir in listOf("include", "lib")) { + into(subDir) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/$subDir") + include("python$PYTHON_VERSION/**") + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + } + } + } + into("lib/python$PYTHON_VERSION") { + // Uncomment this to pick up edits from the source directory + // without having to rerun `make install`. + // from("$PYTHON_DIR/Lib") + // duplicatesStrategy = DuplicatesStrategy.INCLUDE + + into("site-packages") { + from("$projectDir/src/main/python") + } + } + } + exclude("**/__pycache__") + } + + generateTask(variant, variant.sources.jniLibs!!) { + for ((abi, triplet) in ABIS.entries) { + into(abi) { + from("$PYTHON_CROSS_DIR/$triplet/prefix/lib") + include("libpython*.*.so") + include("lib*_python.so") + } + } + } +} + + +fun generateTask( + variant: ApplicationVariant, directories: SourceDirectories, + configure: GenerateTask.() -> Unit +) { + val taskName = "generate" + + listOf(variant.name, "Python", directories.name) + .map { it.replaceFirstChar(Char::uppercase) } + .joinToString("") + + directories.addGeneratedSourceDirectory( + tasks.register(taskName) { + into(outputDir) + configure() + }, + GenerateTask::outputDir) +} + + +// addGeneratedSourceDirectory requires the task to have a DirectoryProperty. +abstract class GenerateTask: Sync() { + @get:OutputDirectory + abstract val outputDir: DirectoryProperty +} diff --git a/Android/testbed/app/src/main/AndroidManifest.xml b/Android/testbed/app/src/main/AndroidManifest.xml new file mode 100644 index 000000000000000..2be8a82d4260999 --- /dev/null +++ b/Android/testbed/app/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/c/CMakeLists.txt b/Android/testbed/app/src/main/c/CMakeLists.txt new file mode 100644 index 000000000000000..1d5df9a73465b6e --- /dev/null +++ b/Android/testbed/app/src/main/c/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.4.1) +project(testbed) + +set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix) +include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION}) +link_directories(${PREFIX_DIR}/lib) +link_libraries(log python${PYTHON_VERSION}) + +add_library(main_activity SHARED main_activity.c) diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c new file mode 100644 index 000000000000000..73aba4164d000f7 --- /dev/null +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -0,0 +1,147 @@ +#include +#include +#include +#include +#include +#include +#include +#include + + +static void throw_runtime_exception(JNIEnv *env, const char *message) { + (*env)->ThrowNew( + env, + (*env)->FindClass(env, "java/lang/RuntimeException"), + message); +} + + +// --- Stdio redirection ------------------------------------------------------ + +// Most apps won't need this, because the Python-level sys.stdout and sys.stderr +// are redirected to the Android logcat by Python itself. However, in the +// testbed it's useful to redirect the native streams as well, to debug problems +// in the Python startup or redirection process. +// +// Based on +// https://github.com/beeware/briefcase-android-gradle-template/blob/v0.3.11/%7B%7B%20cookiecutter.safe_formal_name%20%7D%7D/app/src/main/cpp/native-lib.cpp + +typedef struct { + FILE *file; + int fd; + android_LogPriority priority; + char *tag; + int pipe[2]; +} StreamInfo; + +static StreamInfo STREAMS[] = { + {stdout, STDOUT_FILENO, ANDROID_LOG_INFO, "native.stdout", {-1, -1}}, + {stderr, STDERR_FILENO, ANDROID_LOG_WARN, "native.stderr", {-1, -1}}, + {NULL, -1, ANDROID_LOG_UNKNOWN, NULL, {-1, -1}}, +}; + +// The maximum length of a log message in bytes, including the level marker and +// tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD in +// platform/system/logging/liblog/include/log/log.h. As of API level 30, messages +// longer than this will be be truncated by logcat. This limit has already been +// reduced at least once in the history of Android (from 4076 to 4068 between API +// level 23 and 26), so leave some headroom. +static const int MAX_BYTES_PER_WRITE = 4000; + +static void *redirection_thread(void *arg) { + StreamInfo *si = (StreamInfo*)arg; + ssize_t read_size; + char buf[MAX_BYTES_PER_WRITE]; + while ((read_size = read(si->pipe[0], buf, sizeof buf - 1)) > 0) { + buf[read_size] = '\0'; /* add null-terminator */ + __android_log_write(si->priority, si->tag, buf); + } + return 0; +} + +static char *redirect_stream(StreamInfo *si) { + /* make the FILE unbuffered, to ensure messages are never lost */ + if (setvbuf(si->file, 0, _IONBF, 0)) { + return "setvbuf"; + } + + /* create the pipe and redirect the file descriptor */ + if (pipe(si->pipe)) { + return "pipe"; + } + if (dup2(si->pipe[1], si->fd) == -1) { + return "dup2"; + } + + /* start the logging thread */ + pthread_t thr; + if ((errno = pthread_create(&thr, 0, redirection_thread, si))) { + return "pthread_create"; + } + if ((errno = pthread_detach(thr))) { + return "pthread_detach"; + } + return 0; +} + +JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_redirectStdioToLogcat( + JNIEnv *env, jobject obj +) { + for (StreamInfo *si = STREAMS; si->file; si++) { + char *error_prefix; + if ((error_prefix = redirect_stream(si))) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); + return; + } + } +} + + +// --- Python intialization ---------------------------------------------------- + +static PyStatus set_config_string( + JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value +) { + const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); + PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); + (*env)->ReleaseStringUTFChars(env, value, value_utf8); + return status; +} + +static void throw_status(JNIEnv *env, PyStatus status) { + throw_runtime_exception(env, status.err_msg ? status.err_msg : ""); +} + +JNIEXPORT void JNICALL Java_org_python_testbed_MainActivity_runPython( + JNIEnv *env, jobject obj, jstring home, jstring runModule +) { + PyConfig config; + PyStatus status; + PyConfig_InitIsolatedConfig(&config); + + status = set_config_string(env, &config, &config.home, home); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + status = set_config_string(env, &config, &config.run_module, runModule); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. + config.install_signal_handlers = 1; + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + throw_status(env, status); + return; + } + + Py_RunMain(); +} diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt new file mode 100644 index 000000000000000..5a590d5d04e9542 --- /dev/null +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -0,0 +1,61 @@ +package org.python.testbed + +import android.os.* +import android.system.Os +import android.widget.TextView +import androidx.appcompat.app.* +import java.io.* + +class MainActivity : AppCompatActivity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_main) + + // Python needs this variable to help it find the temporary directory, + // but Android only sets it on API level 33 and later. + Os.setenv("TMPDIR", cacheDir.toString(), false) + + val pythonHome = extractAssets() + System.loadLibrary("main_activity") + redirectStdioToLogcat() + runPython(pythonHome.toString(), "main") + findViewById(R.id.tvHello).text = "Python complete" + } + + private fun extractAssets() : File { + val pythonHome = File(filesDir, "python") + if (pythonHome.exists() && !pythonHome.deleteRecursively()) { + throw RuntimeException("Failed to delete $pythonHome") + } + extractAssetDir("python", filesDir) + return pythonHome + } + + private fun extractAssetDir(path: String, targetDir: File) { + val names = assets.list(path) + ?: throw RuntimeException("Failed to list $path") + val targetSubdir = File(targetDir, path) + if (!targetSubdir.mkdirs()) { + throw RuntimeException("Failed to create $targetSubdir") + } + + for (name in names) { + val subPath = "$path/$name" + val input: InputStream + try { + input = assets.open(subPath) + } catch (e: FileNotFoundException) { + extractAssetDir(subPath, targetDir) + continue + } + input.use { + File(targetSubdir, name).outputStream().use { output -> + input.copyTo(output) + } + } + } + } + + private external fun redirectStdioToLogcat() + private external fun runPython(home: String, runModule: String) +} \ No newline at end of file diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py new file mode 100644 index 000000000000000..a1b6def34ede811 --- /dev/null +++ b/Android/testbed/app/src/main/python/main.py @@ -0,0 +1,17 @@ +import runpy +import signal +import sys + +# Some tests use SIGUSR1, but that's blocked by default in an Android app in +# order to make it available to `sigwait` in the "Signal Catcher" thread. That +# thread's functionality is only relevant to the JVM ("forcing GC (no HPROF) and +# profile save"), so disabling it should not weaken the tests. +signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) + +# To run specific tests, or pass any other arguments to the test suite, edit +# this command line. +sys.argv[1:] = [ + "--use", "all,-cpu", + "--verbose3", +] +runpy.run_module("test") diff --git a/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png b/Android/testbed/app/src/main/res/drawable-xxhdpi/ic_launcher.png new file mode 100644 index 0000000000000000000000000000000000000000..741d6580d60e05080e71e845240863f6b273333c GIT binary patch literal 3110 zcmZ`)c{J1w+y4D#82cK@5~GZz$odeb8q64!?6MDGhU{swg(16WFly|>Sei*Z_NOdK zo1&1NsK?gWm90o#?>XOh&iBuEpL5;Ux$krTcU||nQZ27y`M4#x0RZ4LH9=b+G5NpX zgdO#7#xwsM5m&H@V<-S{uKgFF{+TDsM@#Wr7>8RnLHJu?9yh&#u&}UmzJY$Bo*u#8 z=Ynn$3f7Pk0Kn5=iZ-wfpI9x54i?!KfBRCuY^3GP3{S3hMWz+6`A$n1*RZ9PpMF4q zFLTw__EXm9K#exe9dont8^m%aqd*;snDa{G;Li~ye)#EAoKIWHN|rR0<3e@DBOobK_suVh(?P{3y7vaQk`5{_18_L~&KO+PyM3^Pi3oGBV;%|BsX% zcHOeX@)Ye%OyiAnrm54$hqo;r%Gn(Tj(+h%L>{v#csbv+H|)LJD|uByc!>!Xv)HDgM28MLaR0EYg%%3X z1#ar?p&M(*!~ZGiHwi8J(C573o3fPO1Cr@PnE>ODe;vadHcj=>#Xsm4Pp+9f?RbTT0FB2 zAM=1mBp`1YCA7MSs)m9zX)tBk$T8s0jj-f?s<107y}=N8gawwk)ed`Y51j9)-V3_B zL6MXoR$6`Dco4*20Md%tx(^{JfD-`W02>Jq1R(!+__2szGRkIt2z3=&bp%BE_fYI+ zDreaL=S0MW#H6qs?}QFMp7Av@gf^CLxy&}Hf72Zq$&NDwMeNI7<0Eu2&ZyCh-`z@! zp1sA9CuryQNnUdmY?DT#!?&XQQxfOgI817Sep4J@Xzl*;HbIj#no`}FmG@T%=|IMj z9mxzQPo{=a@3sKS^z~Sean$&RG-zVKor;t>nY4F;Rz6K1P;^lv8JowhT#+C;xqG<- zGZR6LW};PmmY63oLQ|bE>WTBD?9&4i?=Sy=bthnlo4YzYA+HF2wGt}B2U1YVdmo>~ z2#`aOYalukoGD0E;|D6XWDRb`1(5uYhsVc51x<@Rd1!NwdA@}JmP!2L%V*BL=MsE< zVfTd^K=BsIpH@LJ2CAS*AWeDV_L4nz&ny((Dv1YaQuKamHK2~c@ihSC9QVCB3B8xr zm{t!emAw-00BmLP00%FH;YXj$PER-&qNBuDyi*Uh3s{&v`FDnwfo8t~HDq@zzw9Lob6+lDYC-n17119tqo=mFGzY&$P?M)PoF+g6&Dj@gWn}Q{c%vMM@6{Bs@dy=C16%;&hjE%0zMI? zXjl)b<(^KEgg@A4qOlr#ZEs1M~&AN=r?#cF* zQ3)s*a-ovZlb$pV?(uQ3h$mR`=LZ%npTs13-|qc1W8gKUdA-ZCa$lM5+qT1hZ^e!% zs1-XygoVpARHqiTWm*)rpHfDe1}Oh9txJggUhIAoz0V~c8Wn+V#iaSnqJ^Ss1=r!W zdh@BF0iyL^wIv-u}6&2`~AysKWz6z1$NCH+jr_^jqm+M)=f@KR$zqv z;=I-CiPgXM-t1O&?UH(agYu%AhF1%=bMezf{(kU&XG6|z%r2i65_R=&)trR_#>;6) z)n)8?RMdrF=>^BA??zTQ4KFsF>|fXCTd0Uhw3SQ8-5O%KaRWE?EXNE_&0Q&)>#23) z9;SAMv(L!A4svha6`jrOeS#OHs$GiqA-*d#A%1S%o1M_U<@hWvQJVYw!K0Q4MCLrC zYtVFK$RWOg$-IO!)X6#UpiNHL%i*2fGKP;YTJx*p zwH@8PE`9fWOQ&8eO(O2=IN_;eKqhj<{Dk~x{E})oCXscud^dm ziuOL0r%(R${b$q;=Nneh%S5VN^+olGo{y%r0+bak77T^e#Nz^wTg0#URA3S{E-VS7 zT9t3o0+UP)-(DK5z9APgg&=A3l*DlrReQd9me%KB=X=r+FTJw8En|1t;mVg-Haa@D zky)TOFp(bU<>}PhiSJVM&{V(i3{=zu0fQ;l>4t03+N88)Ne&@&D=0`&$sIRtkv=d0 z*t|g>+pDCxy*EACx|0B>*`;WoR?pIEGyw`{QI@u5{h_a}B}LWw$pXT-o|8IL=G*40 zY$%8GT24LKZ^Jrn)=m4M+cwe5?fVPz%FcR9gDi%I zpyF(jg?L=E(gmDz<(QA1_bTxCw}7RZ^@9(`O0~y=ZL`bUuGU^Tej9@7m{y*T2-nUF zb{S0~WV~I2sh^~^5@4NdaJ*;gjWOLthP zfo~xBvafDP^J^2e57iYz)>Eo7uvgX72w_y4gQzNER^j!0=`RSBR9F%#{_o?+d+f*m zH%>yp8F4DtP71&TQEqG|h1+?e|6k@D1(1o)s`m92S-SbN7e0JAPr0#&)~`=DgK&Uy z`MIXOT~Tg@{=Y!H?}Gz9GN9-Vyt5c@8;t@50lndsuwLy4&FiCA!|;F?PJKu7K2&RP z;6Q%tNmb5CZjy!{+F~bMsPb?6zN>RYb4Fh@3>cS=E1yL2lQ%WdA+6_k8JaE=nl5lkwUxgm58Uf2>Pa5t9>>eNrW8Rx<>9vOUl9G)*6b69mmS-kEqQhy|q9HF?2>d3Xe@bT528 zc`+&Z?G=kikGb#isCRag779=soISf_`q08OPtDs;MqgOaV16p22VyaNz$GYFBB3x= z(=u?1g|`9#(p0g*K+npPpAS=k$_5(@Rd7sfBbvq&)j_9QPAh z!BPZ2&U_`2=X}@*F;!nXyI8s%Q|Gnqbuxe*C7qJjYwonj z;k7aAwWXo)dp)9kM|fp+E<3(4{$gmRc=~HYoYsTm0)}x6IJ4cK(;zsmuF{`6AnzJ@ wobE4xi)d3rtlRePkGCZCRb>UJ6+2oVa;Pv|-I|Ho!@yCQVy>cV4Bg}Z16Zh-Q2+n{ literal 0 HcmV?d00001 diff --git a/Android/testbed/app/src/main/res/layout/activity_main.xml b/Android/testbed/app/src/main/res/layout/activity_main.xml new file mode 100644 index 000000000000000..21398609ec9c785 --- /dev/null +++ b/Android/testbed/app/src/main/res/layout/activity_main.xml @@ -0,0 +1,19 @@ + + + + + + \ No newline at end of file diff --git a/Android/testbed/app/src/main/res/values/strings.xml b/Android/testbed/app/src/main/res/values/strings.xml new file mode 100644 index 000000000000000..352d2f9e885a2a2 --- /dev/null +++ b/Android/testbed/app/src/main/res/values/strings.xml @@ -0,0 +1,3 @@ + + Python testbed + \ No newline at end of file diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts new file mode 100644 index 000000000000000..53f4a67287fcc57 --- /dev/null +++ b/Android/testbed/build.gradle.kts @@ -0,0 +1,5 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. +plugins { + id("com.android.application") version "8.2.2" apply false + id("org.jetbrains.kotlin.android") version "1.9.22" apply false +} \ No newline at end of file diff --git a/Android/testbed/gradle.properties b/Android/testbed/gradle.properties new file mode 100644 index 000000000000000..3c5031eb7d63f78 --- /dev/null +++ b/Android/testbed/gradle.properties @@ -0,0 +1,23 @@ +# Project-wide Gradle settings. +# IDE (e.g. Android Studio) users: +# Gradle settings configured through the IDE *will override* +# any settings specified in this file. +# For more details on how to configure your build environment visit +# http://www.gradle.org/docs/current/userguide/build_environment.html +# Specifies the JVM arguments used for the daemon process. +# The setting is particularly useful for tweaking memory settings. +org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8 +# When configured, Gradle will run in incubating parallel mode. +# This option should only be used with decoupled projects. More details, visit +# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects +# org.gradle.parallel=true +# AndroidX package structure to make it clearer which packages are bundled with the +# Android operating system, and which are packaged with your app's APK +# https://developer.android.com/topic/libraries/support-library/androidx-rn +android.useAndroidX=true +# Kotlin code style for this project: "official" or "obsolete": +kotlin.code.style=official +# Enables namespacing of each library's R class so that its R class includes only the +# resources declared in the library itself and none from the library's dependencies, +# thereby reducing the size of the R class for that library +android.nonTransitiveRClass=true \ No newline at end of file diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000000000..2dc3339a3ef2130 --- /dev/null +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Mon Feb 19 20:29:06 GMT 2024 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/Android/testbed/settings.gradle.kts b/Android/testbed/settings.gradle.kts new file mode 100644 index 000000000000000..5e08773e02450f9 --- /dev/null +++ b/Android/testbed/settings.gradle.kts @@ -0,0 +1,18 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} +dependencyResolutionManagement { + repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "Python testbed" +include(":app") + \ No newline at end of file diff --git a/Misc/NEWS.d/next/Build/2024-04-14-19-35-35.gh-issue-116622.8lpX-7.rst b/Misc/NEWS.d/next/Build/2024-04-14-19-35-35.gh-issue-116622.8lpX-7.rst new file mode 100644 index 000000000000000..c270e59cd54c189 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-04-14-19-35-35.gh-issue-116622.8lpX-7.rst @@ -0,0 +1 @@ +A testbed project was added to run the test suite on Android. From fc7e1aa3c001bbce25973261fba457035719a559 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 1 May 2024 07:44:01 +0100 Subject: [PATCH 032/220] GH-117881: fix athrow().throw()/asend().throw() concurrent access (GH-117882) --- Lib/test/test_asyncgen.py | 199 +++++++++++++++++- ...-04-15-07-37-09.gh-issue-117881.07H0wI.rst | 1 + Objects/genobject.c | 37 ++++ 3 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-15-07-37-09.gh-issue-117881.07H0wI.rst diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index a1e9e1b89c6a2cf..1985ede656e7a05 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -393,6 +393,151 @@ async def gen(): r'anext\(\): asynchronous generator is already running'): an.__next__() + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + an.send(None) + + def test_async_gen_asend_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_send(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + while True: + try: + await _async_yield(None) + except MyExc: + pass + return + yield + + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + + def test_async_gen_asend_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.asend(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited __anext__\(\)/asend\(\)"): + gen2.send(None) + + def test_async_gen_athrow_throw_concurrent_with_throw(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + pass + while True: + try: + await _async_yield(None) + except MyExc: + pass + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + + gen = agen.athrow(MyExc) + gen.throw(MyExc) + gen2 = agen.athrow(None) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.throw(MyExc) + + with self.assertRaisesRegex(RuntimeError, + r"cannot reuse already awaited aclose\(\)/athrow\(\)"): + gen2.send(None) + def test_async_gen_3_arg_deprecation_warning(self): async def gen(): yield 123 @@ -1571,6 +1716,8 @@ async def main(): self.assertIsInstance(message['exception'], ZeroDivisionError) self.assertIn('unhandled exception during asyncio.run() shutdown', message['message']) + del message, messages + gc_collect() def test_async_gen_expression_01(self): async def arange(n): @@ -1624,6 +1771,7 @@ async def main(): asyncio.run(main()) self.assertEqual([], messages) + gc_collect() def test_async_gen_await_same_anext_coro_twice(self): async def async_iterate(): @@ -1809,9 +1957,56 @@ class MyException(Exception): g = gen() with self.assertRaises(MyException): g.aclose().throw(MyException) - del g - gc_collect() + del g + gc_collect() # does not warn unawaited + + def test_asend_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.asend(None) + + with self.assertRaisesRegex(RuntimeError, + r'anext\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited + + + def test_athrow_send_already_running(self): + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + while True: + await _async_yield(1) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + gen2 = agen.athrow(Exception) + + with self.assertRaisesRegex(RuntimeError, + r'athrow\(\): asynchronous generator is already running'): + gen2.send(None) + + del gen2 + gc_collect() # does not warn unawaited if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-15-07-37-09.gh-issue-117881.07H0wI.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-07-37-09.gh-issue-117881.07H0wI.rst new file mode 100644 index 000000000000000..75b34269695693e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-15-07-37-09.gh-issue-117881.07H0wI.rst @@ -0,0 +1 @@ +prevent concurrent access to an async generator via athrow().throw() or asend().throw() diff --git a/Objects/genobject.c b/Objects/genobject.c index a1ed1cbd2bfb580..89bb21a8674b9f1 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1774,6 +1774,7 @@ async_gen_asend_send(PyAsyncGenASend *o, PyObject *arg) if (o->ags_state == AWAITABLE_STATE_INIT) { if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; PyErr_SetString( PyExc_RuntimeError, "anext(): asynchronous generator is already running"); @@ -1817,10 +1818,24 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg return NULL; } + if (o->ags_state == AWAITABLE_STATE_INIT) { + if (o->ags_gen->ag_running_async) { + o->ags_state = AWAITABLE_STATE_CLOSED; + PyErr_SetString( + PyExc_RuntimeError, + "anext(): asynchronous generator is already running"); + return NULL; + } + + o->ags_state = AWAITABLE_STATE_ITER; + o->ags_gen->ag_running_async = 1; + } + result = gen_throw((PyGenObject*)o->ags_gen, args, nargs); result = async_gen_unwrap_value(o->ags_gen, result); if (result == NULL) { + o->ags_gen->ag_running_async = 0; o->ags_state = AWAITABLE_STATE_CLOSED; } @@ -2209,10 +2224,31 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } + if (o->agt_state == AWAITABLE_STATE_INIT) { + if (o->agt_gen->ag_running_async) { + o->agt_state = AWAITABLE_STATE_CLOSED; + if (o->agt_args == NULL) { + PyErr_SetString( + PyExc_RuntimeError, + "aclose(): asynchronous generator is already running"); + } + else { + PyErr_SetString( + PyExc_RuntimeError, + "athrow(): asynchronous generator is already running"); + } + return NULL; + } + + o->agt_state = AWAITABLE_STATE_ITER; + o->agt_gen->ag_running_async = 1; + } + retval = gen_throw((PyGenObject*)o->agt_gen, args, nargs); if (o->agt_args) { retval = async_gen_unwrap_value(o->agt_gen, retval); if (retval == NULL) { + o->agt_gen->ag_running_async = 0; o->agt_state = AWAITABLE_STATE_CLOSED; } return retval; @@ -2226,6 +2262,7 @@ async_gen_athrow_throw(PyAsyncGenAThrow *o, PyObject *const *args, Py_ssize_t na return NULL; } if (retval == NULL) { + o->agt_gen->ag_running_async = 0; o->agt_state = AWAITABLE_STATE_CLOSED; } if (PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || From 21c09d9f8195433f34b72ddfa25dd1bda3019ed7 Mon Sep 17 00:00:00 2001 From: da-woods Date: Wed, 1 May 2024 09:33:28 +0100 Subject: [PATCH 033/220] Test syntax error on comma-less tuple-style sequence patterns (#115485) Adds a test that length-1 tuple-style sequence patterns must end in a comma, since there isn't currently one. Spotted while reviewing Cython's proposed implementation of the pattern matching syntax (https://github.com/cython/cython/pull/4897#discussion_r1489177169) where there was a bug my the reimplementation that wasn't caught against the CPython tests here. --- Lib/test/test_patma.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py index 298e78ccee38757..1bdab125dc6ef08 100644 --- a/Lib/test/test_patma.py +++ b/Lib/test/test_patma.py @@ -2957,6 +2957,14 @@ def test_invalid_syntax_3(self): pass """) + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): self.assert_syntax_error(""" match ...: From f6fab21721c8aedc5dca97dbeb6292a067c19bf1 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 May 2024 11:34:50 +0100 Subject: [PATCH 034/220] GH-118095: Make invalidating and clearing executors memory safe (GH-118459) --- Include/cpython/optimizer.h | 3 +- Objects/codeobject.c | 3 +- Python/bytecodes.c | 12 +++- Python/generated_cases.c.h | 12 +++- Python/optimizer.c | 115 +++++++++++++++++++++++++----------- 5 files changed, 103 insertions(+), 42 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 819251a25bb242e..a169280b26a6ad1 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -24,6 +24,7 @@ typedef struct { uint8_t opcode; uint8_t oparg; uint8_t valid; + uint8_t linked; int index; // Index of ENTER_EXECUTOR (if code isn't NULL, below). _PyBloomFilter bloom; _PyExecutorLinkListNode links; @@ -135,7 +136,7 @@ PyAPI_FUNC(_PyOptimizerObject *) PyUnstable_GetOptimizer(void); PyAPI_FUNC(_PyExecutorObject *) PyUnstable_GetExecutor(PyCodeObject *code, int offset); void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); -void _Py_ExecutorClear(_PyExecutorObject *); +void _Py_ExecutorDetach(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 605167c5c0bd3aa..810f847485acdab 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1504,7 +1504,8 @@ clear_executors(PyCodeObject *co) assert(co->co_executors); for (int i = 0; i < co->co_executors->size; i++) { if (co->co_executors->executors[i]) { - _Py_ExecutorClear(co->co_executors->executors[i]); + _Py_ExecutorDetach(co->co_executors->executors[i]); + assert(co->co_executors->executors[i] == NULL); } } PyMem_Free(co->co_executors); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 28766d6d97cc5f9..002b5a3529c127a 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -163,9 +163,15 @@ dummy_func( if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - #if ENABLE_SPECIALIZATION - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - #endif /* ENABLE_SPECIALIZATION */ + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ + } } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 602cce4beb1e28a..32b485ecb9d8cc8 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -4961,9 +4961,15 @@ if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { CHECK_EVAL_BREAKER(); } - #if ENABLE_SPECIALIZATION - FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); - #endif /* ENABLE_SPECIALIZATION */ + assert(this_instr->op.code == RESUME || + this_instr->op.code == RESUME_CHECK || + this_instr->op.code == INSTRUMENTED_RESUME || + this_instr->op.code == ENTER_EXECUTOR); + if (this_instr->op.code == RESUME) { + #if ENABLE_SPECIALIZATION + FT_ATOMIC_STORE_UINT8_RELAXED(this_instr->op.code, RESUME_CHECK); + #endif /* ENABLE_SPECIALIZATION */ + } } DISPATCH(); } diff --git a/Python/optimizer.c b/Python/optimizer.c index a5e7430c46450ef..6576aa1cddc033c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -75,7 +75,7 @@ insert_executor(PyCodeObject *code, _Py_CODEUNIT *instr, int index, _PyExecutorO Py_INCREF(executor); if (instr->op.code == ENTER_EXECUTOR) { assert(index == instr->op.arg); - _Py_ExecutorClear(code->co_executors->executors[index]); + _Py_ExecutorDetach(code->co_executors->executors[index]); } else { assert(code->co_executors->size == index); @@ -270,10 +270,14 @@ static PyMethodDef executor_methods[] = { ///////////////////// Experimental UOp Optimizer ///////////////////// +static int executor_clear(_PyExecutorObject *executor); +static void unlink_executor(_PyExecutorObject *executor); + static void uop_dealloc(_PyExecutorObject *self) { _PyObject_GC_UNTRACK(self); - _Py_ExecutorClear(self); + assert(self->vm_data.code == NULL); + unlink_executor(self); #ifdef _Py_JIT _PyJIT_Free(self); #endif @@ -379,13 +383,6 @@ PySequenceMethods uop_as_sequence = { .sq_item = (ssizeargfunc)uop_item, }; -static int -executor_clear(PyObject *o) -{ - _Py_ExecutorClear((_PyExecutorObject *)o); - return 0; -} - static int executor_traverse(PyObject *o, visitproc visit, void *arg) { @@ -412,7 +409,7 @@ PyTypeObject _PyUOpExecutor_Type = { .tp_as_sequence = &uop_as_sequence, .tp_methods = executor_methods, .tp_traverse = executor_traverse, - .tp_clear = executor_clear, + .tp_clear = (inquiry)executor_clear, .tp_is_gc = executor_is_gc, }; @@ -1190,6 +1187,7 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) inst->opcode = _COLD_EXIT; inst->oparg = oparg; executor->vm_data.valid = true; + executor->vm_data.linked = false; for (int i = 0; i < BLOOM_FILTER_WORDS; i++) { assert(executor->vm_data.bloom.bits[i] == 0); } @@ -1328,7 +1326,7 @@ PyTypeObject _PyCounterExecutor_Type = { .tp_dealloc = (destructor)counter_dealloc, .tp_methods = executor_methods, .tp_traverse = executor_traverse, - .tp_clear = executor_clear, + .tp_clear = (inquiry)executor_clear, }; static int @@ -1503,15 +1501,13 @@ link_executor(_PyExecutorObject *executor) links->next = NULL; } else { - _PyExecutorObject *next = head->vm_data.links.next; - links->previous = head; - links->next = next; - if (next != NULL) { - next->vm_data.links.previous = executor; - } - head->vm_data.links.next = executor; + assert(head->vm_data.links.previous == NULL); + links->previous = NULL; + links->next = head; + head->vm_data.links.previous = executor; + interp->executor_list_head = executor; } - executor->vm_data.valid = true; + executor->vm_data.linked = true; /* executor_list_head must be first in list */ assert(interp->executor_list_head->vm_data.links.previous == NULL); } @@ -1519,7 +1515,11 @@ link_executor(_PyExecutorObject *executor) static void unlink_executor(_PyExecutorObject *executor) { + if (!executor->vm_data.linked) { + return; + } _PyExecutorLinkListNode *links = &executor->vm_data.links; + assert(executor->vm_data.valid); _PyExecutorObject *next = links->next; _PyExecutorObject *prev = links->previous; if (next != NULL) { @@ -1534,7 +1534,7 @@ unlink_executor(_PyExecutorObject *executor) assert(interp->executor_list_head == executor); interp->executor_list_head = next; } - executor->vm_data.valid = false; + executor->vm_data.linked = false; } /* This must be called by optimizers before using the executor */ @@ -1548,23 +1548,15 @@ _Py_ExecutorInit(_PyExecutorObject *executor, const _PyBloomFilter *dependency_s link_executor(executor); } -/* This must be called by executors during dealloc */ +/* Detaches the executor from the code object (if any) that + * holds a reference to it */ void -_Py_ExecutorClear(_PyExecutorObject *executor) +_Py_ExecutorDetach(_PyExecutorObject *executor) { - if (!executor->vm_data.valid) { - return; - } - unlink_executor(executor); PyCodeObject *code = executor->vm_data.code; if (code == NULL) { return; } - for (uint32_t i = 0; i < executor->exit_count; i++) { - Py_DECREF(executor->exits[i].executor); - executor->exits[i].executor = &COLD_EXITS[i]; - executor->exits[i].temperature = initial_unreachable_backoff_counter(); - } _Py_CODEUNIT *instruction = &_PyCode_CODE(code)[executor->vm_data.index]; assert(instruction->op.code == ENTER_EXECUTOR); int index = instruction->op.arg; @@ -1572,7 +1564,36 @@ _Py_ExecutorClear(_PyExecutorObject *executor) instruction->op.code = executor->vm_data.opcode; instruction->op.arg = executor->vm_data.oparg; executor->vm_data.code = NULL; - Py_CLEAR(code->co_executors->executors[index]); + code->co_executors->executors[index] = NULL; + Py_DECREF(executor); +} + +static int +executor_clear(_PyExecutorObject *executor) +{ + if (!executor->vm_data.valid) { + return 0; + } + assert(executor->vm_data.valid == 1); + unlink_executor(executor); + executor->vm_data.valid = 0; + /* It is possible for an executor to form a reference + * cycle with itself, so decref'ing a side exit could + * free the executor unless we hold a strong reference to it + */ + Py_INCREF(executor); + for (uint32_t i = 0; i < executor->exit_count; i++) { + const _PyExecutorObject *cold = &COLD_EXITS[i]; + const _PyExecutorObject *side = executor->exits[i].executor; + executor->exits[i].temperature = initial_unreachable_backoff_counter(); + if (side != cold) { + executor->exits[i].executor = cold; + Py_DECREF(side); + } + } + _Py_ExecutorDetach(executor); + Py_DECREF(executor); + return 0; } void @@ -1593,17 +1614,42 @@ _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is _Py_BloomFilter_Add(&obj_filter, obj); /* Walk the list of executors */ /* TO DO -- Use a tree to avoid traversing as many objects */ + bool no_memory = false; + PyObject *invalidate = PyList_New(0); + if (invalidate == NULL) { + PyErr_Clear(); + no_memory = true; + } + /* Clearing an executor can deallocate others, so we need to make a list of + * executors to invalidate first */ for (_PyExecutorObject *exec = interp->executor_list_head; exec != NULL;) { assert(exec->vm_data.valid); _PyExecutorObject *next = exec->vm_data.links.next; if (bloom_filter_may_contain(&exec->vm_data.bloom, &obj_filter)) { - _Py_ExecutorClear(exec); + unlink_executor(exec); + if (no_memory) { + exec->vm_data.valid = 0; + } else { + if (PyList_Append(invalidate, (PyObject *)exec) < 0) { + PyErr_Clear(); + no_memory = true; + exec->vm_data.valid = 0; + } + } if (is_invalidation) { OPT_STAT_INC(executors_invalidated); } } exec = next; } + if (invalidate != NULL) { + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(invalidate); i++) { + _PyExecutorObject *exec = (_PyExecutorObject *)PyList_GET_ITEM(invalidate, i); + executor_clear(exec); + } + Py_DECREF(invalidate); + } + return; } /* Invalidate all executors */ @@ -1612,12 +1658,13 @@ _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation) { while (interp->executor_list_head) { _PyExecutorObject *executor = interp->executor_list_head; + assert(executor->vm_data.valid == 1 && executor->vm_data.linked == 1); if (executor->vm_data.code) { // Clear the entire code object so its co_executors array be freed: _PyCode_Clear_Executors(executor->vm_data.code); } else { - _Py_ExecutorClear(executor); + executor_clear(executor); } if (is_invalidation) { OPT_STAT_INC(executors_invalidated); From c1bf4874c1e9db2beda1d62c8c241229783c789b Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 1 May 2024 12:01:16 +0100 Subject: [PATCH 035/220] gh-116767: fix crash on 'async with' with many context managers (GH-118348) Account for `add_stopiteration_handler` pushing a block for `async with`. To allow generator functions that previously almost hit the `CO_MAXBLOCKS` limit by nesting non-async blocks, the limit is increased by 1. This increase allows one more block in non-generator functions. --- Include/cpython/code.h | 2 +- Lib/test/test_syntax.py | 36 ++++++++++++++++--- ...-04-27-16-23-29.gh-issue-116767.z9UFpr.rst | 1 + Python/compile.c | 18 ++++++++-- 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-27-16-23-29.gh-issue-116767.z9UFpr.rst diff --git a/Include/cpython/code.h b/Include/cpython/code.h index b0e226e0e1971a8..ef8f9304ccab567 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -226,7 +226,7 @@ struct PyCodeObject _PyCode_DEF(1); */ #define PY_PARSER_REQUIRES_FUTURE_KEYWORD -#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */ +#define CO_MAXBLOCKS 21 /* Max static block nesting within a function */ PyAPI_DATA(PyTypeObject) PyCode_Type; diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index e9bec3317811dd6..de783f714509a31 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -2392,13 +2392,40 @@ def bug(): code += "): yield a" return code - CO_MAXBLOCKS = 20 # static nesting limit of the compiler + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block - for n in range(CO_MAXBLOCKS): + for n in range(MAX_MANAGERS): with self.subTest(f"within range: {n=}"): compile(get_code(n), "", "exec") - for n in range(CO_MAXBLOCKS, CO_MAXBLOCKS + 5): + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): with self.subTest(f"out of range: {n=}"): self._check_error(get_code(n), "too many statically nested blocks") @@ -2536,7 +2563,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-16-23-29.gh-issue-116767.z9UFpr.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-16-23-29.gh-issue-116767.z9UFpr.rst new file mode 100644 index 000000000000000..cec2041d12ee150 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-16-23-29.gh-issue-116767.z9UFpr.rst @@ -0,0 +1 @@ +Fix crash in compiler on 'async with' that has many context managers. diff --git a/Python/compile.c b/Python/compile.c index ca5551a8e64ab03..4e94f9297a32d17 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -113,7 +113,8 @@ compiler IR. enum fblocktype { WHILE_LOOP, FOR_LOOP, TRY_EXCEPT, FINALLY_TRY, FINALLY_END, WITH, ASYNC_WITH, HANDLER_CLEANUP, POP_VALUE, EXCEPTION_HANDLER, - EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR }; + EXCEPTION_GROUP_HANDLER, ASYNC_COMPREHENSION_GENERATOR, + STOP_ITERATION }; struct fblockinfo { enum fblocktype fb_type; @@ -1503,6 +1504,7 @@ compiler_unwind_fblock(struct compiler *c, location *ploc, case EXCEPTION_HANDLER: case EXCEPTION_GROUP_HANDLER: case ASYNC_COMPREHENSION_GENERATOR: + case STOP_ITERATION: return SUCCESS; case FOR_LOOP: @@ -2232,14 +2234,26 @@ compiler_function_body(struct compiler *c, stmt_ty s, int is_async, Py_ssize_t f c->u->u_metadata.u_argcount = asdl_seq_LEN(args->args); c->u->u_metadata.u_posonlyargcount = asdl_seq_LEN(args->posonlyargs); c->u->u_metadata.u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs); + + NEW_JUMP_TARGET_LABEL(c, start); + USE_LABEL(c, start); + bool add_stopiteration_handler = c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator; + if (add_stopiteration_handler) { + /* wrap_in_stopiteration_handler will push a block, so we need to account for that */ + RETURN_IF_ERROR( + compiler_push_fblock(c, NO_LOCATION, STOP_ITERATION, + start, NO_LABEL, NULL)); + } + for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(body); i++) { VISIT_IN_SCOPE(c, stmt, (stmt_ty)asdl_seq_GET(body, i)); } - if (c->u->u_ste->ste_coroutine || c->u->u_ste->ste_generator) { + if (add_stopiteration_handler) { if (wrap_in_stopiteration_handler(c) < 0) { compiler_exit_scope(c); return ERROR; } + compiler_pop_fblock(c, STOP_ITERATION, start); } PyCodeObject *co = optimize_and_assemble(c, 1); compiler_exit_scope(c); From 4a08a75cf4c490f7c43ede69bdf6e5a79c6a3af3 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Wed, 1 May 2024 14:42:10 +0100 Subject: [PATCH 036/220] gh-99180: Remove traceback anchors in return and assign statements that cover all the displayed range (#112670) --- Lib/test/test_traceback.py | 233 +++++++++++++++--- Lib/traceback.py | 38 ++- ...3-12-03-18-21-59.gh-issue-99180.5m0V0q.rst | 2 + 3 files changed, 240 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-12-03-18-21-59.gh-issue-99180.5m0V0q.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 8927fccc2893209..41f82512480d4f6 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -695,7 +695,6 @@ def f_with_multiline(): ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' ' return compile(code, "?", "exec")\n' - ' ~~~~~~~^^^^^^^^^^^^^^^^^^^\n' ' File "?", line 7\n' ' foo(a, z\n' ' ^' @@ -785,8 +784,8 @@ def f_with_binary_operator(): def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): def f_with_binary_operator(): a = 1 - b = "" - return ( a ) +b + b = c = "" + return ( a ) +b + c lineno_f = f_with_binary_operator.__code__.co_firstlineno expected_error = ( @@ -795,7 +794,7 @@ def f_with_binary_operator(): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' - ' return ( a ) +b\n' + ' return ( a ) +b + c\n' ' ~~~~~~~~~~^~\n' ) result_lines = self.get_exception(f_with_binary_operator) @@ -983,7 +982,7 @@ def f1(a): def f2(b): raise RuntimeError("fail") return f2 - return f1("x")("y") + return f1("x")("y")("z") lineno_f = f_with_call.__code__.co_firstlineno expected_error = ( @@ -992,7 +991,7 @@ def f2(b): ' callable()\n' ' ~~~~~~~~^^\n' f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' - ' return f1("x")("y")\n' + ' return f1("x")("y")("z")\n' ' ~~~~~~~^^^^^\n' f' File "{__file__}", line {lineno_f+3}, in f2\n' ' raise RuntimeError("fail")\n' @@ -1507,6 +1506,184 @@ def f(): ' raise MemoryError()'] self.assertEqual(actual, expected) + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + @requires_debug_ranges() class PurePythonTracebackErrorCaretTests( @@ -1701,7 +1878,7 @@ def f(): # Check a known (limited) number of recursive invocations def g(count=10): if count: - return g(count-1) + return g(count-1) + 1 raise ValueError with captured_output("stderr") as stderr_g: @@ -1715,13 +1892,13 @@ def g(count=10): lineno_g = g.__code__.co_firstlineno result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1760,13 +1937,10 @@ def h(count=10): ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' - ' ~^^^^^^^^^\n' ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' @@ -1786,13 +1960,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -1800,7 +1974,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+80}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -1818,13 +1992,13 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' @@ -1833,7 +2007,7 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+112}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) @@ -4287,11 +4461,14 @@ def foo(*args): x = {'a':{'b': None}} y = x['a']['b']['c'] - def baz(*args): - return foo(1,2,3,4) + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) def bar(): - return baz(1, + return baz1(1, 2,3 ,4) try: @@ -4305,10 +4482,10 @@ def bar(): boldr = traceback._ANSIColors.BOLD_RED reset = traceback._ANSIColors.RESET self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) - self.assertIn("return " + red + "foo" + reset + boldr + "(1,2,3,4)" + reset, lines) - self.assertIn("return " + red + "baz" + reset + boldr + "(1," + reset, lines) - self.assertIn(boldr + "2,3" + reset, lines) - self.assertIn(boldr + ",4)" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) def test_colorized_syntax_error(self): diff --git a/Lib/traceback.py b/Lib/traceback.py index fccec0c71c3695d..6ce745f32d13242 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -617,13 +617,10 @@ def format_frame_summary(self, frame_summary, **kwargs): # attempt to parse for anchors anchors = None + show_carets = False with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment(segment) - - # only use carets if there are anchors or the carets do not span all lines - show_carets = False - if anchors or all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): - show_carets = True + show_carets = self.should_show_carets(start_offset, end_offset, all_lines, anchors) result = [] @@ -737,6 +734,37 @@ def output_line(lineno): return ''.join(row) + def should_show_carets(self, start_offset, end_offset, all_lines, anchors): + with suppress(SyntaxError, ImportError): + import ast + tree = ast.parse('\n'.join(all_lines)) + statement = tree.body[0] + value = None + def _spawns_full_line(value): + return ( + value.lineno == 1 + and value.end_lineno == len(all_lines) + and value.col_offset == start_offset + and value.end_col_offset == end_offset + ) + match statement: + case ast.Return(value=ast.Call()): + if isinstance(statement.value.func, ast.Name): + value = statement.value + case ast.Assign(value=ast.Call()): + if ( + len(statement.targets) == 1 and + isinstance(statement.targets[0], ast.Name) + ): + value = statement.value + if value is not None and _spawns_full_line(value): + return False + if anchors: + return True + if all_lines[0][:start_offset].lstrip() or all_lines[-1][end_offset:].rstrip(): + return True + return False + def format(self, **kwargs): """Format the stack ready for printing. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-12-03-18-21-59.gh-issue-99180.5m0V0q.rst b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-18-21-59.gh-issue-99180.5m0V0q.rst new file mode 100644 index 000000000000000..576626b702e75c5 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-12-03-18-21-59.gh-issue-99180.5m0V0q.rst @@ -0,0 +1,2 @@ +Elide uninformative traceback indicators in ``return`` and simple +``assignment`` statements. Patch by Pablo Galindo. From beb653cc24275025708758d444835db2ddbb74e4 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 2 May 2024 00:11:14 +1000 Subject: [PATCH 037/220] gh-117958: Expose JIT code via method in UOpExecutor (#117959) --- ...-04-18-03-49-41.gh-issue-117958.-EsfUs.rst | 2 ++ Python/optimizer.c | 25 ++++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-18-03-49-41.gh-issue-117958.-EsfUs.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-18-03-49-41.gh-issue-117958.-EsfUs.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-18-03-49-41.gh-issue-117958.-EsfUs.rst new file mode 100644 index 000000000000000..c127786bc129b13 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-18-03-49-41.gh-issue-117958.-EsfUs.rst @@ -0,0 +1,2 @@ +Added a ``get_jit_code()`` method to access JIT compiled machine code from the UOp Executor when the experimental JIT is enabled. Patch +by Anthony Shaw. diff --git a/Python/optimizer.c b/Python/optimizer.c index 6576aa1cddc033c..9ba8d84a47dcd9d 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -393,6 +393,29 @@ executor_traverse(PyObject *o, visitproc visit, void *arg) return 0; } +static PyObject * +get_jit_code(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifndef _Py_JIT + PyErr_SetString(PyExc_RuntimeError, "JIT support not enabled."); + return NULL; +#else + _PyExecutorObject *executor = (_PyExecutorObject *)self; + if (executor->jit_code == NULL || executor->jit_size == 0) { + Py_RETURN_NONE; + } + return PyBytes_FromStringAndSize(executor->jit_code, executor->jit_size); +#endif +} + +static PyMethodDef uop_executor_methods[] = { + { "is_valid", is_valid, METH_NOARGS, NULL }, + { "get_jit_code", get_jit_code, METH_NOARGS, NULL}, + { "get_opcode", get_opcode, METH_NOARGS, NULL }, + { "get_oparg", get_oparg, METH_NOARGS, NULL }, + { NULL, NULL }, +}; + static int executor_is_gc(PyObject *o) { @@ -407,7 +430,7 @@ PyTypeObject _PyUOpExecutor_Type = { .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, .tp_dealloc = (destructor)uop_dealloc, .tp_as_sequence = &uop_as_sequence, - .tp_methods = executor_methods, + .tp_methods = uop_executor_methods, .tp_traverse = executor_traverse, .tp_clear = (inquiry)executor_clear, .tp_is_gc = executor_is_gc, From 49baa656cb994122869bc807a88ea2f3f0d7751b Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 1 May 2024 08:05:53 -0700 Subject: [PATCH 038/220] GH-115802: Use the GHC calling convention in JIT code (GH-118287) --- Include/cpython/optimizer.h | 1 + Lib/test/test_perf_profiler.py | 2 +- Python/jit.c | 52 ++++++++++++++++++++++---------- Python/optimizer.c | 2 ++ Tools/jit/_targets.py | 54 ++++++++++++++++++++++++++++------ Tools/jit/_writer.py | 4 +++ Tools/jit/template.c | 4 +-- Tools/jit/trampoline.c | 25 ++++++++++++++++ 8 files changed, 117 insertions(+), 27 deletions(-) create mode 100644 Tools/jit/trampoline.c diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index a169280b26a6ad1..60b35747deb4f3d 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -102,6 +102,7 @@ typedef struct _PyExecutorObject { uint32_t code_size; size_t jit_size; void *jit_code; + void *jit_side_entry; _PyExitData exits[1]; } _PyExecutorObject; diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 040be63da114477..e7c03b990860136 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -216,7 +216,7 @@ def is_unwinding_reliable(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False - return "no-omit-frame-pointer" in cflags + return "no-omit-frame-pointer" in cflags and "_Py_JIT" not in cflags def perf_command_works(): diff --git a/Python/jit.c b/Python/jit.c index 75ec4fb9756eb73..df14e48c564447a 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -385,8 +385,8 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size { // Loop once to find the total compiled size: size_t instruction_starts[UOP_MAX_TRACE_LENGTH]; - size_t code_size = 0; - size_t data_size = 0; + size_t code_size = trampoline.code.body_size; + size_t data_size = trampoline.data.body_size; for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; @@ -408,11 +408,29 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; + { + // Compile the trampoline, which handles converting between the native + // calling convention and the calling convention used by jitted code + // (which may be different for efficiency reasons). On platforms where + // we don't change calling conventions, the trampoline is empty and + // nothing is emitted here: + const StencilGroup *group = &trampoline; + // Think of patches as a dictionary mapping HoleValue to uintptr_t: + uintptr_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uintptr_t)code; + patches[HoleValue_CONTINUE] = (uintptr_t)code + group->code.body_size; + patches[HoleValue_DATA] = (uintptr_t)data; + patches[HoleValue_EXECUTOR] = (uintptr_t)executor; + patches[HoleValue_TOP] = (uintptr_t)memory + trampoline.code.body_size; + patches[HoleValue_ZERO] = 0; + emit(group, patches); + code += group->code.body_size; + data += group->data.body_size; + } assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; const StencilGroup *group = &stencil_groups[instruction->opcode]; - // Think of patches as a dictionary mapping HoleValue to uintptr_t: uintptr_t patches[] = GET_PATCHES(); patches[HoleValue_CODE] = (uintptr_t)code; patches[HoleValue_CONTINUE] = (uintptr_t)code + group->code.body_size; @@ -454,18 +472,20 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size code += group->code.body_size; data += group->data.body_size; } - // Protect against accidental buffer overrun into data: - const StencilGroup *group = &stencil_groups[_FATAL_ERROR]; - uintptr_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uintptr_t)code; - patches[HoleValue_CONTINUE] = (uintptr_t)code; - patches[HoleValue_DATA] = (uintptr_t)data; - patches[HoleValue_EXECUTOR] = (uintptr_t)executor; - patches[HoleValue_TOP] = (uintptr_t)code; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; + { + // Protect against accidental buffer overrun into data: + const StencilGroup *group = &stencil_groups[_FATAL_ERROR]; + uintptr_t patches[] = GET_PATCHES(); + patches[HoleValue_CODE] = (uintptr_t)code; + patches[HoleValue_CONTINUE] = (uintptr_t)code; + patches[HoleValue_DATA] = (uintptr_t)data; + patches[HoleValue_EXECUTOR] = (uintptr_t)executor; + patches[HoleValue_TOP] = (uintptr_t)code; + patches[HoleValue_ZERO] = 0; + emit(group, patches); + code += group->code.body_size; + data += group->data.body_size; + } assert(code == memory + code_size); assert(data == memory + code_size + data_size); if (mark_executable(memory, total_size)) { @@ -473,6 +493,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size return -1; } executor->jit_code = memory; + executor->jit_side_entry = memory + trampoline.code.body_size; executor->jit_size = total_size; return 0; } @@ -484,6 +505,7 @@ _PyJIT_Free(_PyExecutorObject *executor) size_t size = executor->jit_size; if (memory) { executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (jit_free(memory, size)) { PyErr_WriteUnraisable(NULL); diff --git a/Python/optimizer.c b/Python/optimizer.c index 9ba8d84a47dcd9d..2389338531b0f35 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1188,6 +1188,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil #endif #ifdef _Py_JIT executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, length)) { Py_DECREF(executor); @@ -1219,6 +1220,7 @@ init_cold_exit_executor(_PyExecutorObject *executor, int oparg) #endif #ifdef _Py_JIT executor->jit_code = NULL; + executor->jit_side_entry = NULL; executor->jit_size = 0; if (_PyJIT_Compile(executor, executor->trace, 1)) { return -1; diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 91734b36b4ab1bc..274d17bcf38deb0 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -38,6 +38,7 @@ class _Target(typing.Generic[_S, _R]): _: dataclasses.KW_ONLY alignment: int = 1 args: typing.Sequence[str] = () + ghccc: bool = False prefix: str = "" debug: bool = False force: bool = False @@ -85,7 +86,11 @@ async def _parse(self, path: pathlib.Path) -> _stencils.StencilGroup: sections: list[dict[typing.Literal["Section"], _S]] = json.loads(output) for wrapped_section in sections: self._handle_section(wrapped_section["Section"], group) - assert group.symbols["_JIT_ENTRY"] == (_stencils.HoleValue.CODE, 0) + # The trampoline's entry point is just named "_ENTRY", since on some + # platforms we later assume that any function starting with "_JIT_" uses + # the GHC calling convention: + entry_symbol = "_JIT_ENTRY" if "_JIT_ENTRY" in group.symbols else "_ENTRY" + assert group.symbols[entry_symbol] == (_stencils.HoleValue.CODE, 0) if group.data.body: line = f"0: {str(bytes(group.data.body)).removeprefix('b')}" group.data.disassembly.append(line) @@ -103,6 +108,9 @@ def _handle_relocation( async def _compile( self, opname: str, c: pathlib.Path, tempdir: pathlib.Path ) -> _stencils.StencilGroup: + # "Compile" the trampoline to an empty stencil group if it's not needed: + if opname == "trampoline" and not self.ghccc: + return _stencils.StencilGroup() o = tempdir / f"{opname}.o" args = [ f"--target={self.triple}", @@ -130,13 +138,38 @@ async def _compile( "-fno-plt", # Don't call stack-smashing canaries that we can't find or patch: "-fno-stack-protector", - "-o", - f"{o}", "-std=c11", - f"{c}", *self.args, ] - await _llvm.run("clang", args, echo=self.verbose) + if self.ghccc: + # This is a bit of an ugly workaround, but it makes the code much + # smaller and faster, so it's worth it. We want to use the GHC + # calling convention, but Clang doesn't support it. So, we *first* + # compile the code to LLVM IR, perform some text replacements on the + # IR to change the calling convention(!), and then compile *that*. + # Once we have access to Clang 19, we can get rid of this and use + # __attribute__((preserve_none)) directly in the C code instead: + ll = tempdir / f"{opname}.ll" + args_ll = args + [ + # -fomit-frame-pointer is necessary because the GHC calling + # convention uses RBP to pass arguments: + "-S", "-emit-llvm", "-fomit-frame-pointer", "-o", f"{ll}", f"{c}" + ] + await _llvm.run("clang", args_ll, echo=self.verbose) + ir = ll.read_text() + # This handles declarations, definitions, and calls to named symbols + # starting with "_JIT_": + ir = re.sub(r"(((noalias|nonnull|noundef) )*ptr @_JIT_\w+\()", r"ghccc \1", ir) + # This handles calls to anonymous callees, since anything with + # "musttail" needs to use the same calling convention: + ir = ir.replace("musttail call", "musttail call ghccc") + # Sometimes *both* replacements happen at the same site, so fix it: + ir = ir.replace("ghccc ghccc", "ghccc") + ll.write_text(ir) + args_o = args + ["-Wno-unused-command-line-argument", "-o", f"{o}", f"{ll}"] + else: + args_o = args + ["-o", f"{o}", f"{c}"] + await _llvm.run("clang", args_o, echo=self.verbose) return await self._parse(o) async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: @@ -146,6 +179,8 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: with tempfile.TemporaryDirectory() as tempdir: work = pathlib.Path(tempdir).resolve() async with asyncio.TaskGroup() as group: + coro = self._compile("trampoline", TOOLS_JIT / "trampoline.c", work) + tasks.append(group.create_task(coro, name="trampoline")) for opname in opnames: coro = self._compile(opname, TOOLS_JIT_TEMPLATE_C, work) tasks.append(group.create_task(coro, name=opname)) @@ -445,6 +480,7 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" + # ghccc currently crashes Clang when combined with musttail on aarch64. :( if re.fullmatch(r"aarch64-apple-darwin.*", host): return _MachO(host, alignment=8, prefix="_") if re.fullmatch(r"aarch64-pc-windows-msvc", host): @@ -455,13 +491,13 @@ def get_target(host: str) -> _COFF | _ELF | _MachO: return _ELF(host, alignment=8, args=args) if re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] - return _COFF(host, args=args, prefix="_") + return _COFF(host, args=args, ghccc=True, prefix="_") if re.fullmatch(r"x86_64-apple-darwin.*", host): - return _MachO(host, prefix="_") + return _MachO(host, ghccc=True, prefix="_") if re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - return _COFF(host, args=args) + return _COFF(host, args=args, ghccc=True) if re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fpic"] - return _ELF(host, args=args) + return _ELF(host, args=args, ghccc=True) raise ValueError(host) diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index cbc1ed2fa6543af..6b36d8a9c66a3fb 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -53,9 +53,13 @@ def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: yield "" yield "static const StencilGroup stencil_groups[512] = {" for opname in opnames: + if opname == "trampoline": + continue yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," yield "};" yield "" + yield "static const StencilGroup trampoline = INIT_STENCIL_GROUP(trampoline);" + yield "" yield "#define GET_PATCHES() { \\" for value in _stencils.HoleValue: yield f" [HoleValue_{value.name}] = (uintptr_t)0xBADBADBADBADBADB, \\" diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 3e81fd15bb80935..0dd0744f7aec9ca 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -48,7 +48,7 @@ do { \ OPT_STAT_INC(traces_executed); \ __attribute__((musttail)) \ - return ((jit_func)((EXECUTOR)->jit_code))(frame, stack_pointer, tstate); \ + return ((jit_func)((EXECUTOR)->jit_side_entry))(frame, stack_pointer, tstate); \ } while (0) #undef GOTO_TIER_ONE @@ -65,7 +65,7 @@ do { \ #define PATCH_VALUE(TYPE, NAME, ALIAS) \ PyAPI_DATA(void) ALIAS; \ - TYPE NAME = (TYPE)(uint64_t)&ALIAS; + TYPE NAME = (TYPE)(uintptr_t)&ALIAS; #define PATCH_JUMP(ALIAS) \ do { \ diff --git a/Tools/jit/trampoline.c b/Tools/jit/trampoline.c new file mode 100644 index 000000000000000..01b3d63a6790ba3 --- /dev/null +++ b/Tools/jit/trampoline.c @@ -0,0 +1,25 @@ +#include "Python.h" + +#include "pycore_ceval.h" +#include "pycore_frame.h" +#include "pycore_jit.h" + +// This is where the calling convention changes, on platforms that require it. +// The actual change is patched in while the JIT compiler is being built, in +// Tools/jit/_targets.py. On other platforms, this function compiles to nothing. +_Py_CODEUNIT * +_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState *tstate) +{ + // This is subtle. The actual trace will return to us once it exits, so we + // need to make sure that we stay alive until then. If our trace side-exits + // into another trace, and this trace is then invalidated, the code for + // *this function* will be freed and we'll crash upon return: + PyAPI_DATA(void) _JIT_EXECUTOR; + PyObject *executor = (PyObject *)(uintptr_t)&_JIT_EXECUTOR; + Py_INCREF(executor); + // Note that this is *not* a tail call: + PyAPI_DATA(void) _JIT_CONTINUE; + _Py_CODEUNIT *target = ((jit_func)&_JIT_CONTINUE)(frame, stack_pointer, tstate); + Py_SETREF(tstate->previous_executor, executor); + return target; +} From 759e8e7ab83848c527a53d7b2051bc14ac7b7c76 Mon Sep 17 00:00:00 2001 From: Harmen Stoppels Date: Wed, 1 May 2024 18:01:47 +0200 Subject: [PATCH 039/220] gh-99730: urllib.request: Keep HEAD method on redirect (GH-99731) --- Lib/test/test_urllib2.py | 9 +++++++++ Lib/urllib/request.py | 1 + .../2022-11-23-17-16-31.gh-issue-99730.bDQdaX.rst | 1 + 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2022-11-23-17-16-31.gh-issue-99730.bDQdaX.rst diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 6febb491788b428..eed0599642edfb2 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -1402,6 +1402,15 @@ def http_open(self, req): request = handler.last_buf self.assertTrue(request.startswith(expected), repr(request)) + def test_redirect_head_request(self): + from_url = "http://example.com/a.html" + to_url = "http://example.com/b.html" + h = urllib.request.HTTPRedirectHandler() + req = Request(from_url, method="HEAD") + fp = MockFile() + new_req = h.redirect_request(req, fp, 302, "Found", {}, to_url) + self.assertEqual(new_req.get_method(), "HEAD") + def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index d22af6618d80f16..ac6719ce8541824 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -650,6 +650,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): newheaders = {k: v for k, v in req.headers.items() if k.lower() not in CONTENT_HEADERS} return Request(newurl, + method="HEAD" if m == "HEAD" else "GET", headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True) diff --git a/Misc/NEWS.d/next/Library/2022-11-23-17-16-31.gh-issue-99730.bDQdaX.rst b/Misc/NEWS.d/next/Library/2022-11-23-17-16-31.gh-issue-99730.bDQdaX.rst new file mode 100644 index 000000000000000..b5955879c21ce24 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-11-23-17-16-31.gh-issue-99730.bDQdaX.rst @@ -0,0 +1 @@ +HEAD requests are no longer upgraded to GET request during redirects in urllib. From 2d161cb899a0b32e80f0dc4f1c24b1df443d10a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 17:06:54 +0100 Subject: [PATCH 040/220] Bump mypy from 1.9.0 to 1.10.0 in /Tools (#118461) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 61e75cf396ccb40..e6864b16c560ee2 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,6 +1,6 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.9.0 +mypy==1.10.0 # needed for peg_generator: types-psutil==5.9.5.20240316 From 6d12f4469c5f9e24809df28b19900722d52af11b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 17:07:29 +0100 Subject: [PATCH 041/220] Bump types-setuptools from 69.2.0.20240317 to 69.5.0.20240423 in /Tools (#118463) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index e6864b16c560ee2..ddeff674e92457a 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -4,4 +4,4 @@ mypy==1.10.0 # needed for peg_generator: types-psutil==5.9.5.20240316 -types-setuptools==69.2.0.20240317 +types-setuptools==69.5.0.20240423 From 75955110a643875b5d096b6eda06dcc6e542e171 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 1 May 2024 17:47:54 +0100 Subject: [PATCH 042/220] gh-116622: Android sysconfig updates (#118352) --- Android/android-env.sh | 6 ++++++ Lib/sysconfig/__init__.py | 20 +++++++++++++++---- Lib/test/test_sysconfig.py | 20 +++++++++++++++++++ ...-04-27-20-34-56.gh-issue-116622.YlQgXv.rst | 2 ++ configure | 7 ++++++- configure.ac | 3 +++ 6 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-27-20-34-56.gh-issue-116622.YlQgXv.rst diff --git a/Android/android-env.sh b/Android/android-env.sh index 3ce3e035cfb8fe3..545d559d93ab36c 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -61,6 +61,12 @@ done export CFLAGS="" export LDFLAGS="-Wl,--build-id=sha1 -Wl,--no-rosegment" +# Unlike Linux, Android does not implicitly use a dlopened library to resolve +# relocations in subsequently-loaded libraries, even if RTLD_GLOBAL is used +# (https://github.com/android/ndk/issues/1244). So any library that fails to +# build with this flag, would also fail to load at runtime. +LDFLAGS="$LDFLAGS -Wl,--no-undefined" + # Many packages get away with omitting -lm on Linux, but Android is stricter. LDFLAGS="$LDFLAGS -lm" diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 70bdecf2138fd95..98a14e5d3a3187b 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -601,10 +601,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 61c6a5a42502e72..9233304c6a53276 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -232,6 +232,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -347,6 +352,21 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") diff --git a/Misc/NEWS.d/next/Library/2024-04-27-20-34-56.gh-issue-116622.YlQgXv.rst b/Misc/NEWS.d/next/Library/2024-04-27-20-34-56.gh-issue-116622.YlQgXv.rst new file mode 100644 index 000000000000000..c7c57b6dbc515eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-27-20-34-56.gh-issue-116622.YlQgXv.rst @@ -0,0 +1,2 @@ +On Android, :any:`sysconfig.get_platform` now returns the format specified +by :pep:`738`. diff --git a/configure b/configure index 01c00d2198548f5..24e7396e389fd1e 100755 --- a/configure +++ b/configure @@ -7018,8 +7018,13 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( aarch64-apple-ios*/clang) : PY_SUPPORT_TIER=3 ;; #( + aarch64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( + x86_64-*-linux-android/clang) : + PY_SUPPORT_TIER=3 ;; #( *) : - PY_SUPPORT_TIER=0 + + PY_SUPPORT_TIER=0 ;; esac diff --git a/configure.ac b/configure.ac index ae65efeac55366a..0f1177872d91372 100644 --- a/configure.ac +++ b/configure.ac @@ -1148,6 +1148,9 @@ AS_CASE([$host/$ac_cv_cc_name], [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [PY_SUPPORT_TIER=0] ) From a8bcf3ec3284f042dab9cad676951092fb1f9890 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 1 May 2024 13:04:02 -0400 Subject: [PATCH 043/220] Expand the 'Extending' docs with an example. (#113187) * Expand the 'Extending' docs to provide a minimal example. Closes python/importlib_metadata#427. Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/importlib.metadata.rst | 78 ++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 5039bc6e85ee206..674ce5807fdf11e 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -406,6 +406,84 @@ metadata in locations other than the file system, subclass a custom finder, return instances of this derived ``Distribution`` in the ``find_distributions()`` method. +Example +------- + +Consider for example a custom finder that loads Python +modules from a database:: + + class DatabaseImporter(importlib.abc.MetaPathFinder): + def __init__(self, db): + self.db = db + + def find_spec(self, fullname, target=None) -> ModuleSpec: + return self.db.spec_from_name(fullname) + + sys.meta_path.append(DatabaseImporter(connect_db(...))) + +That importer now presumably provides importable modules from a +database, but it provides no metadata or entry points. For this +custom importer to provide metadata, it would also need to implement +``DistributionFinder``:: + + from importlib.metadata import DistributionFinder + + class DatabaseImporter(DistributionFinder): + ... + + def find_distributions(self, context=DistributionFinder.Context()): + query = dict(name=context.name) if context.name else {} + for dist_record in self.db.query_distributions(query): + yield DatabaseDistribution(dist_record) + +In this way, ``query_distributions`` would return records for +each distribution served by the database matching the query. For +example, if ``requests-1.0`` is in the database, ``find_distributions`` +would yield a ``DatabaseDistribution`` for ``Context(name='requests')`` +or ``Context(name=None)``. + +For the sake of simplicity, this example ignores ``context.path``\. The +``path`` attribute defaults to ``sys.path`` and is the set of import paths to +be considered in the search. A ``DatabaseImporter`` could potentially function +without any concern for a search path. Assuming the importer does no +partitioning, the "path" would be irrelevant. In order to illustrate the +purpose of ``path``, the example would need to illustrate a more complex +``DatabaseImporter`` whose behavior varied depending on +``sys.path``/``PYTHONPATH``. In that case, the ``find_distributions`` should +honor the ``context.path`` and only yield ``Distribution``\ s pertinent to that +path. + +``DatabaseDistribution``, then, would look something like:: + + class DatabaseDistribution(importlib.metadata.Distributon): + def __init__(self, record): + self.record = record + + def read_text(self, filename): + """ + Read a file like "METADATA" for the current distribution. + """ + if filename == "METADATA": + return f"""Name: {self.record.name} + Version: {self.record.version} + """ + if filename == "entry_points.txt": + return "\n".join( + f"""[{ep.group}]\n{ep.name}={ep.value}""" + for ep in self.record.entry_points) + + def locate_file(self, path): + raise RuntimeError("This distribution has no file system") + +This basic implementation should provide metadata and entry points for +packages served by the ``DatabaseImporter``, assuming that the +``record`` supplies suitable ``.name``, ``.version``, and +``.entry_points`` attributes. + +The ``DatabaseDistribution`` may also provide other metadata files, like +``RECORD`` (required for ``Distribution.files``) or override the +implementation of ``Distribution.files``. See the source for more inspiration. + .. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points .. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api From b52c753e0fb8b87c7a0e3f1c72acd18327d8d16b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 1 May 2024 20:05:01 +0200 Subject: [PATCH 044/220] gh-110850: Add PyTime_TimeRaw() function (#118394) Add "Raw" variant of PyTime functions: * PyTime_MonotonicRaw() * PyTime_PerfCounterRaw() * PyTime_TimeRaw() Changes: * Add documentation and tests. Tests release the GIL while calling raw clock functions. * py_get_system_clock() and py_get_monotonic_clock() now check that the GIL is hold by the caller if raise_exc is non-zero. * Reimplement "Unchecked" functions with raw clock functions. Co-authored-by: Petr Viktorin --- Doc/c-api/time.rst | 29 ++++++ Doc/whatsnew/3.13.rst | 12 ++- Include/cpython/pytime.h | 4 + Lib/test/test_capi/test_time.py | 19 ++-- ...-04-29-17-19-07.gh-issue-110850.vcpLn1.rst | 7 ++ Modules/_testcapi/time.c | 60 ++++++++++++ Python/pytime.c | 93 ++++++++++++++----- 7 files changed, 189 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2024-04-29-17-19-07.gh-issue-110850.vcpLn1.rst diff --git a/Doc/c-api/time.rst b/Doc/c-api/time.rst index 7791cdb17810550..5cfdef71b3e191e 100644 --- a/Doc/c-api/time.rst +++ b/Doc/c-api/time.rst @@ -72,6 +72,35 @@ with the :term:`GIL` held. See :func:`time.time` for details important on this clock. +Raw Clock Functions +------------------- + +Similar to clock functions, but don't set an exception on error and don't +require the caller to hold the GIL. + +On success, the functions return ``0``. + +On failure, they set ``*result`` to ``0`` and return ``-1``, *without* setting +an exception. To get the cause of the error, acquire the GIL and call the +regular (non-``Raw``) function. Note that the regular function may succeed after +the ``Raw`` one failed. + +.. c:function:: int PyTime_MonotonicRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Monotonic`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_PerfCounterRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_PerfCounter`, + but don't set an exception on error and don't require holding the GIL. + +.. c:function:: int PyTime_TimeRaw(PyTime_t *result) + + Similar to :c:func:`PyTime_Time`, + but don't set an exception on error and don't require holding the GIL. + + Conversion functions -------------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index ee50effd662f12e..5a169556066421a 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1901,9 +1901,15 @@ New Features * :c:type:`PyTime_t` type. * :c:var:`PyTime_MIN` and :c:var:`PyTime_MAX` constants. - * :c:func:`PyTime_AsSecondsDouble` - :c:func:`PyTime_Monotonic`, :c:func:`PyTime_PerfCounter`, and - :c:func:`PyTime_Time` functions. + * Add functions: + + * :c:func:`PyTime_AsSecondsDouble`. + * :c:func:`PyTime_Monotonic`. + * :c:func:`PyTime_MonotonicRaw`. + * :c:func:`PyTime_PerfCounter`. + * :c:func:`PyTime_PerfCounterRaw`. + * :c:func:`PyTime_Time`. + * :c:func:`PyTime_TimeRaw`. (Contributed by Victor Stinner and Petr Viktorin in :gh:`110850`.) diff --git a/Include/cpython/pytime.h b/Include/cpython/pytime.h index d8244700d614ceb..5c68110aeedb867 100644 --- a/Include/cpython/pytime.h +++ b/Include/cpython/pytime.h @@ -16,6 +16,10 @@ PyAPI_FUNC(int) PyTime_Monotonic(PyTime_t *result); PyAPI_FUNC(int) PyTime_PerfCounter(PyTime_t *result); PyAPI_FUNC(int) PyTime_Time(PyTime_t *result); +PyAPI_FUNC(int) PyTime_MonotonicRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_PerfCounterRaw(PyTime_t *result); +PyAPI_FUNC(int) PyTime_TimeRaw(PyTime_t *result); + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_time.py b/Lib/test/test_capi/test_time.py index 10b7fbf2c372a3b..17ebd7c1962d483 100644 --- a/Lib/test/test_capi/test_time.py +++ b/Lib/test/test_capi/test_time.py @@ -18,11 +18,6 @@ def test_min_max(self): self.assertEqual(PyTime_MIN, -2**63) self.assertEqual(PyTime_MAX, 2**63 - 1) - def check_clock(self, c_func, py_func): - t1 = c_func() - t2 = py_func() - self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) - def test_assecondsdouble(self): # Test PyTime_AsSecondsDouble() def ns_to_sec(ns): @@ -58,14 +53,22 @@ def ns_to_sec(ns): self.assertEqual(_testcapi.PyTime_AsSecondsDouble(ns), ns_to_sec(ns)) + def check_clock(self, c_func, py_func): + t1 = c_func() + t2 = py_func() + self.assertAlmostEqual(t1, t2, delta=CLOCK_RES) + def test_monotonic(self): - # Test PyTime_Monotonic() + # Test PyTime_Monotonic() and PyTime_MonotonicRaw() self.check_clock(_testcapi.PyTime_Monotonic, time.monotonic) + self.check_clock(_testcapi.PyTime_MonotonicRaw, time.monotonic) def test_perf_counter(self): - # Test PyTime_PerfCounter() + # Test PyTime_PerfCounter() and PyTime_PerfCounterRaw() self.check_clock(_testcapi.PyTime_PerfCounter, time.perf_counter) + self.check_clock(_testcapi.PyTime_PerfCounterRaw, time.perf_counter) def test_time(self): - # Test PyTime_time() + # Test PyTime_Time() and PyTime_TimeRaw() self.check_clock(_testcapi.PyTime_Time, time.time) + self.check_clock(_testcapi.PyTime_TimeRaw, time.time) diff --git a/Misc/NEWS.d/next/C API/2024-04-29-17-19-07.gh-issue-110850.vcpLn1.rst b/Misc/NEWS.d/next/C API/2024-04-29-17-19-07.gh-issue-110850.vcpLn1.rst new file mode 100644 index 000000000000000..786da0147eae46c --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-04-29-17-19-07.gh-issue-110850.vcpLn1.rst @@ -0,0 +1,7 @@ +Add "Raw" variant of PyTime functions + +* :c:func:`PyTime_MonotonicRaw` +* :c:func:`PyTime_PerfCounterRaw` +* :c:func:`PyTime_TimeRaw` + +Patch by Victor Stinner. diff --git a/Modules/_testcapi/time.c b/Modules/_testcapi/time.c index 68f082bf3f3d88a..464cf5c3125012e 100644 --- a/Modules/_testcapi/time.c +++ b/Modules/_testcapi/time.c @@ -51,6 +51,25 @@ test_pytime_monotonic(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_Monotonic(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_monotonic_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_MonotonicRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_MonotonicRaw() failed"); return NULL; } assert(res == 0); @@ -64,6 +83,25 @@ test_pytime_perf_counter(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_PerfCounter(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_perf_counter_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_PerfCounterRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_PerfCounterRaw() failed"); return NULL; } assert(res == 0); @@ -77,6 +115,25 @@ test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) PyTime_t t; int res = PyTime_Time(&t); if (res < 0) { + assert(t == 0); + return NULL; + } + assert(res == 0); + return pytime_as_float(t); +} + + +static PyObject* +test_pytime_time_raw(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyTime_t t; + int res; + Py_BEGIN_ALLOW_THREADS + res = PyTime_TimeRaw(&t); + Py_END_ALLOW_THREADS + if (res < 0) { + assert(t == 0); + PyErr_SetString(PyExc_RuntimeError, "PyTime_TimeRaw() failed"); return NULL; } assert(res == 0); @@ -87,8 +144,11 @@ test_pytime_time(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) static PyMethodDef test_methods[] = { {"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS}, {"PyTime_Monotonic", test_pytime_monotonic, METH_NOARGS}, + {"PyTime_MonotonicRaw", test_pytime_monotonic_raw, METH_NOARGS}, {"PyTime_PerfCounter", test_pytime_perf_counter, METH_NOARGS}, + {"PyTime_PerfCounterRaw", test_pytime_perf_counter_raw, METH_NOARGS}, {"PyTime_Time", test_pytime_time, METH_NOARGS}, + {"PyTime_TimeRaw", test_pytime_time_raw, METH_NOARGS}, {NULL}, }; diff --git a/Python/pytime.c b/Python/pytime.c index d5b38047b6db31a..12b36bbc881f9ad 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -898,6 +898,10 @@ static int py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); + if (raise_exc) { + // raise_exc requires to hold the GIL + assert(PyGILState_Check()); + } #ifdef MS_WINDOWS FILETIME system_time; @@ -1004,29 +1008,44 @@ py_get_system_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -PyTime_t -_PyTime_TimeUnchecked(void) +int +PyTime_Time(PyTime_t *result) { - PyTime_t t; - if (py_get_system_clock(&t, NULL, 0) < 0) { - // If clock_gettime(CLOCK_REALTIME) or gettimeofday() fails: - // silently ignore the failure and return 0. - t = 0; + if (py_get_system_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -PyTime_Time(PyTime_t *result) +PyTime_TimeRaw(PyTime_t *result) { - if (py_get_system_clock(result, NULL, 1) < 0) { + if (py_get_system_clock(result, NULL, 0) < 0) { *result = 0; return -1; } return 0; } + +PyTime_t +_PyTime_TimeUnchecked(void) +{ + PyTime_t t; +#ifdef Py_DEBUG + int result = PyTime_TimeRaw(&t); + if (result != 0) { + Py_FatalError("unable to read the system clock"); + } +#else + (void)PyTime_TimeRaw(&t); +#endif + return t; +} + + int _PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { @@ -1140,6 +1159,10 @@ static int py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) { assert(info == NULL || raise_exc); + if (raise_exc) { + // raise_exc requires to hold the GIL + assert(PyGILState_Check()); + } #if defined(MS_WINDOWS) if (py_get_win_perf_counter(tp, info, raise_exc) < 0) { @@ -1225,22 +1248,21 @@ py_get_monotonic_clock(PyTime_t *tp, _Py_clock_info_t *info, int raise_exc) } -PyTime_t -_PyTime_MonotonicUnchecked(void) +int +PyTime_Monotonic(PyTime_t *result) { - PyTime_t t; - if (py_get_monotonic_clock(&t, NULL, 0) < 0) { - // Ignore silently the error and return 0. - t = 0; + if (py_get_monotonic_clock(result, NULL, 1) < 0) { + *result = 0; + return -1; } - return t; + return 0; } int -PyTime_Monotonic(PyTime_t *result) +PyTime_MonotonicRaw(PyTime_t *result) { - if (py_get_monotonic_clock(result, NULL, 1) < 0) { + if (py_get_monotonic_clock(result, NULL, 0) < 0) { *result = 0; return -1; } @@ -1248,6 +1270,22 @@ PyTime_Monotonic(PyTime_t *result) } +PyTime_t +_PyTime_MonotonicUnchecked(void) +{ + PyTime_t t; +#ifdef Py_DEBUG + int result = PyTime_MonotonicRaw(&t); + if (result != 0) { + Py_FatalError("unable to read the monotonic clock"); + } +#else + (void)PyTime_MonotonicRaw(&t); +#endif + return t; +} + + int _PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { @@ -1262,17 +1300,24 @@ _PyTime_PerfCounterWithInfo(PyTime_t *t, _Py_clock_info_t *info) } -PyTime_t -_PyTime_PerfCounterUnchecked(void) +int +PyTime_PerfCounter(PyTime_t *result) { - return _PyTime_MonotonicUnchecked(); + return PyTime_Monotonic(result); } int -PyTime_PerfCounter(PyTime_t *result) +PyTime_PerfCounterRaw(PyTime_t *result) { - return PyTime_Monotonic(result); + return PyTime_MonotonicRaw(result); +} + + +PyTime_t +_PyTime_PerfCounterUnchecked(void) +{ + return _PyTime_MonotonicUnchecked(); } From 164e2c31c073b5fd8d25b3432a5e5c721f747393 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 1 May 2024 21:25:11 +0300 Subject: [PATCH 045/220] gh-117225: Document colour use in `doctest` (#118268) --- Doc/using/cmdline.rst | 12 ++++++++++-- Doc/whatsnew/3.13.rst | 16 +++++++++++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 295e3fb09830ce3..1f5d02d1ea54a23 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -632,11 +632,11 @@ behavior can be controlled by setting different environment variables. Setting the environment variable ``TERM`` to ``dumb`` will disable color. -If the environment variable ``FORCE_COLOR`` is set, then color will be +If the |FORCE_COLOR|_ environment variable is set, then color will be enabled regardless of the value of TERM. This is useful on CI systems which aren’t terminals but can still display ANSI escape sequences. -If the environment variable ``NO_COLOR`` is set, Python will disable all color +If the |NO_COLOR|_ environment variable is set, Python will disable all color in the output. This takes precedence over ``FORCE_COLOR``. All these environment variables are used also by other tools to control color @@ -645,6 +645,14 @@ output. To control the color output only in the Python interpreter, the precedence over ``NO_COLOR``, which in turn takes precedence over ``FORCE_COLOR``. +.. Apparently this how you hack together a formatted link: + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5a169556066421a..d67df8109a8ced3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -100,10 +100,18 @@ Improved Error Messages * The interpreter now colorizes error messages when displaying tracebacks by default. This feature can be controlled via the new :envvar:`PYTHON_COLORS` environment - variable as well as the canonical ``NO_COLOR`` and ``FORCE_COLOR`` environment + variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. See also :ref:`using-on-controlling-color`. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) +.. Apparently this how you hack together a formatted link: + +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ + +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ + * A common mistake is to write a script with the same name as a standard library module. When this results in errors, we now display a more helpful error message: @@ -439,6 +447,12 @@ doctest :attr:`doctest.TestResults.skipped` attributes. (Contributed by Victor Stinner in :gh:`108794`.) +* Color is added to the output by default. + This can be controlled via the new :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment + variables. See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`117225`.) + email ----- From 3b3f8dea575d81a9e68f04b434942d06bd03fad5 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 1 May 2024 21:27:06 +0300 Subject: [PATCH 046/220] gh-117225: Move colorize functionality to own internal module (#118283) --- Lib/_colorize.py | 64 ++++++++++++++++++ Lib/doctest.py | 38 +++++------ Lib/test/support/__init__.py | 9 +-- Lib/test/test__colorize.py | 59 ++++++++++++++++ Lib/test/test_doctest/test_doctest.py | 56 ++++++++-------- Lib/test/test_traceback.py | 75 +++++---------------- Lib/traceback.py | 96 ++++++++------------------- Python/stdlib_module_names.h | 1 + 8 files changed, 218 insertions(+), 180 deletions(-) create mode 100644 Lib/_colorize.py create mode 100644 Lib/test/test__colorize.py diff --git a/Lib/_colorize.py b/Lib/_colorize.py new file mode 100644 index 000000000000000..845fb57a90abb8e --- /dev/null +++ b/Lib/_colorize.py @@ -0,0 +1,64 @@ +import io +import os +import sys + +COLORIZE = True + + +class ANSIColors: + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + GREEN = "\x1b[32m" + GREY = "\x1b[90m" + MAGENTA = "\x1b[35m" + RED = "\x1b[31m" + RESET = "\x1b[0m" + YELLOW = "\x1b[33m" + + +NoColors = ANSIColors() + +for attr in dir(NoColors): + if not attr.startswith("__"): + setattr(NoColors, attr, "") + + +def get_colors(colorize: bool = False) -> ANSIColors: + if colorize or can_colorize(): + return ANSIColors() + else: + return NoColors + + +def can_colorize() -> bool: + if sys.platform == "win32": + try: + import nt + + if not nt._supports_virtual_terminal(): + return False + except (ImportError, AttributeError): + return False + if not sys.flags.ignore_environment: + if os.environ.get("PYTHON_COLORS") == "0": + return False + if os.environ.get("PYTHON_COLORS") == "1": + return True + if "NO_COLOR" in os.environ: + return False + if not COLORIZE: + return False + if not sys.flags.ignore_environment: + if "FORCE_COLOR" in os.environ: + return True + if os.environ.get("TERM") == "dumb": + return False + + if not hasattr(sys.stderr, "fileno"): + return False + + try: + return os.isatty(sys.stderr.fileno()) + except io.UnsupportedOperation: + return sys.stderr.isatty() diff --git a/Lib/doctest.py b/Lib/doctest.py index d8c632e47e7b7a3..c531e3ca6a3d5e3 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -104,7 +104,8 @@ def _test(): import unittest from io import StringIO, IncrementalNewlineDecoder from collections import namedtuple -from traceback import _ANSIColors, _can_colorize +import _colorize # Used in doctests +from _colorize import ANSIColors, can_colorize class TestResults(namedtuple('TestResults', 'failed attempted')): @@ -1180,8 +1181,8 @@ class DocTestRunner: The `run` method is used to process a single DocTest case. It returns a TestResults instance. - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> tests = DocTestFinder().find(_TestClass) >>> runner = DocTestRunner(verbose=False) @@ -1234,7 +1235,7 @@ class DocTestRunner: overriding the methods `report_start`, `report_success`, `report_unexpected_exception`, and `report_failure`. - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ # This divider string is used to separate failure messages, and to # separate sections of the summary. @@ -1314,7 +1315,7 @@ def report_unexpected_exception(self, out, test, example, exc_info): def _failure_header(self, test, example): red, reset = ( - (_ANSIColors.RED, _ANSIColors.RESET) if _can_colorize() else ("", "") + (ANSIColors.RED, ANSIColors.RESET) if can_colorize() else ("", "") ) out = [f"{red}{self.DIVIDER}{reset}"] if test.filename: @@ -1556,8 +1557,8 @@ def out(s): # Make sure sys.displayhook just prints the value to stdout save_displayhook = sys.displayhook sys.displayhook = sys.__displayhook__ - saved_can_colorize = traceback._can_colorize - traceback._can_colorize = lambda: False + saved_can_colorize = _colorize.can_colorize + _colorize.can_colorize = lambda: False color_variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} for key in color_variables: color_variables[key] = os.environ.pop(key, None) @@ -1569,7 +1570,7 @@ def out(s): sys.settrace(save_trace) linecache.getlines = self.save_linecache_getlines sys.displayhook = save_displayhook - traceback._can_colorize = saved_can_colorize + _colorize.can_colorize = saved_can_colorize for key, value in color_variables.items(): if value is not None: os.environ[key] = value @@ -1609,20 +1610,13 @@ def summarize(self, verbose=None): else: failed.append((name, (failures, tries, skips))) - if _can_colorize(): - bold_green = _ANSIColors.BOLD_GREEN - bold_red = _ANSIColors.BOLD_RED - green = _ANSIColors.GREEN - red = _ANSIColors.RED - reset = _ANSIColors.RESET - yellow = _ANSIColors.YELLOW - else: - bold_green = "" - bold_red = "" - green = "" - red = "" - reset = "" - yellow = "" + ansi = _colorize.get_colors() + bold_green = ansi.BOLD_GREEN + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW if verbose: if notests: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 52573e665a1273f..999fffb03ed59a1 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2579,20 +2579,21 @@ def copy_python_src_ignore(path, names): } return ignored + def force_not_colorized(func): """Force the terminal not to be colorized.""" @functools.wraps(func) def wrapper(*args, **kwargs): - import traceback - original_fn = traceback._can_colorize + import _colorize + original_fn = _colorize.can_colorize variables = {"PYTHON_COLORS": None, "FORCE_COLOR": None} try: for key in variables: variables[key] = os.environ.pop(key, None) - traceback._can_colorize = lambda: False + _colorize.can_colorize = lambda: False return func(*args, **kwargs) finally: - traceback._can_colorize = original_fn + _colorize.can_colorize = original_fn for key, value in variables.items(): if value is not None: os.environ[key] = value diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py new file mode 100644 index 000000000000000..d55b97ade68ceff --- /dev/null +++ b/Lib/test/test__colorize.py @@ -0,0 +1,59 @@ +import contextlib +import sys +import unittest +import unittest.mock +import _colorize +from test.support import force_not_colorized + +ORIGINAL_CAN_COLORIZE = _colorize.can_colorize + + +def setUpModule(): + _colorize.can_colorize = lambda: False + + +def tearDownModule(): + _colorize.can_colorize = ORIGINAL_CAN_COLORIZE + + +class TestColorizeFunction(unittest.TestCase): + @force_not_colorized + def test_colorized_detection_checks_for_environment_variables(self): + if sys.platform == "win32": + virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", + return_value=True) + else: + virtual_patching = contextlib.nullcontext() + with virtual_patching: + + flags = unittest.mock.MagicMock(ignore_environment=False) + with (unittest.mock.patch("os.isatty") as isatty_mock, + unittest.mock.patch("sys.flags", flags), + unittest.mock.patch("_colorize.can_colorize", ORIGINAL_CAN_COLORIZE)): + isatty_mock.return_value = True + with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), True) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): + self.assertEqual(_colorize.can_colorize(), False) + with unittest.mock.patch("os.environ", + {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): + self.assertEqual(_colorize.can_colorize(), False) + isatty_mock.return_value = False + with unittest.mock.patch("os.environ", {}): + self.assertEqual(_colorize.can_colorize(), False) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 0f1e584e22a888f..3a173e823dd9a3f 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -16,7 +16,7 @@ import tempfile import types import contextlib -import traceback +import _colorize def doctest_skip_if(condition): @@ -893,8 +893,8 @@ def basics(): r""" DocTestRunner is used to run DocTest test cases, and to accumulate statistics. Here's a simple DocTest case we can use: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... ''' @@ -951,7 +951,7 @@ def basics(): r""" ok TestResults(failed=1, attempted=3) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def verbose_flag(): r""" The `verbose` flag makes the test runner generate more detailed @@ -1027,8 +1027,8 @@ def exceptions(): r""" lines between the first line and the type/value may be omitted or replaced with any other string: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... ''' @@ -1261,7 +1261,7 @@ def exceptions(): r""" ZeroDivisionError: integer division or modulo by zero TestResults(failed=1, attempted=1) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def displayhook(): r""" Test that changing sys.displayhook doesn't matter for doctest. @@ -1303,8 +1303,8 @@ def optionflags(): r""" The DONT_ACCEPT_TRUE_FOR_1 flag disables matches between True/False and 1/0: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... '>>> True\n1\n' @@ -1725,7 +1725,7 @@ def optionflags(): r""" Clean up. >>> del doctest.OPTIONFLAGS_BY_NAME[unlikely] - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ @@ -1736,8 +1736,8 @@ def option_directives(): r""" single example. To turn an option on for an example, follow that example with a comment of the form ``# doctest: +OPTION``: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): r''' ... >>> print(list(range(10))) # should fail: no ellipsis @@ -1947,7 +1947,7 @@ def option_directives(): r""" Traceback (most recent call last): ValueError: line 0 of the doctest for s has an option directive on a line with no example: '# doctest: +ELLIPSIS' - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def test_testsource(): r""" @@ -2031,8 +2031,8 @@ def test_pdb_set_trace(): with a version that restores stdout. This is necessary for you to see debugger output. - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> doc = ''' ... >>> x = 42 @@ -2157,7 +2157,7 @@ def test_pdb_set_trace(): 9 TestResults(failed=1, attempted=3) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ def test_pdb_set_trace_nested(): @@ -2694,8 +2694,8 @@ def test_testfile(): r""" We don't want color or `-v` in sys.argv for these tests. - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> save_argv = sys.argv >>> if '-v' in sys.argv: @@ -2863,7 +2863,7 @@ def test_testfile(): r""" TestResults(failed=0, attempted=2) >>> doctest.master = None # Reset master. >>> sys.argv = save_argv - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ class TestImporter(importlib.abc.MetaPathFinder, importlib.abc.ResourceLoader): @@ -3001,8 +3001,8 @@ def test_testmod(): r""" def test_unicode(): """ Check doctest with a non-ascii filename: - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> doc = ''' ... >>> raise Exception('clé') @@ -3030,7 +3030,7 @@ def test_unicode(): """ Exception: clé TestResults(failed=1, attempted=1) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ @@ -3325,8 +3325,8 @@ def test_run_doctestsuite_multiple_times(): def test_exception_with_note(note): """ - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> test_exception_with_note('Note') Traceback (most recent call last): @@ -3378,7 +3378,7 @@ def test_exception_with_note(note): note TestResults(failed=1, attempted=...) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ exc = ValueError('Text') exc.add_note(note) @@ -3459,8 +3459,8 @@ def test_syntax_error_subclass_from_stdlib(): def test_syntax_error_with_incorrect_expected_note(): """ - >>> save_colorize = traceback._COLORIZE - >>> traceback._COLORIZE = False + >>> save_colorize = _colorize.COLORIZE + >>> _colorize.COLORIZE = False >>> def f(x): ... r''' @@ -3491,7 +3491,7 @@ def test_syntax_error_with_incorrect_expected_note(): note2 TestResults(failed=1, attempted=...) - >>> traceback._COLORIZE = save_colorize + >>> _colorize.COLORIZE = save_colorize """ diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 41f82512480d4f6..8969e0174c98c1e 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -26,9 +26,9 @@ import json import textwrap import traceback -import contextlib from functools import partial from pathlib import Path +import _colorize MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' @@ -40,25 +40,18 @@ LEVENSHTEIN_DATA_FILE = Path(__file__).parent / 'levenshtein_examples.json' -ORIGINAL_CAN_COLORIZE = traceback._can_colorize - -def setUpModule(): - traceback._can_colorize = lambda: False - -def tearDownModule(): - traceback._can_colorize = ORIGINAL_CAN_COLORIZE class TracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. def setUp(self): super().setUp() - self.colorize = traceback._COLORIZE - traceback._COLORIZE = False + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False def tearDown(self): super().tearDown() - traceback._COLORIZE = self.colorize + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -4478,9 +4471,9 @@ def bar(): e, capture_locals=True ) lines = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) @@ -4496,11 +4489,11 @@ def test_colorized_syntax_error(self): e, capture_locals=True ) actual = "".join(exc.format(colorize=True)) - red = traceback._ANSIColors.RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - boldr = traceback._ANSIColors.BOLD_RED - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET expected = "".join([ f' File {magenta}""{reset}, line {magenta}1{reset}\n', f' a {boldr}${reset} b\n', @@ -4519,15 +4512,15 @@ def foo(): self.fail("No exception thrown.") except Exception as e: with captured_output("stderr") as tbstderr: - with unittest.mock.patch('traceback._can_colorize', return_value=True): + with unittest.mock.patch('_colorize.can_colorize', return_value=True): exception_print(e) actual = tbstderr.getvalue().splitlines() - red = traceback._ANSIColors.RED - boldr = traceback._ANSIColors.BOLD_RED - magenta = traceback._ANSIColors.MAGENTA - boldm = traceback._ANSIColors.BOLD_MAGENTA - reset = traceback._ANSIColors.RESET + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET lno_foo = foo.__code__.co_firstlineno expected = ['Traceback (most recent call last):', f' File {magenta}"{__file__}"{reset}, ' @@ -4541,38 +4534,6 @@ def foo(): f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] self.assertEqual(actual, expected) - @force_not_colorized - def test_colorized_detection_checks_for_environment_variables(self): - if sys.platform == "win32": - virtual_patching = unittest.mock.patch("nt._supports_virtual_terminal", return_value=True) - else: - virtual_patching = contextlib.nullcontext() - with virtual_patching: - - flags = unittest.mock.MagicMock(ignore_environment=False) - with (unittest.mock.patch("os.isatty") as isatty_mock, - unittest.mock.patch("sys.flags", flags), - unittest.mock.patch("traceback._can_colorize", ORIGINAL_CAN_COLORIZE)): - isatty_mock.return_value = True - with unittest.mock.patch("os.environ", {'TERM': 'dumb'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'PYTHON_COLORS': '0'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'NO_COLOR': '1', "PYTHON_COLORS": '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), True) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', 'NO_COLOR': '1'}): - self.assertEqual(traceback._can_colorize(), False) - with unittest.mock.patch("os.environ", {'FORCE_COLOR': '1', "PYTHON_COLORS": '0'}): - self.assertEqual(traceback._can_colorize(), False) - isatty_mock.return_value = False - with unittest.mock.patch("os.environ", {}): - self.assertEqual(traceback._can_colorize(), False) if __name__ == "__main__": unittest.main() diff --git a/Lib/traceback.py b/Lib/traceback.py index 6ce745f32d13242..8403173ade7b6c9 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1,7 +1,5 @@ """Extract, format and print information about Python stack traces.""" -import os -import io import collections.abc import itertools import linecache @@ -9,6 +7,8 @@ import textwrap import warnings from contextlib import suppress +import _colorize +from _colorize import ANSIColors __all__ = ['extract_stack', 'extract_tb', 'format_exception', 'format_exception_only', 'format_list', 'format_stack', @@ -21,7 +21,6 @@ # Formatting and printing lists of traceback lines. # -_COLORIZE = True def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or @@ -133,41 +132,10 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ BUILTIN_EXCEPTION_LIMIT = object() -def _can_colorize(): - if sys.platform == "win32": - try: - import nt - if not nt._supports_virtual_terminal(): - return False - except (ImportError, AttributeError): - return False - if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": - return False - if os.environ.get("PYTHON_COLORS") == "1": - return True - if "NO_COLOR" in os.environ: - return False - if not _COLORIZE: - return False - if not sys.flags.ignore_environment: - if "FORCE_COLOR" in os.environ: - return True - if os.environ.get("TERM") == "dumb": - return False - - if not hasattr(sys.stderr, "fileno"): - return False - - try: - return os.isatty(sys.stderr.fileno()) - except io.UnsupportedOperation: - return sys.stderr.isatty() - def _print_exception_bltin(exc, /): file = sys.stderr if sys.stderr is not None else sys.__stderr__ - colorize = _can_colorize() + colorize = _colorize.can_colorize() return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file, colorize=colorize) @@ -214,9 +182,9 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= end_char = "\n" if insert_final_newline else "" if colorize: if value is None or not valuestr: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}{end_char}" else: - line = f"{_ANSIColors.BOLD_MAGENTA}{etype}{_ANSIColors.RESET}: {_ANSIColors.MAGENTA}{valuestr}{_ANSIColors.RESET}{end_char}" + line = f"{ANSIColors.BOLD_MAGENTA}{etype}{ANSIColors.RESET}: {ANSIColors.MAGENTA}{valuestr}{ANSIColors.RESET}{end_char}" else: if value is None or not valuestr: line = f"{etype}{end_char}" @@ -224,6 +192,7 @@ def _format_final_exc_line(etype, value, *, insert_final_newline=True, colorize= line = f"{etype}: {valuestr}{end_char}" return line + def _safe_string(value, what, func=str): try: return func(value) @@ -449,17 +418,6 @@ def _get_code_position(code, instruction_index): _RECURSIVE_CUTOFF = 3 # Also hardcoded in traceback.c. -class _ANSIColors: - RED = '\x1b[31m' - BOLD_RED = '\x1b[1;31m' - MAGENTA = '\x1b[35m' - BOLD_MAGENTA = '\x1b[1;35m' - GREEN = "\x1b[32m" - BOLD_GREEN = "\x1b[1;32m" - GREY = '\x1b[90m' - RESET = '\x1b[0m' - YELLOW = "\x1b[33m" - class StackSummary(list): """A list of FrameSummary objects, representing a stack of frames.""" @@ -564,15 +522,15 @@ def format_frame_summary(self, frame_summary, **kwargs): filename = "" if colorize: row.append(' File {}"{}"{}, line {}{}{}, in {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, filename, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.lineno, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, frame_summary.name, - _ANSIColors.RESET, + ANSIColors.RESET, ) ) else: @@ -696,11 +654,11 @@ def output_line(lineno): for color, group in itertools.groupby(itertools.zip_longest(line, carets, fillvalue=""), key=lambda x: x[1]): caret_group = list(group) if color == "^": - colorized_line_parts.append(_ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.BOLD_RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.BOLD_RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) elif color == "~": - colorized_line_parts.append(_ANSIColors.RED + "".join(char for char, _ in caret_group) + _ANSIColors.RESET) - colorized_carets_parts.append(_ANSIColors.RED + "".join(caret for _, caret in caret_group) + _ANSIColors.RESET) + colorized_line_parts.append(ANSIColors.RED + "".join(char for char, _ in caret_group) + ANSIColors.RESET) + colorized_carets_parts.append(ANSIColors.RED + "".join(caret for _, caret in caret_group) + ANSIColors.RESET) else: colorized_line_parts.append("".join(char for char, _ in caret_group)) colorized_carets_parts.append("".join(caret for _, caret in caret_group)) @@ -1307,12 +1265,12 @@ def _format_syntax_error(self, stype, **kwargs): if self.lineno is not None: if colorize: yield ' File {}"{}"{}, line {}{}{}\n'.format( - _ANSIColors.MAGENTA, + ANSIColors.MAGENTA, self.filename or "", - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, self.lineno, - _ANSIColors.RESET, + ANSIColors.RESET, ) else: yield ' File "{}", line {}\n'.format( @@ -1352,11 +1310,11 @@ def _format_syntax_error(self, stype, **kwargs): # colorize from colno to end_colno ltext = ( ltext[:colno] + - _ANSIColors.BOLD_RED + ltext[colno:end_colno] + _ANSIColors.RESET + + ANSIColors.BOLD_RED + ltext[colno:end_colno] + ANSIColors.RESET + ltext[end_colno:] ) - start_color = _ANSIColors.BOLD_RED - end_color = _ANSIColors.RESET + start_color = ANSIColors.BOLD_RED + end_color = ANSIColors.RESET yield ' {}\n'.format(ltext) yield ' {}{}{}{}\n'.format( "".join(caretspace), @@ -1369,12 +1327,12 @@ def _format_syntax_error(self, stype, **kwargs): msg = self.msg or "" if colorize: yield "{}{}{}: {}{}{}{}\n".format( - _ANSIColors.BOLD_MAGENTA, + ANSIColors.BOLD_MAGENTA, stype, - _ANSIColors.RESET, - _ANSIColors.MAGENTA, + ANSIColors.RESET, + ANSIColors.MAGENTA, msg, - _ANSIColors.RESET, + ANSIColors.RESET, filename_suffix) else: yield "{}: {}{}\n".format(stype, msg, filename_suffix) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index f44abf1f9c3121d..ba320842173b484 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -20,6 +20,7 @@ static const char* _Py_stdlib_module_names[] = { "_codecs_tw", "_collections", "_collections_abc", +"_colorize", "_compat_pickle", "_compression", "_contextvars", From 1161ab9085ee6e2d6efdf497c81129ba0837031b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 May 2024 19:15:30 +0000 Subject: [PATCH 047/220] build(deps-dev): bump types-psutil from 5.9.5.20240316 to 5.9.5.20240423 in /Tools (#118464) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Tools/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index ddeff674e92457a..1767727373918f9 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -3,5 +3,5 @@ mypy==1.10.0 # needed for peg_generator: -types-psutil==5.9.5.20240316 +types-psutil==5.9.5.20240423 types-setuptools==69.5.0.20240423 From 6763bfcc0fbab7895514c1f954d79a2555036323 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 1 May 2024 21:51:40 +0100 Subject: [PATCH 048/220] gh-118272: set stacktop to 0 before freeing contents, to avoid access to invalid objects during GC (#118478) --- Python/frame.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/frame.c b/Python/frame.c index ec390e7426ad470..2bb128235720284 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -98,10 +98,11 @@ void _PyFrame_ClearLocals(_PyInterpreterFrame *frame) { assert(frame->stacktop >= 0); - for (int i = 0; i < frame->stacktop; i++) { + int stacktop = frame->stacktop; + frame->stacktop = 0; + for (int i = 0; i < stacktop; i++) { Py_XDECREF(frame->localsplus[i]); } - frame->stacktop = 0; Py_CLEAR(frame->f_locals); } From 8a50544a998764b8a315b316ceadaebdc748d8fe Mon Sep 17 00:00:00 2001 From: mpage Date: Wed, 1 May 2024 13:59:12 -0700 Subject: [PATCH 049/220] gh-118433: Temporarily skip `test_interrupt_main_subthread` in free-threaded builds (#118485) Free-threaded builds can intermittently tickle a longstanding bug (24 years!) in the implementation of `threading.Condition`, leading to flakiness in the test suite. Fixing the underlying issue will require more discussion, and will likely apply to most of the concurrency primitives in the `threading` module that are written in Python. See gh-118433 for more details. --- Lib/test/test_threading.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 362a3f9c4a01d12..0047e8a8798d4e0 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -3,7 +3,7 @@ """ import test.support -from test.support import threading_helper, requires_subprocess +from test.support import threading_helper, requires_subprocess, requires_gil_enabled from test.support import verbose, cpython_only, os_helper from test.support.import_helper import import_module from test.support.script_helper import assert_python_ok, assert_python_failure @@ -2025,6 +2025,7 @@ def check_interrupt_main_noerror(self, signum): # Restore original handler signal.signal(signum, handler) + @requires_gil_enabled("gh-118433: Flaky due to a longstanding bug") def test_interrupt_main_subthread(self): # Calling start_new_thread with a function that executes interrupt_main # should raise KeyboardInterrupt upon completion. From 39981fd07a479ef3cefb38ee96ac319fe00f2dde Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 May 2024 22:18:31 +0100 Subject: [PATCH 050/220] GH-118095: Make sure that progress is made if there are pending calls being handled. (GH-118484) --- Include/internal/pycore_ceval_state.h | 2 +- Python/ceval_gil.c | 21 ++++++++++----------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 11f2a100bf531e6..376d96ad5d334cc 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -41,7 +41,7 @@ struct _pending_call { #define MAXPENDINGCALLSLOOP_MAIN 0 struct _pending_calls { - int busy; + PyThreadState *handling_thread; PyMutex mutex; /* Request for running pending calls. */ int32_t npending; diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index fdbb4882c3d7117..e4f7217255efd83 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -877,21 +877,20 @@ make_pending_calls(PyThreadState *tstate) /* Only one thread (per interpreter) may run the pending calls at once. In the same way, we don't do recursive pending calls. */ PyMutex_Lock(&pending->mutex); - if (pending->busy) { + if (pending->handling_thread != NULL) { /* A pending call was added after another thread was already handling the pending calls (and had already "unsignaled"). Once that thread is done, it may have taken care of all the pending calls, or there might be some still waiting. - Regardless, this interpreter's pending calls will stay - "signaled" until that first thread has finished. At that - point the next thread to trip the eval breaker will take - care of any remaining pending calls. Until then, though, - all the interpreter's threads will be tripping the eval - breaker every time it's checked. */ + To avoid all threads constantly stopping on the eval breaker, + we clear the bit for this thread and make sure it is set + for the thread currently handling the pending call. */ + _Py_set_eval_breaker_bit(pending->handling_thread, _PY_CALLS_TO_DO_BIT); + _Py_unset_eval_breaker_bit(tstate, _PY_CALLS_TO_DO_BIT); PyMutex_Unlock(&pending->mutex); return 0; } - pending->busy = 1; + pending->handling_thread = tstate; PyMutex_Unlock(&pending->mutex); /* unsignal before starting to call callbacks, so that any callback @@ -900,7 +899,7 @@ make_pending_calls(PyThreadState *tstate) int32_t npending; if (_make_pending_calls(pending, &npending) != 0) { - pending->busy = 0; + pending->handling_thread = NULL; /* There might not be more calls to make, but we play it safe. */ signal_pending_calls(tstate, interp); return -1; @@ -912,7 +911,7 @@ make_pending_calls(PyThreadState *tstate) if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) { if (_make_pending_calls(pending_main, &npending) != 0) { - pending->busy = 0; + pending->handling_thread = NULL; /* There might not be more calls to make, but we play it safe. */ signal_pending_calls(tstate, interp); return -1; @@ -923,7 +922,7 @@ make_pending_calls(PyThreadState *tstate) } } - pending->busy = 0; + pending->handling_thread = NULL; return 0; } From 424438b11ec90110054f720bfa6ea67d644cc2ec Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Wed, 1 May 2024 14:35:49 -0700 Subject: [PATCH 051/220] GH-113464: Display a warning when building the JIT (GH-118481) --- Tools/jit/_llvm.py | 1 + Tools/jit/_schema.py | 1 + Tools/jit/_stencils.py | 8 +++---- Tools/jit/_targets.py | 49 +++++++++++++++++++++++++----------------- Tools/jit/_writer.py | 1 + Tools/jit/build.py | 4 ++-- 6 files changed, 38 insertions(+), 26 deletions(-) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index 74a048ccbfcc586..45bd69ff861b56b 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -1,4 +1,5 @@ """Utilities for invoking LLVM tools.""" + import asyncio import functools import os diff --git a/Tools/jit/_schema.py b/Tools/jit/_schema.py index 6aef887e12475b5..228fc389584dd7c 100644 --- a/Tools/jit/_schema.py +++ b/Tools/jit/_schema.py @@ -1,4 +1,5 @@ """Schema for the JSON produced by llvm-readobj --elf-output-style=JSON.""" + import typing HoleKind: typing.TypeAlias = typing.Literal[ diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index f8ecffcf3ddda25..9feceb45388d05d 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -1,4 +1,5 @@ """Core data structures for compiled code templates.""" + import dataclasses import enum import sys @@ -29,7 +30,7 @@ class HoleValue(enum.Enum): OPARG = enum.auto() # The current uop's operand on 64-bit platforms (exposed as _JIT_OPERAND): OPERAND = enum.auto() - # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI and _JIT_OPERAND_LO): + # The current uop's operand on 32-bit platforms (exposed as _JIT_OPERAND_HI/LO): OPERAND_HI = enum.auto() OPERAND_LO = enum.auto() # The current uop's target (exposed as _JIT_TARGET): @@ -203,9 +204,8 @@ def process_relocations(self, *, alignment: int = 1) -> None: """Fix up all GOT and internal relocations for this stencil group.""" for hole in self.code.holes.copy(): if ( - hole.kind in { - "R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26" - } + hole.kind + in {"R_AARCH64_CALL26", "R_AARCH64_JUMP26", "ARM64_RELOC_BRANCH26"} and hole.value is HoleValue.ZERO ): self.code.pad(alignment) diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 274d17bcf38deb0..23bb18947f80ea2 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -1,4 +1,5 @@ """Target-specific code generation, parsing, and processing.""" + import asyncio import dataclasses import hashlib @@ -40,8 +41,8 @@ class _Target(typing.Generic[_S, _R]): args: typing.Sequence[str] = () ghccc: bool = False prefix: str = "" + stable: bool = False debug: bool = False - force: bool = False verbose: bool = False def _compute_digest(self, out: pathlib.Path) -> str: @@ -186,12 +187,19 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: tasks.append(group.create_task(coro, name=opname)) return {task.get_name(): task.result() for task in tasks} - def build(self, out: pathlib.Path, *, comment: str = "") -> None: + def build( + self, out: pathlib.Path, *, comment: str = "", force: bool = False + ) -> None: """Build jit_stencils.h in the given directory.""" + if not self.stable: + warning = f"JIT support for {self.triple} is still experimental!" + request = "Please report any issues you encounter.".center(len(warning)) + outline = "=" * len(warning) + print("\n".join(["", outline, warning, request, outline, ""])) digest = f"// {self._compute_digest(out)}\n" jit_stencils = out / "jit_stencils.h" if ( - not self.force + not force and jit_stencils.exists() and jit_stencils.read_text().startswith(digest) ): @@ -450,9 +458,7 @@ def _handle_relocation( } | { "Offset": offset, "Symbol": {"Name": s}, - "Type": { - "Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind - }, + "Type": {"Name": "X86_64_RELOC_BRANCH" | "X86_64_RELOC_SIGNED" as kind}, }: offset += base s = s.removeprefix(self.prefix) @@ -481,23 +487,26 @@ def _handle_relocation( def get_target(host: str) -> _COFF | _ELF | _MachO: """Build a _Target for the given host "triple" and options.""" # ghccc currently crashes Clang when combined with musttail on aarch64. :( + target: _COFF | _ELF | _MachO if re.fullmatch(r"aarch64-apple-darwin.*", host): - return _MachO(host, alignment=8, prefix="_") - if re.fullmatch(r"aarch64-pc-windows-msvc", host): + target = _MachO(host, alignment=8, prefix="_") + elif re.fullmatch(r"aarch64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - return _COFF(host, alignment=8, args=args) - if re.fullmatch(r"aarch64-.*-linux-gnu", host): + target = _COFF(host, alignment=8, args=args) + elif re.fullmatch(r"aarch64-.*-linux-gnu", host): args = ["-fpic"] - return _ELF(host, alignment=8, args=args) - if re.fullmatch(r"i686-pc-windows-msvc", host): + target = _ELF(host, alignment=8, args=args) + elif re.fullmatch(r"i686-pc-windows-msvc", host): args = ["-DPy_NO_ENABLE_SHARED"] - return _COFF(host, args=args, ghccc=True, prefix="_") - if re.fullmatch(r"x86_64-apple-darwin.*", host): - return _MachO(host, ghccc=True, prefix="_") - if re.fullmatch(r"x86_64-pc-windows-msvc", host): + target = _COFF(host, args=args, ghccc=True, prefix="_") + elif re.fullmatch(r"x86_64-apple-darwin.*", host): + target = _MachO(host, ghccc=True, prefix="_") + elif re.fullmatch(r"x86_64-pc-windows-msvc", host): args = ["-fms-runtime-lib=dll"] - return _COFF(host, args=args, ghccc=True) - if re.fullmatch(r"x86_64-.*-linux-gnu", host): + target = _COFF(host, args=args, ghccc=True) + elif re.fullmatch(r"x86_64-.*-linux-gnu", host): args = ["-fpic"] - return _ELF(host, args=args, ghccc=True) - raise ValueError(host) + target = _ELF(host, args=args, ghccc=True) + else: + raise ValueError(host) + return target diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 6b36d8a9c66a3fb..ccd67850c37787c 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -1,4 +1,5 @@ """Utilities for writing StencilGroups out to a C header file.""" + import typing import _schema diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 4d4ace14ebf26c4..4a23c6f0afa74a1 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -1,4 +1,5 @@ """Build an experimental just-in-time compiler for CPython.""" + import argparse import pathlib import shlex @@ -23,6 +24,5 @@ ) args = parser.parse_args() args.target.debug = args.debug - args.target.force = args.force args.target.verbose = args.verbose - args.target.build(pathlib.Path.cwd(), comment=comment) + args.target.build(pathlib.Path.cwd(), comment=comment, force=args.force) From a7711a2a4e5cf16b34fc284085da724a8c2c06dd Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 1 May 2024 23:44:55 +0200 Subject: [PATCH 052/220] gh-117607: Speedup os.path.relpath() (GH-117608) --- Lib/ntpath.py | 19 ++++++++++--------- Lib/posixpath.py | 8 +++++--- ...-04-07-18-42-09.gh-issue-117607.C978BD.rst | 1 + 3 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-07-18-42-09.gh-issue-117607.C978BD.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e810b655e5ac851..b833e0bad2645f7 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -805,6 +805,9 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -816,22 +819,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) + start_abs = abspath(start) + path_abs = abspath(path) start_drive, _, start_rest = splitroot(start_abs) path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -842,7 +843,7 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 56b7915826daf4b..f189c3359fbea64 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -532,15 +532,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-07-18-42-09.gh-issue-117607.C978BD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-07-18-42-09.gh-issue-117607.C978BD.rst new file mode 100644 index 000000000000000..0c17dfd95ec0ec3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-07-18-42-09.gh-issue-117607.C978BD.rst @@ -0,0 +1 @@ +Speedup :func:`os.path.relpath`. From c408c36e9b346f9f15a34e98a5596f311df65efa Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Wed, 1 May 2024 17:58:22 -0400 Subject: [PATCH 053/220] gh-118413: Temporarily skip `test_release_task_refs` in free-threaded builds (#118491) --- Lib/test/_test_multiprocessing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index a74b61013c48488..91f85990116ee7c 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2804,6 +2804,7 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.monotonic() - t_start, 0.9) + @support.requires_gil_enabled("gh-118413: test is flaky with GIL disabled") def test_release_task_refs(self): # Issue #29861: task arguments and results should not be kept # alive after we are done with them. From 97feb4a78bf1ebdec26b685fd509cf57b3333e4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Thu, 2 May 2024 00:13:32 +0100 Subject: [PATCH 054/220] GH-118174: specify the type for the path argument of shutil.which --- Doc/library/shutil.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 8e5828c789e4e23..542d28864777924 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -449,8 +449,9 @@ Directory and files operations *mode* is a permission mask passed to :func:`os.access`, by default determining if the file exists and executable. - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the lookup directory list. When no + *path* is specified, the results of :func:`os.environ` are used, returning + either the "PATH" value or a fallback of :data:`os.defpath`. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the From a524152b8c56f82d807ecdd7ae9d39cfd643c469 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 May 2024 16:36:29 -0700 Subject: [PATCH 055/220] gh-118335: Make REGEN_JIT_COMMAND empty if tier2 interpreter enabled (#118493) Also patch up news blurb for gh-118339 (add warning that PYTHON_UOPS is now PYTHON_JIT). --- ...-04-26-14-06-18.gh-issue-118335.SRFsxO.rst | 3 +++ configure | 24 ++++++++++++------- configure.ac | 22 ++++++++++------- 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst index e13edbbbe528eb2..54295a75c02e855 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-26-14-06-18.gh-issue-118335.SRFsxO.rst @@ -2,3 +2,6 @@ Change how to use the tier 2 interpreter. Instead of running Python with ``-X uops`` or setting the environment variable ``PYTHON_UOPS=1``, this choice is now made at build time by configuring with ``--enable-experimental-jit=interpreter``. + +**Beware!** This changes the environment variable to enable or disable +micro-ops to ``PYTHON_JIT``. The old ``PYTHON_UOPS`` is no longer used. diff --git a/configure b/configure index 24e7396e389fd1e..cc85aed2aa51c2f 100755 --- a/configure +++ b/configure @@ -8235,18 +8235,24 @@ else $as_nop fi case $enable_experimental_jit in - no) enable_experimental_jit=no ;; - yes) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=1" ;; - yes-off) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=3" ;; - interpreter) enable_experimental_jit="-D_Py_TIER2=4" ;; - interpreter-off) enable_experimental_jit="-D_Py_TIER2=6" ;; # Secret option + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option *) as_fn_error $? "invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter" "$LINENO" 5 ;; esac -if test "x$enable_experimental_jit" = xno +if ${tier2_flags:+false} : then : else $as_nop - as_fn_append CFLAGS_NODIST " $enable_experimental_jit" + as_fn_append CFLAGS_NODIST " $tier2_flags" +fi +if ${jit_flags:+false} : +then : + +else $as_nop + as_fn_append CFLAGS_NODIST " $jit_flags" REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue @@ -8256,8 +8262,8 @@ fi fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $enable_experimental_jit" >&5 -printf "%s\n" "$enable_experimental_jit" >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $tier2_flags $jit_flags" >&5 +printf "%s\n" "$tier2_flags $jit_flags" >&6; } # Enable optimization flags diff --git a/configure.ac b/configure.ac index 0f1177872d91372..c55e33add20fdec 100644 --- a/configure.ac +++ b/configure.ac @@ -1776,18 +1776,22 @@ AC_ARG_ENABLE([experimental-jit], [], [enable_experimental_jit=no]) case $enable_experimental_jit in - no) enable_experimental_jit=no ;; - yes) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=1" ;; - yes-off) enable_experimental_jit="-D_Py_JIT -D_Py_TIER2=3" ;; - interpreter) enable_experimental_jit="-D_Py_TIER2=4" ;; - interpreter-off) enable_experimental_jit="-D_Py_TIER2=6" ;; # Secret option + no) jit_flags=""; tier2_flags="" ;; + yes) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=1" ;; + yes-off) jit_flags="-D_Py_JIT"; tier2_flags="-D_Py_TIER2=3" ;; + interpreter) jit_flags=""; tier2_flags="-D_Py_TIER2=4" ;; + interpreter-off) jit_flags=""; tier2_flags="-D_Py_TIER2=6" ;; # Secret option *) AC_MSG_ERROR( [invalid argument: --enable-experimental-jit=$enable_experimental_jit; expected no|yes|yes-off|interpreter]) ;; esac -AS_VAR_IF([enable_experimental_jit], - [no], +AS_VAR_IF([tier2_flags], [], - [AS_VAR_APPEND([CFLAGS_NODIST], [" $enable_experimental_jit"]) + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $tier2_flags"])]) +AS_VAR_IF([jit_flags], + [], + [], + [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py $host"]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) @@ -1797,7 +1801,7 @@ AS_VAR_IF([enable_experimental_jit], [])]) AC_SUBST([REGEN_JIT_COMMAND]) AC_SUBST([JIT_STENCILS_H]) -AC_MSG_RESULT([$enable_experimental_jit]) +AC_MSG_RESULT([$tier2_flags $jit_flags]) # Enable optimization flags AC_SUBST([DEF_MAKE_ALL_RULE]) From 526ca4c09eed63e19ea5ffc46a76c5d8077a8bc9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 1 May 2024 17:40:28 -0600 Subject: [PATCH 056/220] gh-117953: Work Relative to Specific Extension Kinds in the Import Machinery (gh-118205) This change will make some later changes simpler. --- Include/internal/pycore_importdl.h | 32 ++++- Python/import.c | 151 +++++++++++++++------ Python/importdl.c | 207 +++++++++++++++++++++++++---- 3 files changed, 324 insertions(+), 66 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index adb89167d124eb4..b0af28da34e0871 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -15,8 +15,15 @@ extern "C" { extern const char *_PyImport_DynLoadFiletab[]; -typedef PyObject *(*PyModInitFunction)(void); +typedef enum ext_module_kind { + _Py_ext_module_kind_UNKNOWN = 0, + _Py_ext_module_kind_SINGLEPHASE = 1, + _Py_ext_module_kind_MULTIPHASE = 2, + _Py_ext_module_kind_INVALID = 3, +} _Py_ext_module_kind; + +/* Input for loading an extension module. */ struct _Py_ext_module_loader_info { PyObject *filename; #ifndef MS_WINDOWS @@ -43,10 +50,33 @@ extern int _Py_ext_module_loader_info_init_from_spec( struct _Py_ext_module_loader_info *info, PyObject *spec); +/* The result from running an extension module's init function. */ struct _Py_ext_module_loader_result { PyModuleDef *def; PyObject *module; + _Py_ext_module_kind kind; + struct _Py_ext_module_loader_result_error *err; + struct _Py_ext_module_loader_result_error { + enum _Py_ext_module_loader_result_error_kind { + _Py_ext_module_loader_result_EXCEPTION = 0, + _Py_ext_module_loader_result_ERR_MISSING = 1, + _Py_ext_module_loader_result_ERR_UNREPORTED_EXC = 2, + _Py_ext_module_loader_result_ERR_UNINITIALIZED = 3, + _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE = 4, + _Py_ext_module_loader_result_ERR_NOT_MODULE = 5, + _Py_ext_module_loader_result_ERR_MISSING_DEF = 6, + } kind; + PyObject *exc; + } _err; }; +extern void _Py_ext_module_loader_result_clear( + struct _Py_ext_module_loader_result *res); +extern void _Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name); + +/* The module init function. */ +typedef PyObject *(*PyModInitFunction)(void); extern PyModInitFunction _PyImport_GetModInitFunc( struct _Py_ext_module_loader_info *info, FILE *fp); diff --git a/Python/import.c b/Python/import.c index 447114ab115bc7d..0c51ffc6285a9c6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1193,26 +1193,63 @@ is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *path) } #ifndef NDEBUG -static bool -is_singlephase(PyModuleDef *def) +static _Py_ext_module_kind +_get_extension_kind(PyModuleDef *def, bool check_size) { + _Py_ext_module_kind kind; if (def == NULL) { /* It must be a module created by reload_singlephase_extension() * from m_copy. Ideally we'd do away with this case. */ - return true; + kind = _Py_ext_module_kind_SINGLEPHASE; } - else if (def->m_slots == NULL) { - return true; + else if (def->m_slots != NULL) { + kind = _Py_ext_module_kind_MULTIPHASE; } - else { - return false; + else if (check_size && def->m_size == -1) { + kind = _Py_ext_module_kind_SINGLEPHASE; } -} + else if (def->m_base.m_init != NULL) { + kind = _Py_ext_module_kind_SINGLEPHASE; + } + else { + // This is probably single-phase init, but a multi-phase + // module *can* have NULL m_slots. + kind = _Py_ext_module_kind_UNKNOWN; + } + return kind; +} + +/* The module might not be fully initialized yet + * and PyModule_FromDefAndSpec() checks m_size + * so we skip m_size. */ +#define assert_multiphase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, false); \ + assert(kind == _Py_ext_module_kind_MULTIPHASE \ + /* m_slots can be NULL. */ \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase_def(def) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE \ + || kind == _Py_ext_module_kind_UNKNOWN); \ + } while (0) + +#define assert_singlephase(def) \ + assert_singlephase_def(def) + +#else /* defined(NDEBUG) */ +#define assert_multiphase_def(def) +#define assert_singlephase_def(def) +#define assert_singlephase(def) #endif struct singlephase_global_update { PyObject *m_dict; + PyModInitFunction m_init; }; static int @@ -1226,10 +1263,24 @@ update_global_state_for_extension(PyThreadState *tstate, assert(def->m_base.m_copy == NULL); } else { - assert(def->m_base.m_init != NULL - || is_core_module(tstate->interp, name, path)); - if (singlephase->m_dict == NULL) { + if (singlephase->m_init != NULL) { + assert(singlephase->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + assert(def->m_size >= 0); + /* Remember pointer to module init function. */ + // XXX If two modules share a def then def->m_base will + // reflect the last one added (here) to the global cache. + // We should prevent this somehow. The simplest solution + // is probably to store m_copy/m_init in the cache along + // with the def, rather than within the def. + def->m_base.m_init = singlephase->m_init; + } + else if (singlephase->m_dict == NULL) { + /* It must be a core builtin module. */ + assert(is_core_module(tstate->interp, name, path)); + assert(def->m_size == -1); assert(def->m_base.m_copy == NULL); + assert(def->m_base.m_init == NULL); } else { assert(PyDict_Check(singlephase->m_dict)); @@ -1316,7 +1367,7 @@ import_find_extension(PyThreadState *tstate, if (def == NULL) { return NULL; } - assert(is_singlephase(def)); + assert_singlephase(def); /* It may have been successfully imported previously in an interpreter that allows legacy modules @@ -1369,11 +1420,26 @@ import_find_extension(PyThreadState *tstate, } struct _Py_ext_module_loader_result res; if (_PyImport_RunModInitFunc(def->m_base.m_init, info, &res) < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); return NULL; } assert(!PyErr_Occurred()); + assert(res.err == NULL); + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); mod = res.module; - // XXX __file__ doesn't get set! + /* Tchnically, the init function could return a different module def. + * Then we would probably need to update the global cache. + * However, we don't expect anyone to change the def. */ + assert(res.def == def); + _Py_ext_module_loader_result_clear(&res); + + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + if (PyObject_SetItem(modules, info->name, mod) == -1) { Py_DECREF(mod); return NULL; @@ -1398,78 +1464,89 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct _Py_ext_module_loader_info *info, PyObject *spec, PyObject *modules) { + /* Core modules go through _PyImport_FixupBuiltin(). */ + assert(!is_core_module(tstate->interp, info->name, info->path)); + PyObject *mod = NULL; PyModuleDef *def = NULL; + const char *name_buf = PyBytes_AS_STRING(info->name_encoded); struct _Py_ext_module_loader_result res; if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { /* We discard res.def. */ assert(res.module == NULL); - assert(PyErr_Occurred()); - goto finally; + _Py_ext_module_loader_result_apply_error(&res, name_buf); + return NULL; } assert(!PyErr_Occurred()); + assert(res.err == NULL); mod = res.module; res.module = NULL; def = res.def; assert(def != NULL); - if (mod == NULL) { - //assert(!is_singlephase(def)); + if (res.kind == _Py_ext_module_kind_MULTIPHASE) { + assert_multiphase_def(def); assert(mod == NULL); mod = PyModule_FromDefAndSpec(def, spec); if (mod == NULL) { - goto finally; + goto error; } } else { - assert(is_singlephase(def)); + assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); + assert_singlephase_def(def); assert(PyModule_Check(mod)); - const char *name_buf = PyBytes_AS_STRING(info->name_encoded); if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { - Py_CLEAR(mod); - goto finally; + goto error; } - /* Remember pointer to module init function. */ - def->m_base.m_init = p0; - + /* Remember the filename as the __file__ attribute */ if (info->filename != NULL) { - /* Remember the filename as the __file__ attribute */ if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { PyErr_Clear(); /* Not important enough to report */ } } + /* Update global import state. */ struct singlephase_global_update singlephase = {0}; // gh-88216: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. - if (def->m_size == -1 - && !is_core_module(tstate->interp, info->name, info->path)) - { + if (def->m_size == -1) { + /* We will reload from m_copy. */ + assert(def->m_base.m_init == NULL); singlephase.m_dict = PyModule_GetDict(mod); assert(singlephase.m_dict != NULL); } + else { + /* We will reload via the init function. */ + assert(def->m_size >= 0); + singlephase.m_init = p0; + } if (update_global_state_for_extension( tstate, info->path, info->name, def, &singlephase) < 0) { - Py_CLEAR(mod); - goto finally; + goto error; } + /* Update per-interpreter import state. */ PyObject *modules = get_modules_dict(tstate, true); if (finish_singlephase_extension( tstate, mod, def, info->name, modules) < 0) { - Py_CLEAR(mod); - goto finally; + goto error; } } -finally: + _Py_ext_module_loader_result_clear(&res); return mod; + +error: + Py_XDECREF(mod); + _Py_ext_module_loader_result_clear(&res); + return NULL; } @@ -1532,7 +1609,7 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, * module state, but we also don't populate def->m_base.m_copy * for them. */ assert(is_core_module(tstate->interp, nameobj, nameobj)); - assert(is_singlephase(def)); + assert_singlephase_def(def); assert(def->m_size == -1); assert(def->m_base.m_copy == NULL); @@ -1586,7 +1663,7 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) PyObject *mod = import_find_extension(tstate, &info); if (mod != NULL) { assert(!_PyErr_Occurred(tstate)); - assert(is_singlephase(_PyModule_GetDef(mod))); + assert_singlephase(_PyModule_GetDef(mod)); goto finally; } else if (_PyErr_Occurred(tstate)) { @@ -3940,7 +4017,7 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) mod = import_find_extension(tstate, &info); if (mod != NULL) { assert(!_PyErr_Occurred(tstate)); - assert(is_singlephase(_PyModule_GetDef(mod))); + assert_singlephase(_PyModule_GetDef(mod)); goto finally; } else if (_PyErr_Occurred(tstate)) { diff --git a/Python/importdl.c b/Python/importdl.c index 3e4ea175c0cfbc9..a3091946bc94bfe 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -28,6 +28,11 @@ extern dl_funcptr _PyImport_FindSharedFuncptr(const char *prefix, const char *pathname, FILE *fp); #endif + +/***********************************/ +/* module info to use when loading */ +/***********************************/ + static const char * const ascii_only_prefix = "PyInit"; static const char * const nonascii_prefix = "PyInitU"; @@ -205,6 +210,150 @@ _Py_ext_module_loader_info_init_from_spec( } +/********************************/ +/* module init function results */ +/********************************/ + +void +_Py_ext_module_loader_result_clear(struct _Py_ext_module_loader_result *res) +{ + /* Instead, the caller should have called + * _Py_ext_module_loader_result_apply_error(). */ + assert(res->err == NULL); + *res = (struct _Py_ext_module_loader_result){0}; +} + +static void +_Py_ext_module_loader_result_set_error( + struct _Py_ext_module_loader_result *res, + enum _Py_ext_module_loader_result_error_kind kind) +{ +#ifndef NDEBUG + switch (kind) { + case _Py_ext_module_loader_result_EXCEPTION: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(PyErr_Occurred()); + break; + case _Py_ext_module_loader_result_ERR_MISSING: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: /* fall through */ + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: /* fall through */ + case _Py_ext_module_loader_result_ERR_NOT_MODULE: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(!PyErr_Occurred()); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +#endif + + assert(res->err == NULL && res->_err.exc == NULL); + res->err = &res->_err; + *res->err = (struct _Py_ext_module_loader_result_error){ + .kind=kind, + .exc=PyErr_GetRaisedException(), + }; + + /* For some kinds, we also set/check res->kind. */ + switch (kind) { + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + assert(res->kind == _Py_ext_module_kind_UNKNOWN); + res->kind = _Py_ext_module_kind_INVALID; + break; + /* None of the rest affect the result kind. */ + case _Py_ext_module_loader_result_EXCEPTION: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: /* fall through */ + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: /* fall through */ + case _Py_ext_module_loader_result_ERR_NOT_MODULE: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +} + +void +_Py_ext_module_loader_result_apply_error( + struct _Py_ext_module_loader_result *res, + const char *name) +{ + assert(!PyErr_Occurred()); + assert(res->err != NULL && res->err == &res->_err); + struct _Py_ext_module_loader_result_error err = *res->err; + res->err = NULL; + + /* We're otherwise done with the result at this point. */ + _Py_ext_module_loader_result_clear(res); + +#ifndef NDEBUG + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + assert(err.exc != NULL); + break; + case _Py_ext_module_loader_result_ERR_MISSING: /* fall through */ + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: /* fall through */ + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: /* fall through */ + case _Py_ext_module_loader_result_ERR_NOT_MODULE: /* fall through */ + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + assert(err.exc == NULL); + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + } +#endif + + const char *msg = NULL; + switch (err.kind) { + case _Py_ext_module_loader_result_EXCEPTION: + break; + case _Py_ext_module_loader_result_ERR_MISSING: + msg = "initialization of %s failed without raising an exception"; + break; + case _Py_ext_module_loader_result_ERR_UNREPORTED_EXC: + msg = "initialization of %s raised unreported exception"; + break; + case _Py_ext_module_loader_result_ERR_UNINITIALIZED: + msg = "init function of %s returned uninitialized object"; + break; + case _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE: + msg = "initialization of %s did not return PyModuleDef"; + break; + case _Py_ext_module_loader_result_ERR_NOT_MODULE: + msg = "initialization of %s did not return an extension module"; + break; + case _Py_ext_module_loader_result_ERR_MISSING_DEF: + msg = "initialization of %s did not return a valid extension module"; + break; + default: + /* We added a new error kind but forgot to add it to this switch. */ + assert(0); + PyErr_Format(PyExc_SystemError, + "loading %s failed due to init function", name); + return; + } + + if (err.exc != NULL) { + PyErr_SetRaisedException(err.exc); + err.exc = NULL; /* PyErr_SetRaisedException() stole our reference. */ + if (msg != NULL) { + _PyErr_FormatFromCause(PyExc_SystemError, msg, name); + } + } + else { + assert(msg != NULL); + PyErr_Format(PyExc_SystemError, msg, name); + } +} + + +/********************************************/ +/* getting/running the module init function */ +/********************************************/ + PyModInitFunction _PyImport_GetModInitFunc(struct _Py_ext_module_loader_info *info, FILE *fp) @@ -245,8 +394,9 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, struct _Py_ext_module_loader_info *info, struct _Py_ext_module_loader_result *p_res) { - struct _Py_ext_module_loader_result res = {0}; - const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + struct _Py_ext_module_loader_result res = { + .kind=_Py_ext_module_kind_UNKNOWN, + }; /* Call the module init function. */ @@ -258,18 +408,18 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, /* Validate the result (and populate "res". */ if (m == NULL) { - if (!PyErr_Occurred()) { - PyErr_Format( - PyExc_SystemError, - "initialization of %s failed without raising an exception", - name_buf); + if (PyErr_Occurred()) { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_EXCEPTION); + } + else { + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING); } goto error; } else if (PyErr_Occurred()) { - _PyErr_FormatFromCause( - PyExc_SystemError, - "initialization of %s raised unreported exception", - name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNREPORTED_EXC); /* We would probably be correct to decref m here, * but we weren't doing so before, * so we stick with doing nothing. */ @@ -281,9 +431,8 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, /* This can happen when a PyModuleDef is returned without calling * PyModuleDef_Init on it */ - PyErr_Format(PyExc_SystemError, - "init function of %s returned uninitialized object", - name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_UNINITIALIZED); /* Likewise, decref'ing here makes sense. However, the original * code has a note about "prevent segfault in DECREF", * so we play it safe and leave it alone. */ @@ -293,48 +442,50 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, if (PyObject_TypeCheck(m, &PyModuleDef_Type)) { /* multi-phase init */ + res.kind = _Py_ext_module_kind_MULTIPHASE; res.def = (PyModuleDef *)m; /* Run PyModule_FromDefAndSpec() to finish loading the module. */ } else if (info->hook_prefix == nonascii_prefix) { - /* It should have been multi-phase init? */ + /* Non-ASCII is only supported for multi-phase init. */ + res.kind = _Py_ext_module_kind_MULTIPHASE; /* Don't allow legacy init for non-ASCII module names. */ - PyErr_Format( - PyExc_SystemError, - "initialization of %s did not return PyModuleDef", - name_buf); - Py_DECREF(m); - return -1; + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NONASCII_NOT_MULTIPHASE); + goto error; } else { /* single-phase init (legacy) */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; res.module = m; if (!PyModule_Check(m)) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return an extension " - "module", name_buf); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_NOT_MODULE); goto error; } res.def = _PyModule_GetDef(m); if (res.def == NULL) { - PyErr_Format(PyExc_SystemError, - "initialization of %s did not return a valid extension " - "module", name_buf); + PyErr_Clear(); + _Py_ext_module_loader_result_set_error( + &res, _Py_ext_module_loader_result_ERR_MISSING_DEF); goto error; } } assert(!PyErr_Occurred()); + assert(res.err == NULL); *p_res = res; return 0; error: - assert(PyErr_Occurred()); + assert(!PyErr_Occurred()); + assert(res.err != NULL); Py_CLEAR(res.module); res.def = NULL; *p_res = res; + p_res->err = &p_res->_err; return -1; } From a37b0932285b5e883b13a46ff2a32f15d7339894 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 1 May 2024 17:48:34 -0700 Subject: [PATCH 057/220] gh-118335: Rename --experimental-interpreter on Windows to --experimental-jit-interpreter (#118497) Also fix docs for this in whatsnew. --- Doc/whatsnew/3.13.rst | 3 ++- PCbuild/build.bat | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d67df8109a8ced3..52fb9752cf41325 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -946,7 +946,8 @@ The ``--enable-experimental-jit`` flag has the following optional values: The interpreter can be disabled by running with ``PYTHON_JIT=0``. -(On Windows, use ``PCbuild/build.bat --enable-jit`` to enable the JIT.) +(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) See :pep:`744` for more details. diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 43a99aa684483f3..13bd895694f176b 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -38,7 +38,7 @@ echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. echo. --experimental-jit Enable the experimental just-in-time compiler. echo. --experimental-jit-off Ditto but off by default (PYTHON_JIT=1 enables). -echo. --experimental-interpreter Enable the experimental Tier 2 interpreter. +echo. --experimental-jit-interpreter Enable the experimental Tier 2 interpreter. echo. echo.Available flags to avoid building certain modules. echo.These flags have no effect if '-e' is not given: @@ -91,8 +91,8 @@ if "%~1"=="-V" shift & goto Version if "%~1"=="--regen" (set Regen=true) & shift & goto CheckOpts if "%~1"=="--experimental-jit" (set UseJIT=true) & (set UseTIER2=1) & shift & goto CheckOpts if "%~1"=="--experimental-jit-off" (set UseJIT=true) & (set UseTIER2=3) & shift & goto CheckOpts -if "%~1"=="--experimental-interpreter" (set UseTIER2=4) & shift & goto CheckOpts -if "%~1"=="--experimental-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter" (set UseTIER2=4) & shift & goto CheckOpts +if "%~1"=="--experimental-jit-interpreter-off" (set UseTIER2=6) & shift & goto CheckOpts rem These use the actual property names used by MSBuild. We could just let rem them in through the environment, but we specify them on the command line rem anyway for visibility so set defaults after this From a6b610a94bee0e4436aee2825c14f05ec2f22f75 Mon Sep 17 00:00:00 2001 From: Andrew Zipperer <47086307+zipperer@users.noreply.github.com> Date: Thu, 2 May 2024 00:37:12 -0500 Subject: [PATCH 058/220] docs: typo: tiny grammar change: "pointed by" -> "pointed to by" (#118411) * docs: tiny grammar change: "pointed by" -> "pointed to by" This commit uses "file pointed to by" to replace "file pointed by" in - doc for shutil.copytree - docstring for shutil.copytree - docstring _abc.PathBase.open - docstring for pathlib.Path.open - doc for os.copy_file_range - doc for os.splice The docs use "file pointed to by" more frequently than "file pointed by". So, this commit replaces the uses of "file pointed by" in order to make the uses consistent through the docs. ```bash $ grep -ri 'pointed to by' cpython/ ``` yields more results than ```bash $ grep -ri 'pointed by' cpython/ ``` Separately: There are two occurrences of "tree pointed by": - cpython/Doc/library/xml.etree.elementtree.rst for `xml.etree.ElementInclude.include` - cpython/Lib/xml/etree/ElementInclude.py for `include` For those uses of "tree pointed by", I expect "tree pointed to by" instead. However, I found enough uses online of (a) "tree pointed by" rather than (b) "tree pointed to by" to convince me that (a) is in common use. So, this commit does not replace those occurrences of "tree pointed by" to "tree pointed to by". But I will replace them if a reviewer believes it is correct to replace them. * docs: typo: "exists and executable" -> "exists and is executable" --------- Co-authored-by: Andrew-Zipperer --- Doc/library/os.rst | 4 ++-- Doc/library/shutil.rst | 4 ++-- Lib/pathlib/__init__.py | 2 +- Lib/pathlib/_abc.py | 2 +- Lib/shutil.py | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 06ec7da14ebfaeb..844b5f26d8d4d14 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -926,7 +926,7 @@ as internal buffering of data. If *offset_src* is None, then *src* is read from the current position; respectively for *offset_dst*. - In Linux kernel older than 5.3, the files pointed by *src* and *dst* + In Linux kernel older than 5.3, the files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. @@ -1720,7 +1720,7 @@ or `the MSDN `_ on Windo At least one of the file descriptors must refer to a pipe. If *offset_src* is None, then *src* is read from the current position; respectively for *offset_dst*. The offset associated to the file descriptor that refers to a - pipe must be ``None``. The files pointed by *src* and *dst* must reside in + pipe must be ``None``. The files pointed to by *src* and *dst* must reside in the same filesystem, otherwise an :exc:`OSError` is raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 542d28864777924..331acfce3afee37 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -242,7 +242,7 @@ Directory and files operations be copied as far as the platform allows; if false or omitted, the contents and metadata of the linked files are copied to the new tree. - When *symlinks* is false, if the file pointed by the symlink doesn't + When *symlinks* is false, if the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an :exc:`Error` exception at the end of the copy process. You can set the optional *ignore_dangling_symlinks* flag to true if you @@ -447,7 +447,7 @@ Directory and files operations called. If no *cmd* would be called, return ``None``. *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. + determining if the file exists and is executable. *path* is a "``PATH`` string" specifying the lookup directory list. When no *path* is specified, the results of :func:`os.environ` are used, returning diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index f03f317ef6c16a3..8eecf2cefb790fa 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -540,7 +540,7 @@ def is_junction(self): def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ - Open the file pointed by this path and return a file object, as + Open the file pointed to by this path and return a file object, as the built-in open() function does. """ if "b" not in mode: diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 05698d5de24afb5..c7e8e2f68ed0587 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -637,7 +637,7 @@ def samefile(self, other_path): def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ - Open the file pointed by this path and return a file object, as + Open the file pointed to by this path and return a file object, as the built-in open() function does. """ raise UnsupportedOperation(self._unsupported_msg('open()')) diff --git a/Lib/shutil.py b/Lib/shutil.py index 910d6b6c63ac088..c9b4da34b1e19ba 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -556,7 +556,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. From f8e088df2a87f613ee23ea4f6787f87d9196b9de Mon Sep 17 00:00:00 2001 From: Andrej Date: Thu, 2 May 2024 13:57:45 +0500 Subject: [PATCH 059/220] gdb/libpython.py: Update PyLongObjectPtr docstring (GH-118438) --- Tools/gdb/libpython.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 74165acd831131b..5fdc812a00f0593 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -255,7 +255,7 @@ def proxyval(self, visited): Derived classes will override this. - For example, a PyIntObject* with ob_ival 42 in the inferior process + For example, a PyLongObjectPtr* with long_value 42 in the inferior process should result in an int(42) in this process. visited: a set of all gdb.Value pyobject pointers already visited @@ -867,7 +867,7 @@ class PyLongObjectPtr(PyObjectPtr): def proxyval(self, visited): ''' - Python's Include/longobjrep.h has this declaration: + Python's Include/longinterpr.h has this declaration: typedef struct _PyLongValue { uintptr_t lv_tag; /* Number of digits, sign and flags */ @@ -876,14 +876,18 @@ def proxyval(self, visited): struct _longobject { PyObject_HEAD - _PyLongValue long_value; + _PyLongValue long_value; }; with this description: The absolute value of a number is equal to - SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) - Negative numbers are represented with ob_size < 0; - zero is represented by ob_size == 0. + SUM(for i=0 through ndigits-1) ob_digit[i] * 2**(PyLong_SHIFT*i) + The sign of the value is stored in the lower 2 bits of lv_tag. + - 0: Positive + - 1: Zero + - 2: Negative + The third lowest bit of lv_tag is reserved for an immortality flag, but is + not currently used. where SHIFT can be either: #define PyLong_SHIFT 30 From 67bba9dd0f5b9c2d24c2bc6d239c4502040484af Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 2 May 2024 13:10:31 +0100 Subject: [PATCH 060/220] GH-117442: Check eval-breaker at start (rather than end) of tier 2 loops (GH-118482) --- Include/cpython/optimizer.h | 2 +- Include/internal/pycore_opcode_metadata.h | 2 +- Include/internal/pycore_uop_ids.h | 207 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 8 +- Python/bytecodes.c | 38 ++-- Python/executor_cases.c.h | 11 -- Python/generated_cases.c.h | 16 +- Python/optimizer.c | 22 +-- Python/optimizer_cases.c.h | 4 - Tools/jit/template.c | 1 - 10 files changed, 137 insertions(+), 174 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 60b35747deb4f3d..744a272251e75c1 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -142,7 +142,7 @@ void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); -extern void _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 808badea90d8eda..d10224c70f82f51 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1029,7 +1029,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [END_ASYNC_FOR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [END_FOR] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [END_SEND] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, - [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG }, + [ENTER_EXECUTOR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [EXIT_INIT_CHECK] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [EXTENDED_ARG] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 0a2231e88488c90..aa3940456b62f2e 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -94,47 +94,46 @@ extern "C" { #define _DYNAMIC_EXIT 343 #define _END_SEND END_SEND #define _ERROR_POP_N 344 -#define _EVAL_BREAKER_EXIT 345 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 346 +#define _FATAL_ERROR 345 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 347 -#define _FOR_ITER_GEN_FRAME 348 -#define _FOR_ITER_TIER_TWO 349 +#define _FOR_ITER 346 +#define _FOR_ITER_GEN_FRAME 347 +#define _FOR_ITER_TIER_TWO 348 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 350 -#define _GUARD_BOTH_INT 351 -#define _GUARD_BOTH_UNICODE 352 -#define _GUARD_BUILTINS_VERSION 353 -#define _GUARD_DORV_NO_DICT 354 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 355 -#define _GUARD_GLOBALS_VERSION 356 -#define _GUARD_IS_FALSE_POP 357 -#define _GUARD_IS_NONE_POP 358 -#define _GUARD_IS_NOT_NONE_POP 359 -#define _GUARD_IS_TRUE_POP 360 -#define _GUARD_KEYS_VERSION 361 -#define _GUARD_NOS_FLOAT 362 -#define _GUARD_NOS_INT 363 -#define _GUARD_NOT_EXHAUSTED_LIST 364 -#define _GUARD_NOT_EXHAUSTED_RANGE 365 -#define _GUARD_NOT_EXHAUSTED_TUPLE 366 -#define _GUARD_TOS_FLOAT 367 -#define _GUARD_TOS_INT 368 -#define _GUARD_TYPE_VERSION 369 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 370 -#define _INIT_CALL_PY_EXACT_ARGS 371 -#define _INIT_CALL_PY_EXACT_ARGS_0 372 -#define _INIT_CALL_PY_EXACT_ARGS_1 373 -#define _INIT_CALL_PY_EXACT_ARGS_2 374 -#define _INIT_CALL_PY_EXACT_ARGS_3 375 -#define _INIT_CALL_PY_EXACT_ARGS_4 376 +#define _GUARD_BOTH_FLOAT 349 +#define _GUARD_BOTH_INT 350 +#define _GUARD_BOTH_UNICODE 351 +#define _GUARD_BUILTINS_VERSION 352 +#define _GUARD_DORV_NO_DICT 353 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 354 +#define _GUARD_GLOBALS_VERSION 355 +#define _GUARD_IS_FALSE_POP 356 +#define _GUARD_IS_NONE_POP 357 +#define _GUARD_IS_NOT_NONE_POP 358 +#define _GUARD_IS_TRUE_POP 359 +#define _GUARD_KEYS_VERSION 360 +#define _GUARD_NOS_FLOAT 361 +#define _GUARD_NOS_INT 362 +#define _GUARD_NOT_EXHAUSTED_LIST 363 +#define _GUARD_NOT_EXHAUSTED_RANGE 364 +#define _GUARD_NOT_EXHAUSTED_TUPLE 365 +#define _GUARD_TOS_FLOAT 366 +#define _GUARD_TOS_INT 367 +#define _GUARD_TYPE_VERSION 368 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 369 +#define _INIT_CALL_PY_EXACT_ARGS 370 +#define _INIT_CALL_PY_EXACT_ARGS_0 371 +#define _INIT_CALL_PY_EXACT_ARGS_1 372 +#define _INIT_CALL_PY_EXACT_ARGS_2 373 +#define _INIT_CALL_PY_EXACT_ARGS_3 374 +#define _INIT_CALL_PY_EXACT_ARGS_4 375 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -151,65 +150,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 377 -#define _IS_NONE 378 +#define _INTERNAL_INCREMENT_OPT_COUNTER 376 +#define _IS_NONE 377 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 379 -#define _ITER_CHECK_RANGE 380 -#define _ITER_CHECK_TUPLE 381 -#define _ITER_JUMP_LIST 382 -#define _ITER_JUMP_RANGE 383 -#define _ITER_JUMP_TUPLE 384 -#define _ITER_NEXT_LIST 385 -#define _ITER_NEXT_RANGE 386 -#define _ITER_NEXT_TUPLE 387 -#define _JUMP_TO_TOP 388 +#define _ITER_CHECK_LIST 378 +#define _ITER_CHECK_RANGE 379 +#define _ITER_CHECK_TUPLE 380 +#define _ITER_JUMP_LIST 381 +#define _ITER_JUMP_RANGE 382 +#define _ITER_JUMP_TUPLE 383 +#define _ITER_NEXT_LIST 384 +#define _ITER_NEXT_RANGE 385 +#define _ITER_NEXT_TUPLE 386 +#define _JUMP_TO_TOP 387 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 389 -#define _LOAD_ATTR_CLASS 390 -#define _LOAD_ATTR_CLASS_0 391 -#define _LOAD_ATTR_CLASS_1 392 +#define _LOAD_ATTR 388 +#define _LOAD_ATTR_CLASS 389 +#define _LOAD_ATTR_CLASS_0 390 +#define _LOAD_ATTR_CLASS_1 391 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 393 -#define _LOAD_ATTR_INSTANCE_VALUE_0 394 -#define _LOAD_ATTR_INSTANCE_VALUE_1 395 -#define _LOAD_ATTR_METHOD_LAZY_DICT 396 -#define _LOAD_ATTR_METHOD_NO_DICT 397 -#define _LOAD_ATTR_METHOD_WITH_VALUES 398 -#define _LOAD_ATTR_MODULE 399 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 400 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 401 +#define _LOAD_ATTR_INSTANCE_VALUE 392 +#define _LOAD_ATTR_INSTANCE_VALUE_0 393 +#define _LOAD_ATTR_INSTANCE_VALUE_1 394 +#define _LOAD_ATTR_METHOD_LAZY_DICT 395 +#define _LOAD_ATTR_METHOD_NO_DICT 396 +#define _LOAD_ATTR_METHOD_WITH_VALUES 397 +#define _LOAD_ATTR_MODULE 398 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 399 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 400 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 402 -#define _LOAD_ATTR_SLOT_0 403 -#define _LOAD_ATTR_SLOT_1 404 -#define _LOAD_ATTR_WITH_HINT 405 +#define _LOAD_ATTR_SLOT 401 +#define _LOAD_ATTR_SLOT_0 402 +#define _LOAD_ATTR_SLOT_1 403 +#define _LOAD_ATTR_WITH_HINT 404 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 406 -#define _LOAD_CONST_INLINE_BORROW 407 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 408 -#define _LOAD_CONST_INLINE_WITH_NULL 409 +#define _LOAD_CONST_INLINE 405 +#define _LOAD_CONST_INLINE_BORROW 406 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 407 +#define _LOAD_CONST_INLINE_WITH_NULL 408 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 410 -#define _LOAD_FAST_0 411 -#define _LOAD_FAST_1 412 -#define _LOAD_FAST_2 413 -#define _LOAD_FAST_3 414 -#define _LOAD_FAST_4 415 -#define _LOAD_FAST_5 416 -#define _LOAD_FAST_6 417 -#define _LOAD_FAST_7 418 +#define _LOAD_FAST 409 +#define _LOAD_FAST_0 410 +#define _LOAD_FAST_1 411 +#define _LOAD_FAST_2 412 +#define _LOAD_FAST_3 413 +#define _LOAD_FAST_4 414 +#define _LOAD_FAST_5 415 +#define _LOAD_FAST_6 416 +#define _LOAD_FAST_7 417 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 419 -#define _LOAD_GLOBAL_BUILTINS 420 -#define _LOAD_GLOBAL_MODULE 421 +#define _LOAD_GLOBAL 418 +#define _LOAD_GLOBAL_BUILTINS 419 +#define _LOAD_GLOBAL_MODULE 420 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -223,51 +222,51 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 422 -#define _POP_JUMP_IF_FALSE 423 -#define _POP_JUMP_IF_TRUE 424 +#define _POP_FRAME 421 +#define _POP_JUMP_IF_FALSE 422 +#define _POP_JUMP_IF_TRUE 423 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 425 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 424 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 426 +#define _PUSH_FRAME 425 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 427 +#define _REPLACE_WITH_TRUE 426 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR -#define _SAVE_RETURN_OFFSET 428 -#define _SEND 429 +#define _SAVE_RETURN_OFFSET 427 +#define _SEND 428 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 430 -#define _START_EXECUTOR 431 -#define _STORE_ATTR 432 -#define _STORE_ATTR_INSTANCE_VALUE 433 -#define _STORE_ATTR_SLOT 434 +#define _SIDE_EXIT 429 +#define _START_EXECUTOR 430 +#define _STORE_ATTR 431 +#define _STORE_ATTR_INSTANCE_VALUE 432 +#define _STORE_ATTR_SLOT 433 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 435 -#define _STORE_FAST_0 436 -#define _STORE_FAST_1 437 -#define _STORE_FAST_2 438 -#define _STORE_FAST_3 439 -#define _STORE_FAST_4 440 -#define _STORE_FAST_5 441 -#define _STORE_FAST_6 442 -#define _STORE_FAST_7 443 +#define _STORE_FAST 434 +#define _STORE_FAST_0 435 +#define _STORE_FAST_1 436 +#define _STORE_FAST_2 437 +#define _STORE_FAST_3 438 +#define _STORE_FAST_4 439 +#define _STORE_FAST_5 440 +#define _STORE_FAST_6 441 +#define _STORE_FAST_7 442 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 444 +#define _STORE_SUBSCR 443 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 445 -#define _TO_BOOL 446 +#define _TIER2_RESUME_CHECK 444 +#define _TO_BOOL 445 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -277,13 +276,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 447 +#define _UNPACK_SEQUENCE 446 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 447 +#define MAX_UOP_ID 446 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 5697b13d1fc51d2..d0860024e0db4d8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -233,7 +233,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GUARD_IS_FALSE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NONE_POP] = HAS_EXIT_FLAG, [_GUARD_IS_NOT_NONE_POP] = HAS_EXIT_FLAG, - [_JUMP_TO_TOP] = HAS_EVAL_BREAK_FLAG, + [_JUMP_TO_TOP] = 0, [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, @@ -254,8 +254,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DEOPT] = 0, [_SIDE_EXIT] = 0, [_ERROR_POP_N] = HAS_ARG_FLAG, - [_TIER2_RESUME_CHECK] = HAS_EXIT_FLAG, - [_EVAL_BREAKER_EXIT] = HAS_ESCAPES_FLAG, + [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -339,7 +338,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_DYNAMIC_EXIT] = "_DYNAMIC_EXIT", [_END_SEND] = "_END_SEND", [_ERROR_POP_N] = "_ERROR_POP_N", - [_EVAL_BREAKER_EXIT] = "_EVAL_BREAKER_EXIT", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", [_FATAL_ERROR] = "_FATAL_ERROR", @@ -978,8 +976,6 @@ int _PyUop_num_popped(int opcode, int oparg) return oparg; case _TIER2_RESUME_CHECK: return 0; - case _EVAL_BREAKER_EXIT: - return 0; default: return -1; } diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 002b5a3529c127a..b1be40dd38ce213 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2410,20 +2410,21 @@ dummy_func( tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 - int prevoparg = oparg; - CHECK_EVAL_BREAKER(); - if (this_instr->op.code != ENTER_EXECUTOR || - this_instr->op.arg != prevoparg) { - next_instr = this_instr; - DISPATCH(); - } - PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + DISPATCH_GOTO(); + } tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); @@ -4111,7 +4112,6 @@ dummy_func( #ifndef _Py_JIT next_uop = ¤t_executor->trace[1]; #endif - CHECK_EVAL_BREAKER(); } tier2 op(_SET_IP, (instr_ptr/4 --)) { @@ -4277,30 +4277,18 @@ dummy_func( GOTO_UNWIND(); } - - /* Special version of RESUME_CHECK that (when paired with _EVAL_BREAKER_EXIT) - * is safe for tier 2. Progress is guaranteed because _EVAL_BREAKER_EXIT calls - * _Py_HandlePending which clears the eval_breaker so that _TIER2_RESUME_CHECK - * will not exit if it is immediately executed again. */ + /* Progress is guaranteed if we DEOPT on the eval breaker, because + * ENTER_EXECUTOR will not re-enter tier 2 with the eval breaker set. */ tier2 op(_TIER2_RESUME_CHECK, (--)) { #if defined(__EMSCRIPTEN__) - EXIT_IF(_Py_emscripten_signal_clock == 0); + DEOPT_IF(_Py_emscripten_signal_clock == 0); _Py_emscripten_signal_clock -= Py_EMSCRIPTEN_SIGNAL_HANDLING; #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); - EXIT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); + DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } - tier2 op(_EVAL_BREAKER_EXIT, (--)) { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_HandlePending(tstate) != 0) { - GOTO_UNWIND(); - } - EXIT_TO_TRACE(); - } - // END BYTECODES // } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index b17f3762714c72c..f97f279836eb660 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4094,7 +4094,6 @@ #ifndef _Py_JIT next_uop = ¤t_executor->trace[1]; #endif - CHECK_EVAL_BREAKER(); break; } @@ -4353,14 +4352,4 @@ break; } - case _EVAL_BREAKER_EXIT: { - _Py_CHECK_EMSCRIPTEN_SIGNALS_PERIODICALLY(); - QSBR_QUIESCENT_STATE(tstate); - if (_Py_HandlePending(tstate) != 0) { - GOTO_UNWIND(); - } - EXIT_TO_TRACE(); - break; - } - #undef TIER_TWO diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 32b485ecb9d8cc8..2a0f268ce6ed547 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2493,19 +2493,21 @@ next_instr += 1; INSTRUCTION_STATS(ENTER_EXECUTOR); #ifdef _Py_TIER2 - int prevoparg = oparg; - CHECK_EVAL_BREAKER(); - if (this_instr->op.code != ENTER_EXECUTOR || - this_instr->op.arg != prevoparg) { - next_instr = this_instr; - DISPATCH(); - } PyCodeObject *code = _PyFrame_GetCode(frame); _PyExecutorObject *executor = code->co_executors->executors[oparg & 255]; assert(executor->vm_data.index == INSTR_OFFSET() - 1); assert(executor->vm_data.code == code); assert(executor->vm_data.valid); assert(tstate->previous_executor == NULL); + /* If the eval breaker is set then stay in tier 1. + * This avoids any potentially infinite loops + * involving _RESUME_CHECK */ + if (_Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & _PY_EVAL_EVENTS_MASK) { + opcode = executor->vm_data.opcode; + oparg = (oparg & ~255) | executor->vm_data.oparg; + next_instr = this_instr; + DISPATCH_GOTO(); + } tstate->previous_executor = Py_None; Py_INCREF(executor); GOTO_TIER_TWO(executor); diff --git a/Python/optimizer.c b/Python/optimizer.c index 2389338531b0f35..050947396d1724c 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -627,6 +627,9 @@ translate_bytecode_to_trace( if (opcode == JUMP_BACKWARD || opcode == JUMP_BACKWARD_NO_INTERRUPT) { instr += 1 + _PyOpcode_Caches[opcode] - (int32_t)oparg; initial_instr = instr; + if (opcode == JUMP_BACKWARD) { + ADD_TO_TRACE(_TIER2_RESUME_CHECK, 0, 0, target); + } continue; } else { @@ -976,6 +979,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t current_error = -1; int32_t current_error_target = -1; int32_t current_popped = -1; + int32_t current_exit_op = -1; /* Leaving in NOPs slows down the interpreter and messes up the stats */ _PyUOpInstruction *copy_to = &buffer[0]; for (int i = 0; i < length; i++) { @@ -994,20 +998,11 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int opcode = inst->opcode; int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { - if (target != current_jump_target) { - uint16_t exit_op; - if (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) { - if (opcode == _TIER2_RESUME_CHECK) { - exit_op = _EVAL_BREAKER_EXIT; - } - else { - exit_op = _SIDE_EXIT; - } - } - else { - exit_op = _DEOPT; - } + uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? + _SIDE_EXIT : _DEOPT; + if (target != current_jump_target || current_exit_op != exit_op) { make_exit(&buffer[next_spare], exit_op, target); + current_exit_op = exit_op; current_jump_target = target; current_jump = next_spare; next_spare++; @@ -1114,7 +1109,6 @@ sanity_check(_PyExecutorObject *executor) CHECK( opcode == _DEOPT || opcode == _SIDE_EXIT || - opcode == _EVAL_BREAKER_EXIT || opcode == _ERROR_POP_N); if (opcode == _SIDE_EXIT) { CHECK(inst->format == UOP_FORMAT_EXIT); diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 0b696921ebfc9f4..e680d76141776f7 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2153,7 +2153,3 @@ break; } - case _EVAL_BREAKER_EXIT: { - break; - } - diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 0dd0744f7aec9ca..a81e866e9da4b3e 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -106,7 +106,6 @@ _JIT_ENTRY(_PyInterpreterFrame *frame, PyObject **stack_pointer, PyThreadState * // The actual instruction definitions (only one will be used): if (uopcode == _JUMP_TO_TOP) { - CHECK_EVAL_BREAKER(); PATCH_JUMP(_JIT_TOP); } switch (uopcode) { From 7c97dc8c9594c71bd3d1f69758a27de45f57e4c3 Mon Sep 17 00:00:00 2001 From: Crowthebird <78076854+thatbirdguythatuknownot@users.noreply.github.com> Date: Thu, 2 May 2024 21:32:20 +0800 Subject: [PATCH 061/220] gh-118216: Don't consider dotted `__future__` imports (#118267) --- Doc/whatsnew/3.13.rst | 4 ++++ Lib/test/test_future_stmt/test_future.py | 19 +++++++++++++++++++ ...-04-25-11-48-28.gh-issue-118216.SVg700.rst | 1 + Python/compile.c | 2 +- Python/future.c | 2 +- 5 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-25-11-48-28.gh-issue-118216.SVg700.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 52fb9752cf41325..3ccf17be9796d50 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -288,6 +288,10 @@ Other Language Changes class scopes are not inlined into their parent scope. (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) +* ``from __future__ import ...`` statements are now just normal + relative imports if dots are present before the module name. + (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) + New Modules =========== diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 2c8ceb664cb362d..69ae58b0fbcae35 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -203,6 +203,25 @@ def test_syntactical_future_repl(self): out = kill_python(p) self.assertNotIn(b'SyntaxError: invalid syntax', out) + def test_future_dotted_import(self): + with self.assertRaises(ImportError): + exec("from .__future__ import spam") + + code = dedent( + """ + from __future__ import print_function + from ...__future__ import ham + """ + ) + with self.assertRaises(ImportError): + exec(code) + + code = """ + from .__future__ import nested_scopes + from __future__ import barry_as_FLUFL + """ + self.assertSyntaxError(code, lineno=2) + class AnnotationsFutureTestCase(unittest.TestCase): template = dedent( """ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-25-11-48-28.gh-issue-118216.SVg700.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-11-48-28.gh-issue-118216.SVg700.rst new file mode 100644 index 000000000000000..937cdffefda6f1f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-25-11-48-28.gh-issue-118216.SVg700.rst @@ -0,0 +1 @@ +Don't consider :mod:`__future__` imports with dots before the module name. diff --git a/Python/compile.c b/Python/compile.c index 4e94f9297a32d17..feedd9888343970 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -3849,7 +3849,7 @@ compiler_from_import(struct compiler *c, stmt_ty s) } if (location_is_after(LOC(s), c->c_future.ff_location) && - s->v.ImportFrom.module && + s->v.ImportFrom.module && s->v.ImportFrom.level == 0 && _PyUnicode_EqualToASCIIString(s->v.ImportFrom.module, "__future__")) { Py_DECREF(names); diff --git a/Python/future.c b/Python/future.c index 399345bd8fcbd95..8d94d515605dcd6 100644 --- a/Python/future.c +++ b/Python/future.c @@ -77,7 +77,7 @@ future_parse(_PyFutureFeatures *ff, mod_ty mod, PyObject *filename) * are another future statement and a doc string. */ - if (s->kind == ImportFrom_kind) { + if (s->kind == ImportFrom_kind && s->v.ImportFrom.level == 0) { identifier modname = s->v.ImportFrom.module; if (modname && _PyUnicode_EqualToASCIIString(modname, "__future__")) { From 7d2ffada0a6490e6839697f729bcd80380e9f561 Mon Sep 17 00:00:00 2001 From: NGRsoftlab <78017794+NGRsoftlab@users.noreply.github.com> Date: Thu, 2 May 2024 16:43:03 +0300 Subject: [PATCH 062/220] gh-116180: Check the globals argument in PyRun_* C API (GH-116637) It used to crash when passing NULL or non-dict as globals. Now it sets a SystemError. --- Lib/test/test_capi/test_run.py | 36 ++++++++++++++++++++++------------ Python/pythonrun.c | 21 +++++++++++--------- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py index 3a390273ec21add..894f66b437a39c2 100644 --- a/Lib/test/test_capi/test_run.py +++ b/Lib/test/test_capi/test_run.py @@ -11,6 +11,10 @@ Py_eval_input = _testcapi.Py_eval_input +class DictSubclass(dict): + pass + + class CAPITest(unittest.TestCase): # TODO: Test the following functions: # @@ -50,15 +54,19 @@ def run(s, *args): self.assertRaises(TypeError, run, b'a\n', dict(a=1), []) self.assertRaises(TypeError, run, b'a\n', dict(a=1), 1) + self.assertIsNone(run(b'a\n', DictSubclass(a=1))) + self.assertIsNone(run(b'a\n', DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, b'a\n', DictSubclass()) + self.assertIsNone(run(b'\xc3\xa4\n', {'\xe4': 1})) self.assertRaises(SyntaxError, run, b'\xe4\n', {}) - # CRASHES run(b'a\n', NULL) - # CRASHES run(b'a\n', NULL, {}) - # CRASHES run(b'a\n', NULL, dict(a=1)) - # CRASHES run(b'a\n', UserDict()) - # CRASHES run(b'a\n', UserDict(), {}) - # CRASHES run(b'a\n', UserDict(), dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', NULL) + self.assertRaises(SystemError, run, b'a\n', NULL, {}) + self.assertRaises(SystemError, run, b'a\n', NULL, dict(a=1)) + self.assertRaises(SystemError, run, b'a\n', UserDict()) + self.assertRaises(SystemError, run, b'a\n', UserDict(), {}) + self.assertRaises(SystemError, run, b'a\n', UserDict(), dict(a=1)) # CRASHES run(NULL, {}) @@ -82,12 +90,16 @@ def run(*args): self.assertRaises(TypeError, run, dict(a=1), []) self.assertRaises(TypeError, run, dict(a=1), 1) - # CRASHES run(NULL) - # CRASHES run(NULL, {}) - # CRASHES run(NULL, dict(a=1)) - # CRASHES run(UserDict()) - # CRASHES run(UserDict(), {}) - # CRASHES run(UserDict(), dict(a=1)) + self.assertIsNone(run(DictSubclass(a=1))) + self.assertIsNone(run(DictSubclass(), dict(a=1))) + self.assertRaises(NameError, run, DictSubclass()) + + self.assertRaises(SystemError, run, NULL) + self.assertRaises(SystemError, run, NULL, {}) + self.assertRaises(SystemError, run, NULL, dict(a=1)) + self.assertRaises(SystemError, run, UserDict()) + self.assertRaises(SystemError, run, UserDict(), {}) + self.assertRaises(SystemError, run, UserDict(), dict(a=1)) @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') @unittest.skipIf(os.name == 'nt', 'does not work on Windows') diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 2970248da137055..31213aec3cd9c30 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1275,17 +1275,20 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py _PyRuntime.signals.unhandled_keyboard_interrupt = 0; /* Set globals['__builtins__'] if it doesn't exist */ - if (globals != NULL) { - int has_builtins = PyDict_ContainsString(globals, "__builtins__"); - if (has_builtins < 0) { + if (!globals || !PyDict_Check(globals)) { + PyErr_SetString(PyExc_SystemError, "globals must be a real dict"); + return NULL; + } + int has_builtins = PyDict_ContainsString(globals, "__builtins__"); + if (has_builtins < 0) { + return NULL; + } + if (!has_builtins) { + if (PyDict_SetItemString(globals, "__builtins__", + tstate->interp->builtins) < 0) + { return NULL; } - if (!has_builtins) { - if (PyDict_SetItemString(globals, "__builtins__", - tstate->interp->builtins) < 0) { - return NULL; - } - } } v = PyEval_EvalCode((PyObject*)co, globals, locals); From b3372481b6cae5766330b041c4622c28cee2119f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Thu, 2 May 2024 16:56:33 +0300 Subject: [PATCH 063/220] gh-117903: Clarify that the staticmethod descriptor is callable (GH-117925) --- Doc/library/functions.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index e598ef423de4972..be3a64cf41b425a 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1733,8 +1733,9 @@ are always available. They are listed here in alphabetical order. :ref:`function` for details. A static method can be called either on the class (such as ``C.f()``) or on - an instance (such as ``C().f()``). Moreover, they can be called as regular - functions (such as ``f()``). + an instance (such as ``C().f()``). + Moreover, the static method :term:`descriptor` is also callable, so it can + be used in the class definition (such as ``f()``). Static methods in Python are similar to those found in Java or C++. Also, see :func:`classmethod` for a variant that is useful for creating alternate class From 81939dad77001556c527485d31a2d0f4a759033e Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 2 May 2024 15:20:43 +0100 Subject: [PATCH 064/220] gh-118486: Support mkdir(mode=0o700) on Windows (GH-118488) --- Doc/library/os.rst | 7 + Lib/test/test_os.py | 19 +++ Lib/test/test_tempfile.py | 28 ++++ ...-05-01-20-57-09.gh-issue-118486.K44KJG.rst | 2 + Modules/posixmodule.c | 158 +++++++++++++++++- 5 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 844b5f26d8d4d14..6c92eed9c063f13 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2430,6 +2430,10 @@ features: platform-dependent. On some platforms, they are ignored and you should call :func:`chmod` explicitly to set them. + On Windows, a *mode* of ``0o700`` is specifically handled to apply access + control to the new directory such that only the current user and + administrators have access. Other values of *mode* are ignored. + This function can also support :ref:`paths relative to directory descriptors `. @@ -2444,6 +2448,9 @@ features: .. versionchanged:: 3.6 Accepts a :term:`path-like object`. + .. versionchanged:: 3.13 + Windows now handles a *mode* of ``0o700``. + .. function:: makedirs(name, mode=0o777, exist_ok=False) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index eaa676673f8af07..9c9c8536dc75424 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1811,6 +1811,25 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path1 = os.path.join(os_helper.TESTFN, 'dir1') + path2 = os.path.join(os_helper.TESTFN, 'dir2') + # mode=0o700 is special-cased to override ACLs on Windows + # There's no way to know exactly how the ACLs will look, so we'll + # check that they are different from a regularly created directory. + os.mkdir(path1, mode=0o700) + os.mkdir(path2, mode=0o777) + + out1 = subprocess.check_output(["icacls.exe", path1], encoding="oem") + out2 = subprocess.check_output(["icacls.exe", path2], encoding="oem") + os.rmdir(path1) + os.rmdir(path2) + out1 = out1.replace(path1, "") + out2 = out2.replace(path2, "") + self.assertNotEqual(out1, out2) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b64b6a4f2baeb57..19ddeaa169bf935 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -13,6 +13,7 @@ import weakref import gc import shutil +import subprocess from unittest import mock import unittest @@ -803,6 +804,33 @@ def test_mode(self): finally: os.rmdir(dir) + @unittest.skipUnless(os.name == "nt", "Only on Windows.") + def test_mode_win32(self): + # Use icacls.exe to extract the users with some level of access + # Main thing we are testing is that the BUILTIN\Users group has + # no access. The exact ACL is going to vary based on which user + # is running the test. + dir = self.do_create() + try: + out = subprocess.check_output(["icacls.exe", dir], encoding="oem").casefold() + finally: + os.rmdir(dir) + + dir = dir.casefold() + users = set() + found_user = False + for line in out.strip().splitlines(): + acl = None + # First line of result includes our directory + if line.startswith(dir): + acl = line.removeprefix(dir).strip() + elif line and line[:1].isspace(): + acl = line.strip() + if acl: + users.add(acl.partition(":")[0]) + + self.assertNotIn(r"BUILTIN\Users".casefold(), users) + def test_collision_with_existing_file(self): # mkdtemp tries another name when a file with # the chosen name already exists diff --git a/Misc/NEWS.d/next/Windows/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst b/Misc/NEWS.d/next/Windows/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst new file mode 100644 index 000000000000000..cdbce9a0bebf6b9 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-01-20-57-09.gh-issue-118486.K44KJG.rst @@ -0,0 +1,2 @@ +:func:`os.mkdir` now accepts *mode* of ``0o700`` to restrict the new +directory to the current user. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 722159a39d098f0..f9533577a8fa340 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -37,6 +37,8 @@ # include # include // UNLEN # include "osdefs.h" // SEP +# include // SetEntriesInAcl +# include // SDDL_REVISION_1 # if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SYMLINK # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ @@ -5539,6 +5541,133 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } +#ifdef MS_WINDOWS + +/* We centralise SECURITY_ATTRIBUTE initialization based around +templates that will probably mostly match common POSIX mode settings. +The _Py_SECURITY_ATTRIBUTE_DATA structure contains temporary data, as +a constructed SECURITY_ATTRIBUTE structure typically refers to memory +that has to be alive while it's being used. + +Typical use will look like: + SECURITY_ATTRIBUTES *pSecAttr = NULL; + struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; + int error, error2; + + Py_BEGIN_ALLOW_THREADS + switch (mode) { + case 0x1C0: // 0o700 + error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); + break; + ... + default: + error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); + break; + } + + if (!error) { + // do operation, passing pSecAttr + } + + // Unconditionally clear secAttrData. + error2 = clearSecurityAttributes(&pSecAttr, &secAttrData); + if (!error) { + error = error2; + } + Py_END_ALLOW_THREADS + + if (error) { + PyErr_SetFromWindowsErr(error); + return NULL; + } +*/ + +struct _Py_SECURITY_ATTRIBUTE_DATA { + SECURITY_ATTRIBUTES securityAttributes; + PACL acl; + SECURITY_DESCRIPTOR sd; + EXPLICIT_ACCESS_W ea[4]; +}; + +static int +initializeDefaultSecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + memset(data, 0, sizeof(*data)); + return 0; +} + +static int +initializeMkdir700SecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + memset(data, 0, sizeof(*data)); + + if (!InitializeSecurityDescriptor(&data->sd, SECURITY_DESCRIPTOR_REVISION) + || !SetSecurityDescriptorGroup(&data->sd, NULL, TRUE)) { + return GetLastError(); + } + + data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); + data->ea[0].grfAccessPermissions = GENERIC_ALL; + data->ea[0].grfAccessMode = SET_ACCESS; + data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; + + data->ea[1].grfAccessPermissions = GENERIC_ALL; + data->ea[1].grfAccessMode = SET_ACCESS; + data->ea[1].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[1].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[1].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[1].Trustee.ptstrName = L"SYSTEM"; + + data->ea[2].grfAccessPermissions = GENERIC_ALL; + data->ea[2].grfAccessMode = SET_ACCESS; + data->ea[2].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[2].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[2].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[2].Trustee.ptstrName = L"ADMINISTRATORS"; + + int r = SetEntriesInAclW(3, data->ea, NULL, &data->acl); + if (r) { + return r; + } + if (!SetSecurityDescriptorDacl(&data->sd, TRUE, data->acl, FALSE)) { + return GetLastError(); + } + data->securityAttributes.lpSecurityDescriptor = &data->sd; + *securityAttributes = &data->securityAttributes; + return 0; +} + +static int +clearSecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + if (data->acl) { + if (LocalFree((void *)data->acl)) { + return GetLastError(); + } + } + return 0; +} + +#endif + /*[clinic input] os.mkdir @@ -5568,6 +5697,12 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ { int result; +#ifdef MS_WINDOWS + int error = 0; + int pathError = 0; + SECURITY_ATTRIBUTES *pSecAttr = NULL; + struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; +#endif #ifdef HAVE_MKDIRAT int mkdirat_unavailable = 0; #endif @@ -5579,11 +5714,30 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS - result = CreateDirectoryW(path->wide, NULL); + switch (mode) { + case 0x1C0: // 0o700 + error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); + break; + default: + error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); + break; + } + if (!error) { + result = CreateDirectoryW(path->wide, pSecAttr); + error = clearSecurityAttributes(&pSecAttr, &secAttrData); + } else { + // Ignore error from "clear" - we have a more interesting one already + clearSecurityAttributes(&pSecAttr, &secAttrData); + } Py_END_ALLOW_THREADS - if (!result) + if (error) { + PyErr_SetFromWindowsErr(error); + return NULL; + } + if (!result) { return path_error(path); + } #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT From 9789440de387219bb7677fe0d66860aa8c9deb02 Mon Sep 17 00:00:00 2001 From: Sergey B Kirpichev Date: Thu, 2 May 2024 17:44:33 +0300 Subject: [PATCH 065/220] gh-82062: Fix support of parameter defaults on methods in extension modules (GH-115270) Now inspect.signature() supports references to the module globals in parameter defaults on methods in extension modules. Previously it was only supported in functions. The workaround was to specify the fully qualified name, including the module name. --- Lib/inspect.py | 5 +++++ Lib/test/test_inspect/test_inspect.py | 7 +++++++ .../2024-02-11-07-31-43.gh-issue-82062.eeS6w7.rst | 3 +++ Modules/_testcapi/docstring.c | 10 ++++++++++ 4 files changed, 25 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-02-11-07-31-43.gh-issue-82062.eeS6w7.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index d46514f4b104677..a0c80bd5c8b6014 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2285,7 +2285,12 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index fbef34eddacb3a4..d12240353ff8324 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -3069,6 +3069,13 @@ def test_signature_on_builtins_no_signature(self): self.assertEqual(inspect.signature(builtin), inspect.signature(template)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + def test_signature_on_non_function(self): with self.assertRaisesRegex(TypeError, 'is not a callable object'): inspect.signature(42) diff --git a/Misc/NEWS.d/next/Library/2024-02-11-07-31-43.gh-issue-82062.eeS6w7.rst b/Misc/NEWS.d/next/Library/2024-02-11-07-31-43.gh-issue-82062.eeS6w7.rst new file mode 100644 index 000000000000000..a57a5918b135bb8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-02-11-07-31-43.gh-issue-82062.eeS6w7.rst @@ -0,0 +1,3 @@ +Fix :func:`inspect.signature()` to correctly handle parameter defaults +on methods in extension modules that use names defined in the module +namespace. diff --git a/Modules/_testcapi/docstring.c b/Modules/_testcapi/docstring.c index d99fbdd904b5949..3f7acbae1b181b2 100644 --- a/Modules/_testcapi/docstring.c +++ b/Modules/_testcapi/docstring.c @@ -169,6 +169,13 @@ static PyMethodDef DocStringUnrepresentableSignatureTest_methods[] = { "--\n\n" "This docstring has a signature with unrepresentable default." )}, + {"with_default", + (PyCFunction)test_with_docstring, METH_VARARGS, + PyDoc_STR( + "with_default($self, /, x=ONE)\n" + "--\n\n" + "This instance method has a default parameter value from the module scope." + )}, {NULL}, }; @@ -193,5 +200,8 @@ _PyTestCapi_Init_Docstring(PyObject *mod) if (PyModule_AddType(mod, &DocStringUnrepresentableSignatureTest) < 0) { return -1; } + if (PyModule_AddObject(mod, "ONE", PyLong_FromLong(1)) < 0) { + return -1; + } return 0; } From 72867c962cc59c6d56805f86530696bea6beb039 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Thu, 2 May 2024 16:17:59 +0100 Subject: [PATCH 066/220] GH-118095: Unify the behavior of tier 2 FOR_ITER branch micro-ops (GH-118420) * Target _FOR_ITER_TIER_TWO at POP_TOP following the matching END_FOR * Modify _GUARD_NOT_EXHAUSTED_RANGE, _GUARD_NOT_EXHAUSTED_LIST and _GUARD_NOT_EXHAUSTED_TUPLE so that they also target the POP_TOP following the matching END_FOR --- Python/bytecodes.c | 4 +-- Python/executor_cases.c.h | 4 +-- Python/optimizer.c | 50 ++++++++++++++++++++++++++++++-------- Python/optimizer_symbols.c | 13 +++++++--- Python/specialize.c | 6 +++-- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b1be40dd38ce213..9769cfe68aaeba2 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2610,9 +2610,7 @@ dummy_func( _PyErr_Clear(tstate); } /* iterator ended normally */ - Py_DECREF(iter); - STACK_SHRINK(1); - /* The translator sets the deopt target just past END_FOR */ + /* The translator sets the deopt target just past the matching END_FOR */ DEOPT_IF(true); } // Common case: no jump, leave it to the code generator diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index f97f279836eb660..03db9b623cbd869 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -2664,9 +2664,7 @@ _PyErr_Clear(tstate); } /* iterator ended normally */ - Py_DECREF(iter); - STACK_SHRINK(1); - /* The translator sets the deopt target just past END_FOR */ + /* The translator sets the deopt target just past the matching END_FOR */ if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); diff --git a/Python/optimizer.c b/Python/optimizer.c index 050947396d1724c..56768ae8f542f61 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -23,6 +23,19 @@ #define MAX_EXECUTORS_SIZE 256 +#ifdef Py_DEBUG +static int +base_opcode(PyCodeObject *code, int offset) +{ + int opcode = _Py_GetBaseOpcode(code, offset); + if (opcode == ENTER_EXECUTOR) { + int oparg = _PyCode_CODE(code)[offset].op.arg; + _PyExecutorObject *ex = code->co_executors->executors[oparg]; + return ex->vm_data.opcode; + } + return opcode; +} +#endif static bool has_space_for_executor(PyCodeObject *code, _Py_CODEUNIT *instr) @@ -445,6 +458,14 @@ _PyUOp_Replacements[MAX_UOP_ID + 1] = { [_FOR_ITER] = _FOR_ITER_TIER_TWO, }; +static const uint8_t +is_for_iter_test[MAX_UOP_ID + 1] = { + [_GUARD_NOT_EXHAUSTED_RANGE] = 1, + [_GUARD_NOT_EXHAUSTED_LIST] = 1, + [_GUARD_NOT_EXHAUSTED_TUPLE] = 1, + [_FOR_ITER_TIER_TWO] = 1, +}; + static const uint16_t BRANCH_TO_GUARD[4][2] = { [POP_JUMP_IF_FALSE - POP_JUMP_IF_FALSE][0] = _GUARD_IS_TRUE_POP, @@ -594,7 +615,6 @@ translate_bytecode_to_trace( uint32_t opcode = instr->op.code; uint32_t oparg = instr->op.arg; - uint32_t extended = 0; DPRINTF(2, "%d: %s(%d)\n", target, _PyOpcode_OpName[opcode], oparg); @@ -608,7 +628,6 @@ translate_bytecode_to_trace( if (opcode == EXTENDED_ARG) { instr++; - extended = 1; opcode = instr->op.code; oparg = (oparg << 8) | instr->op.arg; if (opcode == EXTENDED_ARG) { @@ -772,12 +791,15 @@ translate_bytecode_to_trace( case OPARG_REPLACED: uop = _PyUOp_Replacements[uop]; assert(uop != 0); - if (uop == _FOR_ITER_TIER_TWO) { - target += 1 + INLINE_CACHE_ENTRIES_FOR_ITER + oparg + 2 + extended; - assert(_PyCode_CODE(code)[target-2].op.code == END_FOR || - _PyCode_CODE(code)[target-2].op.code == INSTRUMENTED_END_FOR); - assert(_PyCode_CODE(code)[target-1].op.code == POP_TOP); +#ifdef Py_DEBUG + { + uint32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + (oparg > 255); + uint32_t jump_target = next_inst + oparg; + assert(base_opcode(code, jump_target) == END_FOR || + base_opcode(code, jump_target) == INSTRUMENTED_END_FOR); + assert(base_opcode(code, jump_target+1) == POP_TOP); } +#endif break; default: fprintf(stderr, @@ -1000,10 +1022,18 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? _SIDE_EXIT : _DEOPT; - if (target != current_jump_target || current_exit_op != exit_op) { - make_exit(&buffer[next_spare], exit_op, target); + int32_t jump_target = target; + if (is_for_iter_test[opcode]) { + /* Target the POP_TOP immediately after the END_FOR, + * leaving only the iterator on the stack. */ + int extended_arg = inst->oparg > 255; + int32_t next_inst = target + 1 + INLINE_CACHE_ENTRIES_FOR_ITER + extended_arg; + jump_target = next_inst + inst->oparg + 1; + } + if (jump_target != current_jump_target || current_exit_op != exit_op) { + make_exit(&buffer[next_spare], exit_op, jump_target); current_exit_op = exit_op; - current_jump_target = target; + current_jump_target = jump_target; current_jump = next_spare; next_spare++; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index d52f490853c006a..4aeb04fe0405d22 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -164,19 +164,26 @@ _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val) return true; } - bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym) { + if (_Py_uop_sym_is_not_null(sym)) { + sym_set_bottom(sym); + return false; + } sym_set_flag(sym, IS_NULL); - return !_Py_uop_sym_is_bottom(sym); + return true; } bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym) { + if (_Py_uop_sym_is_null(sym)) { + sym_set_bottom(sym); + return false; + } sym_set_flag(sym, NOT_NULL); - return !_Py_uop_sym_is_bottom(sym); + return true; } diff --git a/Python/specialize.c b/Python/specialize.c index ee51781372166a8..72114f27f69c52a 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -215,6 +215,7 @@ print_gc_stats(FILE *out, GCStats *stats) } } +#ifdef _Py_TIER2 static void print_histogram(FILE *out, const char *name, uint64_t hist[_Py_UOP_HIST_SIZE]) { @@ -249,7 +250,6 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) stats->optimizer_failure_reason_no_memory); fprintf(out, "Optimizer remove globals builtins changed: %" PRIu64 "\n", stats->remove_globals_builtins_changed); fprintf(out, "Optimizer remove globals incorrect keys: %" PRIu64 "\n", stats->remove_globals_incorrect_keys); - for (int i = 0; i <= MAX_UOP_ID; i++) { if (stats->opcode[i].execution_count) { fprintf(out, "uops[%s].execution_count : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].execution_count); @@ -258,7 +258,6 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) fprintf(out, "uops[%s].specialization.miss : %" PRIu64 "\n", _PyUOpName(i), stats->opcode[i].miss); } } - for (int i = 0; i < 256; i++) { if (stats->unsupported_opcode[i]) { fprintf( @@ -289,6 +288,7 @@ print_optimization_stats(FILE *out, OptimizationStats *stats) } } } +#endif static void print_rare_event_stats(FILE *out, RareEventStats *stats) @@ -309,7 +309,9 @@ print_stats(FILE *out, PyStats *stats) print_call_stats(out, &stats->call_stats); print_object_stats(out, &stats->object_stats); print_gc_stats(out, stats->gc_stats); +#ifdef _Py_TIER2 print_optimization_stats(out, &stats->optimization_stats); +#endif print_rare_event_stats(out, &stats->rare_event_stats); } From 2770d5caca42d48102f8e18210132a964c34af7c Mon Sep 17 00:00:00 2001 From: Raphael Gaschignard Date: Fri, 3 May 2024 01:55:29 +1000 Subject: [PATCH 067/220] gh-105879: Add support for keyword arguments to eval and exec (#105885) Co-authored-by: Jelle Zijlstra --- Doc/library/functions.rst | 18 ++-- Lib/test/test_builtin.py | 20 +++++ ...-06-18-00-27-57.gh-issue-105879.dPw78k.rst | 2 + Python/bltinmodule.c | 8 +- Python/clinic/bltinmodule.c.h | 83 +++++++++++++------ 5 files changed, 98 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-06-18-00-27-57.gh-issue-105879.dPw78k.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index be3a64cf41b425a..4fdf30e0d252ae1 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -524,11 +524,11 @@ are always available. They are listed here in alphabetical order. .. _func-eval: -.. function:: eval(expression, globals=None, locals=None) +.. function:: eval(source, /, globals=None, locals=None) - :param expression: + :param source: A Python expression. - :type expression: :class:`str` | :ref:`code object ` + :type source: :class:`str` | :ref:`code object ` :param globals: The global namespace (default: ``None``). @@ -583,11 +583,15 @@ are always available. They are listed here in alphabetical order. Raises an :ref:`auditing event ` ``exec`` with the code object as the argument. Code compilation events may also be raised. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + .. index:: pair: built-in function; exec -.. function:: exec(object, globals=None, locals=None, /, *, closure=None) +.. function:: exec(source, /, globals=None, locals=None, *, closure=None) - This function supports dynamic execution of Python code. *object* must be + This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as a suite of Python statements which is then executed (unless a syntax error occurs). [#]_ If it is a code object, it is simply executed. In all cases, @@ -640,6 +644,10 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 Added the *closure* parameter. + .. versionchanged:: 3.13 + + The *globals* and *locals* arguments can now be passed as keywords. + .. function:: filter(function, iterable) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9a0bf524e3943fd..230789f29ff788d 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -46,6 +46,8 @@ x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) +# used as proof of globals being used +A_GLOBAL_VALUE = 123 class Squares: @@ -684,6 +686,11 @@ def __getitem__(self, key): raise ValueError self.assertRaises(ValueError, eval, "foo", {}, X()) + def test_eval_kwargs(self): + data = {"A_GLOBAL_VALUE": 456} + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", globals=data), 456) + self.assertEqual(eval("globals()['A_GLOBAL_VALUE']", locals=data), 123) + def test_general_eval(self): # Tests that general mappings can be used for the locals argument @@ -777,6 +784,19 @@ def test_exec(self): del l['__builtins__'] self.assertEqual((g, l), ({'a': 1}, {'b': 2})) + def test_exec_kwargs(self): + g = {} + exec('global z\nz = 1', globals=g) + if '__builtins__' in g: + del g['__builtins__'] + self.assertEqual(g, {'z': 1}) + + # if we only set locals, the global assignment will not + # reach this locals dictionary + g = {} + exec('global z\nz = 1', locals=g) + self.assertEqual(g, {}) + def test_exec_globals(self): code = compile("print('Hello World!')", "", "exec") # no builtin function diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-18-00-27-57.gh-issue-105879.dPw78k.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-18-00-27-57.gh-issue-105879.dPw78k.rst new file mode 100644 index 000000000000000..e666688d09cb5cb --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-06-18-00-27-57.gh-issue-105879.dPw78k.rst @@ -0,0 +1,2 @@ +Allow the *globals* and *locals* arguments to :func:`exec` +and :func:`eval` to be passed as keywords. diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 7af3ac9c5158d6e..722353ebcbfc3d8 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -925,9 +925,9 @@ builtin_divmod_impl(PyObject *module, PyObject *x, PyObject *y) eval as builtin_eval source: object + / globals: object = None locals: object = None - / Evaluate the given source in the context of globals and locals. @@ -941,7 +941,7 @@ If only globals is given, locals defaults to it. static PyObject * builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) -/*[clinic end generated code: output=0a0824aa70093116 input=11ee718a8640e527]*/ +/*[clinic end generated code: output=0a0824aa70093116 input=7c7bce5299a89062]*/ { PyObject *result = NULL, *source_copy; const char *str; @@ -1024,9 +1024,9 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, exec as builtin_exec source: object + / globals: object = None locals: object = None - / * closure: object(c_default="NULL") = None @@ -1044,7 +1044,7 @@ when source is a code object requiring exactly that many cellvars. static PyObject * builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) -/*[clinic end generated code: output=7579eb4e7646743d input=f13a7e2b503d1d9a]*/ +/*[clinic end generated code: output=7579eb4e7646743d input=25e989b6d87a3a21]*/ { PyObject *v; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index 3f005bcbfb6a1a2..f75a8d4ac0ccd4b 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -395,7 +395,7 @@ builtin_divmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_eval__doc__, -"eval($module, source, globals=None, locals=None, /)\n" +"eval($module, source, /, globals=None, locals=None)\n" "--\n" "\n" "Evaluate the given source in the context of globals and locals.\n" @@ -407,33 +407,63 @@ PyDoc_STRVAR(builtin_eval__doc__, "If only globals is given, locals defaults to it."); #define BUILTIN_EVAL_METHODDEF \ - {"eval", _PyCFunction_CAST(builtin_eval), METH_FASTCALL, builtin_eval__doc__}, + {"eval", _PyCFunction_CAST(builtin_eval), METH_FASTCALL|METH_KEYWORDS, builtin_eval__doc__}, static PyObject * builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals); static PyObject * -builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(globals), &_Py_ID(locals), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "globals", "locals", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "eval", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *source; PyObject *globals = Py_None; PyObject *locals = Py_None; - if (!_PyArg_CheckPositional("eval", nargs, 1, 3)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); + if (!args) { goto exit; } source = args[0]; - if (nargs < 2) { - goto skip_optional; + if (!noptargs) { + goto skip_optional_pos; } - globals = args[1]; - if (nargs < 3) { - goto skip_optional; + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } } locals = args[2]; -skip_optional: +skip_optional_pos: return_value = builtin_eval_impl(module, source, globals, locals); exit: @@ -441,7 +471,7 @@ builtin_eval(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } PyDoc_STRVAR(builtin_exec__doc__, -"exec($module, source, globals=None, locals=None, /, *, closure=None)\n" +"exec($module, source, /, globals=None, locals=None, *, closure=None)\n" "--\n" "\n" "Execute the given source in the context of globals and locals.\n" @@ -467,14 +497,14 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(closure), }, + .ob_item = { &_Py_ID(globals), &_Py_ID(locals), &_Py_ID(closure), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -483,7 +513,7 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "", "", "closure", NULL}; + static const char * const _keywords[] = {"", "globals", "locals", "closure", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "exec", @@ -502,17 +532,22 @@ builtin_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto exit; } source = args[0]; - if (nargs < 2) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - globals = args[1]; - if (nargs < 3) { - goto skip_optional_posonly; + if (args[1]) { + globals = args[1]; + if (!--noptargs) { + goto skip_optional_pos; + } } - noptargs--; - locals = args[2]; -skip_optional_posonly: + if (args[2]) { + locals = args[2]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -1193,4 +1228,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=6d15edfc194b2c08 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=435d3f286a863c49 input=a9049054013a1b77]*/ From 6bcbee09df9f6fa6496fb7f68b6d655f190903a7 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Thu, 2 May 2024 19:30:00 +0200 Subject: [PATCH 068/220] gh-93502: Add new C-API functions to trace object creation and destruction (#115945) --- Doc/c-api/init.rst | 52 ++++++++++++ Doc/whatsnew/3.13.rst | 5 ++ Include/cpython/object.h | 10 +++ Include/internal/pycore_object.h | 2 +- Include/internal/pycore_runtime.h | 7 ++ Include/internal/pycore_runtime_init.h | 4 + ...4-02-26-13-14-52.gh-issue-93502.JMWRvA.rst | 4 + Modules/_testcapimodule.c | 84 +++++++++++++++++++ Objects/object.c | 37 ++++++-- Python/tracemalloc.c | 10 ++- 10 files changed, 207 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 8725ce085145aad..9e118d4f36145f8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1904,6 +1904,58 @@ Python-level trace functions in previous versions. .. versionadded:: 3.12 +Reference tracing +================= + +.. versionadded:: 3.13 + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + +.. versionadded:: 3.13 + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Not that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. The GIL will be held every time + the tracer function is called. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + The GIL must be held when calling this function. + +.. versionadded:: 3.13 .. _advanced-debugging: diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 3ccf17be9796d50..fbf2f4c447468db 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1961,6 +1961,11 @@ New Features * Add :c:func:`PyType_GetModuleByDef` to the limited C API (Contributed by Victor Stinner in :gh:`116936`.) +* Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and + :c:func:`PyRefTracer_GetTracer`, that allows to track object creation and + destruction the same way the :mod:`tracemalloc` module does. (Contributed + by Pablo Galindo in :gh:`93502`.) + Porting to Python 3.13 ---------------------- diff --git a/Include/cpython/object.h b/Include/cpython/object.h index a6b93b93ab0f7a7..c2830b75e66fbe5 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -510,3 +510,13 @@ PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); * assigned, or 0 if a new tag could not be assigned. */ PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); + + +typedef enum { + PyRefTracer_CREATE = 0, + PyRefTracer_DESTROY = 1, +} PyRefTracerEvent; + +typedef int (*PyRefTracer)(PyObject *, PyRefTracerEvent event, void *); +PyAPI_FUNC(int) PyRefTracer_SetTracer(PyRefTracer tracer, void *data); +PyAPI_FUNC(PyRefTracer) PyRefTracer_GetTracer(void**); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 7df8003196d8cc0..3b0222b05cbd70d 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -257,7 +257,7 @@ extern int _PyDict_CheckConsistency(PyObject *mp, int check_content); when a memory block is reused from a free list. Internal function called by _Py_NewReference(). */ -extern int _PyTraceMalloc_NewReference(PyObject *op); +extern int _PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void*); // Fast inlined version of PyType_HasFeature() static inline int diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index dc6f6f100f7a922..f58eccf729cb2a1 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -132,6 +132,12 @@ typedef struct _Py_DebugOffsets { } unicode_object; } _Py_DebugOffsets; +/* Reference tracer state */ +struct _reftracer_runtime_state { + PyRefTracer tracer_func; + void* tracer_data; +}; + /* Full Python runtime state */ /* _PyRuntimeState holds the global state for the CPython runtime. @@ -236,6 +242,7 @@ typedef struct pyruntimestate { struct _fileutils_state fileutils; struct _faulthandler_runtime_state faulthandler; struct _tracemalloc_runtime_state tracemalloc; + struct _reftracer_runtime_state ref_tracer; // The rwmutex is used to prevent overlapping global and per-interpreter // stop-the-world events. Global stop-the-world events lock the mutex diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 41331df8320a9c9..98920dbb7c7a921 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -128,6 +128,10 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .faulthandler = _faulthandler_runtime_state_INIT, \ .tracemalloc = _tracemalloc_runtime_state_INIT, \ + .ref_tracer = { \ + .tracer_func = NULL, \ + .tracer_data = NULL, \ + }, \ .stoptheworld = { \ .is_global = 1, \ }, \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst new file mode 100644 index 000000000000000..524626950c02e65 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-26-13-14-52.gh-issue-93502.JMWRvA.rst @@ -0,0 +1,4 @@ +Add two new functions to the C-API, :c:func:`PyRefTracer_SetTracer` and +:c:func:`PyRefTracer_GetTracer`, that allows to track object creation and +destruction the same way the :mod:`tracemalloc` module does. Patch by Pablo +Galindo diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3448291e401e357..f5892fc5ed2a2c1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3219,6 +3219,89 @@ test_weakref_capi(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) _Py_COMP_DIAG_POP } +struct simpletracer_data { + int create_count; + int destroy_count; + void* addresses[10]; +}; + +static int _simpletracer(PyObject *obj, PyRefTracerEvent event, void* data) { + struct simpletracer_data* the_data = (struct simpletracer_data*)data; + assert(the_data->create_count + the_data->destroy_count < (int)Py_ARRAY_LENGTH(the_data->addresses)); + the_data->addresses[the_data->create_count + the_data->destroy_count] = obj; + if (event == PyRefTracer_CREATE) { + the_data->create_count++; + } else { + the_data->destroy_count++; + } + return 0; +} + +static PyObject * +test_reftracer(PyObject *ob, PyObject *Py_UNUSED(ignored)) +{ + // Save the current tracer and data to restore it later + void* current_data; + PyRefTracer current_tracer = PyRefTracer_GetTracer(¤t_data); + + struct simpletracer_data tracer_data = {0}; + void* the_data = &tracer_data; + // Install a simple tracer function + if (PyRefTracer_SetTracer(_simpletracer, the_data) != 0) { + goto failed; + } + + // Check that the tracer was correctly installed + void* data; + if (PyRefTracer_GetTracer(&data) != _simpletracer || data != the_data) { + PyErr_SetString(PyExc_AssertionError, "The reftracer not correctly installed"); + (void)PyRefTracer_SetTracer(NULL, NULL); + goto failed; + } + + // Create a bunch of objects + PyObject* obj = PyList_New(0); + if (obj == NULL) { + goto failed; + } + PyObject* obj2 = PyDict_New(); + if (obj2 == NULL) { + Py_DECREF(obj); + goto failed; + } + + // Kill all objects + Py_DECREF(obj); + Py_DECREF(obj2); + + // Remove the tracer + (void)PyRefTracer_SetTracer(NULL, NULL); + + // Check that the tracer was removed + if (PyRefTracer_GetTracer(&data) != NULL || data != NULL) { + PyErr_SetString(PyExc_ValueError, "The reftracer was not correctly removed"); + goto failed; + } + + if (tracer_data.create_count != 2 || + tracer_data.addresses[0] != obj || + tracer_data.addresses[1] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object creation was not correctly traced"); + goto failed; + } + + if (tracer_data.destroy_count != 2 || + tracer_data.addresses[2] != obj || + tracer_data.addresses[3] != obj2) { + PyErr_SetString(PyExc_ValueError, "The object destruction was not correctly traced"); + goto failed; + } + PyRefTracer_SetTracer(current_tracer, current_data); + Py_RETURN_NONE; +failed: + PyRefTracer_SetTracer(current_tracer, current_data); + return NULL; +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3257,6 +3340,7 @@ static PyMethodDef TestMethods[] = { {"get_type_fullyqualname", get_type_fullyqualname, METH_O}, {"get_type_module_name", get_type_module_name, METH_O}, {"test_get_type_dict", test_get_type_dict, METH_NOARGS}, + {"test_reftracer", test_reftracer, METH_NOARGS}, {"_test_thread_state", test_thread_state, METH_VARARGS}, #ifndef MS_WINDOWS {"_spawn_pthread_waiter", spawn_pthread_waiter, METH_NOARGS}, diff --git a/Objects/object.c b/Objects/object.c index 45310a6c22d6774..79e4fb4dbbf7c69 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2372,9 +2372,6 @@ _PyTypes_FiniTypes(PyInterpreterState *interp) static inline void new_reference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) op->ob_refcnt = 1; @@ -2389,6 +2386,11 @@ new_reference(PyObject *op) #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_CREATE, data); + } } void @@ -2450,12 +2452,13 @@ _PyObject_SetDeferredRefcount(PyObject *op) void _Py_ResurrectReference(PyObject *op) { - if (_PyRuntime.tracemalloc.config.tracing) { - _PyTraceMalloc_NewReference(op); - } #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op); #endif + if (_PyRuntime.ref_tracer.tracer_func != NULL) { + void* data = _PyRuntime.ref_tracer.tracer_data; + _PyRuntime.ref_tracer.tracer_func(op, PyRefTracer_CREATE, data); + } } @@ -2845,6 +2848,12 @@ _Py_Dealloc(PyObject *op) Py_INCREF(type); #endif + struct _reftracer_runtime_state *tracer = &_PyRuntime.ref_tracer; + if (tracer->tracer_func != NULL) { + void* data = tracer->tracer_data; + tracer->tracer_func(op, PyRefTracer_DESTROY, data); + } + #ifdef Py_TRACE_REFS _Py_ForgetReference(op); #endif @@ -2933,6 +2942,22 @@ _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt) Py_SET_REFCNT(ob, refcnt); } +int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) { + assert(PyGILState_Check()); + _PyRuntime.ref_tracer.tracer_func = tracer; + _PyRuntime.ref_tracer.tracer_data = data; + return 0; +} + +PyRefTracer PyRefTracer_GetTracer(void** data) { + assert(PyGILState_Check()); + if (data != NULL) { + *data = _PyRuntime.ref_tracer.tracer_data; + } + return _PyRuntime.ref_tracer.tracer_func; +} + + static PyObject* constants[] = { &_Py_NoneStruct, // Py_CONSTANT_NONE diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 19b64c619feb6aa..e3ec72062f69313 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -906,6 +906,10 @@ _PyTraceMalloc_Start(int max_nframe) return -1; } + if (PyRefTracer_SetTracer(_PyTraceMalloc_TraceRef, NULL) < 0) { + return -1; + } + if (tracemalloc_config.tracing) { /* hook already installed: do nothing */ return 0; @@ -1352,8 +1356,12 @@ _PyTraceMalloc_Fini(void) Do nothing if tracemalloc is not tracing memory allocations or if the object memory block is not already traced. */ int -_PyTraceMalloc_NewReference(PyObject *op) +_PyTraceMalloc_TraceRef(PyObject *op, PyRefTracerEvent event, void* Py_UNUSED(ignore)) { + if (event != PyRefTracer_CREATE) { + return 0; + } + assert(PyGILState_Check()); if (!tracemalloc_config.tracing) { From b28a3339e4c63ea3a801dba9bbbc6af5af42c3a0 Mon Sep 17 00:00:00 2001 From: infohash <46137868+infohash@users.noreply.github.com> Date: Thu, 2 May 2024 23:06:35 +0530 Subject: [PATCH 069/220] gh-90848: Fixed create_autospec ignoring configure_mock style kwargs (#118163) --- Lib/test/test_unittest/testmock/testmock.py | 13 ++++++++++++ Lib/unittest/mock.py | 20 +++++++++++-------- ...4-04-22-21-54-12.gh-issue-90848.5jHEEc.rst | 1 + 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-22-21-54-12.gh-issue-90848.5jHEEc.rst diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index b81b3049d56bf82..77f6f1eb4b76b95 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -115,6 +115,19 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) def test_repr(self): mock = Mock(name='foo') diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 1799e9bbf58592f..a2634b6164062a8 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -2788,8 +2788,8 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if _parent is not None and not instance: _parent._mock_children[_name] = mock - wrapped = kwargs.get('wraps') - + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, _name='()', _parent=mock, @@ -2814,12 +2814,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, except AttributeError: continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} # Wrap child attributes also. if wrapped and hasattr(wrapped, entry): - kwargs.update(wraps=original) + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2830,14 +2830,13 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) @@ -2848,6 +2847,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock diff --git a/Misc/NEWS.d/next/Library/2024-04-22-21-54-12.gh-issue-90848.5jHEEc.rst b/Misc/NEWS.d/next/Library/2024-04-22-21-54-12.gh-issue-90848.5jHEEc.rst new file mode 100644 index 000000000000000..adbca0127ed0ceb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-22-21-54-12.gh-issue-90848.5jHEEc.rst @@ -0,0 +1 @@ +Fixed :func:`unittest.mock.create_autospec` to configure parent mock with keyword arguments. From 83c51da6cebdced80ebc59de70e8844244de7298 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 2 May 2024 13:41:15 -0400 Subject: [PATCH 070/220] gh-118413: Fix test_release_task_refs on free-threaded build (#118494) The `time.sleep()` call should happen before the GC to give the worker threads time to clean-up their remaining references to objs. Additionally, use `support.gc_collect()` instead of `gc.collect()` just in case the extra GC calls matter. --- Lib/test/_test_multiprocessing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 91f85990116ee7c..5fc4181a1eeadb6 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -2804,7 +2804,6 @@ def test_map_no_failfast(self): # check that we indeed waited for all jobs self.assertGreater(time.monotonic() - t_start, 0.9) - @support.requires_gil_enabled("gh-118413: test is flaky with GIL disabled") def test_release_task_refs(self): # Issue #29861: task arguments and results should not be kept # alive after we are done with them. @@ -2813,8 +2812,8 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs - gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run + support.gc_collect() # For PyPy or other GCs. self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. From 16acecd77924b849f51b36f41cd21ca759c9ff2a Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 2 May 2024 13:51:28 -0400 Subject: [PATCH 071/220] Remove stray `__cplusplus` guard in sysmodule.c (#118511) --- Python/sysmodule.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 726051521cf574e..d3fbfcd3e79636b 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2496,10 +2496,6 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { return 0; } -#ifdef __cplusplus -} -#endif - static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ From 8ed546679524140d8282175411fd141fe7df070d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Thu, 2 May 2024 19:43:54 +0100 Subject: [PATCH 072/220] gh-118486: Switch mkdir(mode=0o700) on Windows to use OWNER RIGHTS instead of CURRENT_USER (GH-118515) --- Modules/posixmodule.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f9533577a8fa340..e1a14e772c4bd03 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5587,6 +5587,7 @@ struct _Py_SECURITY_ATTRIBUTE_DATA { PACL acl; SECURITY_DESCRIPTOR sd; EXPLICIT_ACCESS_W ea[4]; + char sid[64]; }; static int @@ -5616,13 +5617,25 @@ initializeMkdir700SecurityAttributes( return GetLastError(); } + int use_alias = 0; + DWORD cbSid = sizeof(data->sid); + if (!CreateWellKnownSid(WinCreatorOwnerRightsSid, NULL, (PSID)data->sid, &cbSid)) { + use_alias = 1; + } + data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); data->ea[0].grfAccessPermissions = GENERIC_ALL; data->ea[0].grfAccessMode = SET_ACCESS; data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; + if (use_alias) { + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; + } else { + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; + data->ea[0].Trustee.ptstrName = (LPWCH)(SID*)data->sid; + } data->ea[1].grfAccessPermissions = GENERIC_ALL; data->ea[1].grfAccessMode = SET_ACCESS; From 1e67b9207c31a92d76bfac09fc706c4dc703669e Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 2 May 2024 13:03:05 -0700 Subject: [PATCH 073/220] gh-117657: Fix TSAN list set failure (#118260) * Fix TSAN list set failure * Relaxed atomic is sufficient, add targetted test * More list * Remove atomic assign in list * Fixup white space --- Include/internal/pycore_list.h | 4 ++ Lib/test/test_free_threading/test_list.py | 80 +++++++++++++++++++++++ Objects/listobject.c | 9 ++- 3 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_free_threading/test_list.py diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 2a82912e41d5571..73695d10e0c3721 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -28,7 +28,11 @@ _PyList_AppendTakeRef(PyListObject *self, PyObject *newitem) Py_ssize_t allocated = self->allocated; assert((size_t)len + 1 < PY_SSIZE_T_MAX); if (allocated > len) { +#ifdef Py_GIL_DISABLED + _Py_atomic_store_ptr_release(&self->ob_item[len], newitem); +#else PyList_SET_ITEM(self, len, newitem); +#endif Py_SET_SIZE(self, len + 1); return 0; } diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py new file mode 100644 index 000000000000000..79cb0a930923657 --- /dev/null +++ b/Lib/test/test_free_threading/test_list.py @@ -0,0 +1,80 @@ +import unittest + +from threading import Thread +from unittest import TestCase + +from test.support import is_wasi + + +class C: + def __init__(self, v): + self.v = v + + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class TestList(TestCase): + def test_racing_iter_append(self): + + l = [] + OBJECT_COUNT = 10000 + + def writer_func(): + for i in range(OBJECT_COUNT): + l.append(C(i + OBJECT_COUNT)) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + def test_racing_iter_extend(self): + iters = [ + lambda x: [x], + ] + for iter_case in iters: + with self.subTest(iter=iter_case): + l = [] + OBJECT_COUNT = 10000 + + def writer_func(): + for i in range(OBJECT_COUNT): + l.extend(iter_case(C(i + OBJECT_COUNT))) + + def reader_func(): + while True: + count = len(l) + for i, x in enumerate(l): + self.assertEqual(x.v, i + OBJECT_COUNT) + if count == OBJECT_COUNT: + break + + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/Objects/listobject.c b/Objects/listobject.c index 4eaf20033fa2627..3c4e2d2e6ed7ded 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -141,6 +141,9 @@ list_resize(PyListObject *self, Py_ssize_t newsize) target_bytes = allocated * sizeof(PyObject*); } memcpy(array->ob_item, self->ob_item, target_bytes); + } + if (new_allocated > (size_t)allocated) { + memset(array->ob_item + allocated, 0, sizeof(PyObject *) * (new_allocated - allocated)); } _Py_atomic_store_ptr_release(&self->ob_item, &array->ob_item); self->allocated = new_allocated; @@ -502,7 +505,7 @@ _PyList_AppendTakeRefListResize(PyListObject *self, PyObject *newitem) Py_DECREF(newitem); return -1; } - PyList_SET_ITEM(self, len, newitem); + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], newitem); return 0; } @@ -1181,7 +1184,7 @@ list_extend_fast(PyListObject *self, PyObject *iterable) PyObject **dest = self->ob_item + m; for (Py_ssize_t i = 0; i < n; i++) { PyObject *o = src[i]; - dest[i] = Py_NewRef(o); + FT_ATOMIC_STORE_PTR_RELEASE(dest[i], Py_NewRef(o)); } return 0; } @@ -1238,7 +1241,7 @@ list_extend_iter_lock_held(PyListObject *self, PyObject *iterable) if (Py_SIZE(self) < self->allocated) { Py_ssize_t len = Py_SIZE(self); - PyList_SET_ITEM(self, len, item); // steals item ref + FT_ATOMIC_STORE_PTR_RELEASE(self->ob_item[len], item); // steals item ref Py_SET_SIZE(self, len + 1); } else { From e54b0c8a4ad6c6e958245eb3ea4ecc47e0f97ff0 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Thu, 2 May 2024 13:03:29 -0700 Subject: [PATCH 074/220] gh-118519: Fix empty weakref list check (#118520) Fix empty list check --- Objects/weakrefobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index 93c5fe3aacecfd2..88afaec86827ed4 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -988,7 +988,7 @@ PyObject_ClearWeakRefs(PyObject *object) } list = GET_WEAKREFS_LISTPTR(object); - if (FT_ATOMIC_LOAD_PTR(list) == NULL) { + if (FT_ATOMIC_LOAD_PTR(*list) == NULL) { // Fast path for the common case return; } From 4e2caf2aa046bf80e87e9b858837bb527459a034 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Thu, 2 May 2024 13:53:27 -0700 Subject: [PATCH 075/220] gh-118500: Add pdb support for zipapp (#118501) --- Doc/whatsnew/3.13.rst | 3 ++ Lib/pdb.py | 50 +++++++++++++++++-- Lib/test/test_pdb.py | 25 ++++++++++ Lib/test/test_pyclbr.py | 2 +- ...-05-02-04-27-12.gh-issue-118500.pBGGtQ.rst | 1 + 5 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-02-04-27-12.gh-issue-118500.pBGGtQ.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index fbf2f4c447468db..d59c4ee22e67d3e 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -705,6 +705,9 @@ pdb command line option or :envvar:`PYTHONSAFEPATH` environment variable). (Contributed by Tian Gao and Christian Walther in :gh:`111762`.) +* :mod:`zipapp` is supported as a debugging target. + (Contributed by Tian Gao in :gh:`118501`.) + queue ----- diff --git a/Lib/pdb.py b/Lib/pdb.py index fa2e1ec94be2358..bb669a0d2c1ce56 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -120,7 +120,10 @@ def find_function(funcname, filename): try: fp = tokenize.open(filename) except OSError: - return None + lines = linecache.getlines(filename) + if not lines: + return None + fp = io.StringIO(''.join(lines)) funcdef = "" funcstart = None # consumer of this info expects the first line to be 1 @@ -237,6 +240,44 @@ def namespace(self): ) +class _ZipTarget(_ExecutableTarget): + def __init__(self, target): + import runpy + + self._target = os.path.realpath(target) + sys.path.insert(0, self._target) + try: + _, self._spec, self._code = runpy._get_main_module_details() + except ImportError as e: + print(f"ImportError: {e}") + sys.exit(1) + except Exception: + traceback.print_exc() + sys.exit(1) + + def __repr__(self): + return self._target + + @property + def filename(self): + return self._code.co_filename + + @property + def code(self): + return self._code + + @property + def namespace(self): + return dict( + __name__='__main__', + __file__=os.path.normcase(os.path.abspath(self.filename)), + __package__=self._spec.parent, + __loader__=self._spec.loader, + __spec__=self._spec, + __builtins__=__builtins__, + ) + + class _PdbInteractiveConsole(code.InteractiveConsole): def __init__(self, ns, message): self._message = message @@ -1076,7 +1117,7 @@ def lineinfo(self, identifier): if f: fname = f item = parts[1] - answer = find_function(item, fname) + answer = find_function(item, self.canonic(fname)) return answer or failed def checkline(self, filename, lineno): @@ -2282,7 +2323,10 @@ def main(): if not opts.args: parser.error("no module or script to run") file = opts.args.pop(0) - target = _ScriptTarget(file) + if file.endswith('.pyz'): + target = _ZipTarget(file) + else: + target = _ScriptTarget(file) sys.argv[:] = [file] + opts.args # Hide "pdb.py" and pdb options from argument list diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 5635d17c99d2a32..a3d2dda43b086d9 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -10,6 +10,7 @@ import subprocess import textwrap import linecache +import zipapp from contextlib import ExitStack, redirect_stdout from io import StringIO @@ -3532,6 +3533,30 @@ def test_non_utf8_encoding(self): if filename.endswith(".py"): self._run_pdb([os.path.join(script_dir, filename)], 'q') + def test_zipapp(self): + with os_helper.temp_dir() as temp_dir: + os.mkdir(os.path.join(temp_dir, 'source')) + script = textwrap.dedent( + """ + def f(x): + return x + 1 + f(21 + 21) + """ + ) + with open(os.path.join(temp_dir, 'source', '__main__.py'), 'w') as f: + f.write(script) + zipapp.create_archive(os.path.join(temp_dir, 'source'), + os.path.join(temp_dir, 'zipapp.pyz')) + stdout, _ = self._run_pdb([os.path.join(temp_dir, 'zipapp.pyz')], '\n'.join([ + 'b f', + 'c', + 'p x', + 'q' + ])) + self.assertIn('42', stdout) + self.assertIn('return x + 1', stdout) + + class ChecklineTests(unittest.TestCase): def setUp(self): linecache.clearcache() # Pdb.checkline() uses linecache.getline() diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index c7c5419ffe3e379..46206accbafc366 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -226,7 +226,7 @@ def test_others(self): cm( 'pdb', # pyclbr does not handle elegantly `typing` or properties - ignore=('Union', '_ModuleTarget', '_ScriptTarget'), + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Misc/NEWS.d/next/Library/2024-05-02-04-27-12.gh-issue-118500.pBGGtQ.rst b/Misc/NEWS.d/next/Library/2024-05-02-04-27-12.gh-issue-118500.pBGGtQ.rst new file mode 100644 index 000000000000000..62c7b5f8f24f268 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-02-04-27-12.gh-issue-118500.pBGGtQ.rst @@ -0,0 +1 @@ +Add :mod:`pdb` support for zipapps From f8290df63f1fd970dfd6bbfdc9a86341a9f97d05 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 2 May 2024 15:25:36 -0700 Subject: [PATCH 076/220] gh-116738: Make `_codecs` module thread-safe (#117530) The module itself is a thin wrapper around calls to functions in `Python/codecs.c`, so that's where the meaningful changes happened: - Move codecs-related state that lives on `PyInterpreterState` to a struct declared in `pycore_codecs.h`. - In free-threaded builds, add a mutex to `codecs_state` to synchronize operations on `search_path`. Because `search_path_mutex` is used as a normal mutex and not a critical section, we must be extremely careful with operations called while holding it. - The codec registry is explicitly initialized as part of `_PyUnicode_InitEncodings` to simplify thread-safety. --- Include/internal/pycore_codecs.h | 31 ++++++ Include/internal/pycore_interp.h | 6 +- Objects/unicodeobject.c | 6 +- Python/codecs.c | 150 ++++++++++++++------------- Python/pystate.c | 4 +- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 6 files changed, 120 insertions(+), 79 deletions(-) diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h index a2a7151d50ade7c..5e2d5c5ce9d868a 100644 --- a/Include/internal/pycore_codecs.h +++ b/Include/internal/pycore_codecs.h @@ -8,6 +8,17 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + +/* Initialize codecs-related state for the given interpreter, including + registering the first codec search function. Must be called before any other + PyCodec-related functions, and while only one thread is active. */ +extern PyStatus _PyCodec_InitRegistry(PyInterpreterState *interp); + +/* Finalize codecs-related state for the given interpreter. No PyCodec-related + functions other than PyCodec_Unregister() may be called after this. */ +extern void _PyCodec_Fini(PyInterpreterState *interp); + extern PyObject* _PyCodec_Lookup(const char *encoding); /* Text codec specific encoding and decoding API. @@ -48,6 +59,26 @@ extern PyObject* _PyCodecInfo_GetIncrementalEncoder( PyObject *codec_info, const char *errors); +// Per-interpreter state used by codecs.c. +struct codecs_state { + // A list of callable objects used to search for codecs. + PyObject *search_path; + + // A dict mapping codec names to codecs returned from a callable in + // search_path. + PyObject *search_cache; + + // A dict mapping error handling strategies to functions to implement them. + PyObject *error_registry; + +#ifdef Py_GIL_DISABLED + // Used to safely delete a specific item from search_path. + PyMutex search_path_mutex; +#endif + + // Whether or not the rest of the state is initialized. + int initialized; +}; #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d38959e3ce4ec52..a2aec6dd4b72411 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -14,6 +14,7 @@ extern "C" { #include "pycore_atexit.h" // struct atexit_state #include "pycore_ceval_state.h" // struct _ceval_state #include "pycore_code.h" // struct callable_cache +#include "pycore_codecs.h" // struct codecs_state #include "pycore_context.h" // struct _Py_context_state #include "pycore_crossinterp.h" // struct _xidregistry #include "pycore_dict_state.h" // struct _Py_dict_state @@ -182,10 +183,7 @@ struct _is { possible to facilitate out-of-process observability tools. */ - PyObject *codec_search_path; - PyObject *codec_search_cache; - PyObject *codec_error_registry; - int codecs_initialized; + struct codecs_state codecs; PyConfig config; unsigned long feature_flags; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2c259b7e869efeb..67b1282de790e09 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15441,7 +15441,11 @@ init_fs_encoding(PyThreadState *tstate) PyStatus _PyUnicode_InitEncodings(PyThreadState *tstate) { - PyStatus status = init_fs_encoding(tstate); + PyStatus status = _PyCodec_InitRegistry(tstate->interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = init_fs_encoding(tstate); if (_PyStatus_EXCEPTION(status)) { return status; } diff --git a/Python/codecs.c b/Python/codecs.c index d8fe7b22063a809..bed245366f92342 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -11,6 +11,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_interp.h" // PyInterpreterState.codec_search_path +#include "pycore_lock.h" // PyMutex #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI @@ -19,24 +20,10 @@ const char *Py_hexdigits = "0123456789abcdef"; /* --- Codec Registry ----------------------------------------------------- */ -/* Import the standard encodings package which will register the first - codec search function. - - This is done in a lazy way so that the Unicode implementation does - not downgrade startup time of scripts not needing it. - - ImportErrors are silently ignored by this function. Only one try is - made. - -*/ - -static int _PyCodecRegistry_Init(void); /* Forward */ - int PyCodec_Register(PyObject *search_function) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - goto onError; + assert(interp->codecs.initialized); if (search_function == NULL) { PyErr_BadArgument(); goto onError; @@ -45,7 +32,14 @@ int PyCodec_Register(PyObject *search_function) PyErr_SetString(PyExc_TypeError, "argument must be callable"); goto onError; } - return PyList_Append(interp->codec_search_path, search_function); +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->codecs.search_path_mutex); +#endif + int ret = PyList_Append(interp->codecs.search_path, search_function); +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->codecs.search_path_mutex); +#endif + return ret; onError: return -1; @@ -55,22 +49,34 @@ int PyCodec_Unregister(PyObject *search_function) { PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *codec_search_path = interp->codec_search_path; - /* Do nothing if codec_search_path is not created yet or was cleared. */ - if (codec_search_path == NULL) { + if (interp->codecs.initialized != 1) { + /* Do nothing if codecs state was cleared (only possible during + interpreter shutdown). */ return 0; } + PyObject *codec_search_path = interp->codecs.search_path; assert(PyList_CheckExact(codec_search_path)); - Py_ssize_t n = PyList_GET_SIZE(codec_search_path); - for (Py_ssize_t i = 0; i < n; i++) { - PyObject *item = PyList_GET_ITEM(codec_search_path, i); + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(codec_search_path); i++) { +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->codecs.search_path_mutex); +#endif + PyObject *item = PyList_GetItemRef(codec_search_path, i); + int ret = 1; if (item == search_function) { - if (interp->codec_search_cache != NULL) { - assert(PyDict_CheckExact(interp->codec_search_cache)); - PyDict_Clear(interp->codec_search_cache); - } - return PyList_SetSlice(codec_search_path, i, i+1, NULL); + // We hold a reference to the item, so its destructor can't run + // while we hold search_path_mutex. + ret = PyList_SetSlice(codec_search_path, i, i+1, NULL); + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->codecs.search_path_mutex); +#endif + Py_DECREF(item); + if (ret != 1) { + assert(interp->codecs.search_cache != NULL); + assert(PyDict_CheckExact(interp->codecs.search_cache)); + PyDict_Clear(interp->codecs.search_cache); + return ret; } } return 0; @@ -132,9 +138,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) { - return NULL; - } + assert(interp->codecs.initialized); /* Convert the encoding to a normalized Python string: all characters are converted to lower case, spaces and hyphens are @@ -147,7 +151,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) /* First, try to lookup the name in the registry dictionary */ PyObject *result; - if (PyDict_GetItemRef(interp->codec_search_cache, v, &result) < 0) { + if (PyDict_GetItemRef(interp->codecs.search_cache, v, &result) < 0) { goto onError; } if (result != NULL) { @@ -156,7 +160,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } /* Next, scan the search functions in order of registration */ - const Py_ssize_t len = PyList_Size(interp->codec_search_path); + const Py_ssize_t len = PyList_Size(interp->codecs.search_path); if (len < 0) goto onError; if (len == 0) { @@ -170,14 +174,15 @@ PyObject *_PyCodec_Lookup(const char *encoding) for (i = 0; i < len; i++) { PyObject *func; - func = PyList_GetItem(interp->codec_search_path, i); + func = PyList_GetItemRef(interp->codecs.search_path, i); if (func == NULL) goto onError; result = PyObject_CallOneArg(func, v); + Py_DECREF(func); if (result == NULL) goto onError; if (result == Py_None) { - Py_DECREF(result); + Py_CLEAR(result); continue; } if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 4) { @@ -188,7 +193,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } break; } - if (i == len) { + if (result == NULL) { /* XXX Perhaps we should cache misses too ? */ PyErr_Format(PyExc_LookupError, "unknown encoding: %s", encoding); @@ -196,7 +201,7 @@ PyObject *_PyCodec_Lookup(const char *encoding) } /* Cache and return the result */ - if (PyDict_SetItem(interp->codec_search_cache, v, result) < 0) { + if (PyDict_SetItem(interp->codecs.search_cache, v, result) < 0) { Py_DECREF(result); goto onError; } @@ -600,13 +605,12 @@ PyObject *_PyCodec_DecodeText(PyObject *object, int PyCodec_RegisterError(const char *name, PyObject *error) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - return -1; + assert(interp->codecs.initialized); if (!PyCallable_Check(error)) { PyErr_SetString(PyExc_TypeError, "handler must be callable"); return -1; } - return PyDict_SetItemString(interp->codec_error_registry, + return PyDict_SetItemString(interp->codecs.error_registry, name, error); } @@ -616,13 +620,12 @@ int PyCodec_RegisterError(const char *name, PyObject *error) PyObject *PyCodec_LookupError(const char *name) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (interp->codec_search_path == NULL && _PyCodecRegistry_Init()) - return NULL; + assert(interp->codecs.initialized); if (name==NULL) name = "strict"; PyObject *handler; - if (PyDict_GetItemStringRef(interp->codec_error_registry, name, &handler) < 0) { + if (PyDict_GetItemStringRef(interp->codecs.error_registry, name, &handler) < 0) { return NULL; } if (handler == NULL) { @@ -1375,7 +1378,8 @@ static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc) return PyCodec_SurrogateEscapeErrors(exc); } -static int _PyCodecRegistry_Init(void) +PyStatus +_PyCodec_InitRegistry(PyInterpreterState *interp) { static struct { const char *name; @@ -1463,45 +1467,51 @@ static int _PyCodecRegistry_Init(void) } }; - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyObject *mod; - - if (interp->codec_search_path != NULL) - return 0; - - interp->codec_search_path = PyList_New(0); - if (interp->codec_search_path == NULL) { - return -1; + assert(interp->codecs.initialized == 0); + interp->codecs.search_path = PyList_New(0); + if (interp->codecs.search_path == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_search_cache = PyDict_New(); - if (interp->codec_search_cache == NULL) { - return -1; + interp->codecs.search_cache = PyDict_New(); + if (interp->codecs.search_cache == NULL) { + return PyStatus_NoMemory(); } - - interp->codec_error_registry = PyDict_New(); - if (interp->codec_error_registry == NULL) { - return -1; + interp->codecs.error_registry = PyDict_New(); + if (interp->codecs.error_registry == NULL) { + return PyStatus_NoMemory(); } - for (size_t i = 0; i < Py_ARRAY_LENGTH(methods); ++i) { PyObject *func = PyCFunction_NewEx(&methods[i].def, NULL, NULL); - if (!func) { - return -1; + if (func == NULL) { + return PyStatus_NoMemory(); } - int res = PyCodec_RegisterError(methods[i].name, func); + int res = PyDict_SetItemString(interp->codecs.error_registry, + methods[i].name, func); Py_DECREF(func); - if (res) { - return -1; + if (res < 0) { + return PyStatus_Error("Failed to insert into codec error registry"); } } - mod = PyImport_ImportModule("encodings"); + interp->codecs.initialized = 1; + + // Importing `encodings' will call back into this module to register codec + // search functions, so this is done after everything else is initialized. + PyObject *mod = PyImport_ImportModule("encodings"); if (mod == NULL) { - return -1; + return PyStatus_Error("Failed to import encodings module"); } Py_DECREF(mod); - interp->codecs_initialized = 1; - return 0; + + return PyStatus_Ok(); +} + +void +_PyCodec_Fini(PyInterpreterState *interp) +{ + Py_CLEAR(interp->codecs.search_path); + Py_CLEAR(interp->codecs.search_cache); + Py_CLEAR(interp->codecs.error_registry); + interp->codecs.initialized = 0; } diff --git a/Python/pystate.c b/Python/pystate.c index 3d6e76e88bd7310..f442d87ba3150e2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -843,9 +843,7 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) } PyConfig_Clear(&interp->config); - Py_CLEAR(interp->codec_search_path); - Py_CLEAR(interp->codec_search_cache); - Py_CLEAR(interp->codec_error_registry); + _PyCodec_Fini(interp); assert(interp->imports.modules == NULL); assert(interp->imports.modules_by_index == NULL); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 87b695de23e25e2..c4dbdcfe80bb4a4 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -344,7 +344,7 @@ Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS - Python/codecs.c - Py_hexdigits - Python/codecs.c - ucnhash_capi - -Python/codecs.c _PyCodecRegistry_Init methods - +Python/codecs.c _PyCodec_InitRegistry methods - Python/compile.c - NO_LABEL - Python/compile.c - NO_LOCATION - Python/dynload_shlib.c - _PyImport_DynLoadFiletab - From f201628073f22a785a096eccb010e2f78601b60f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 2 May 2024 18:51:43 -0600 Subject: [PATCH 077/220] gh-117953: Other Cleanups in the Extensions Machinery (gh-118206) This change will make some later changes simpler. --- Lib/test/test_import/__init__.py | 116 +++++++++++++++- Modules/_testsinglephase.c | 168 ++++++++++++++++++++++- Python/import.c | 228 ++++++++++++++++++------------- 3 files changed, 410 insertions(+), 102 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index f88b51f916cc818..b2aae774a122056 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2285,6 +2285,107 @@ def test_disallowed_reimport(self): class TestSinglePhaseSnapshot(ModuleSnapshot): + """A representation of a single-phase init module for testing. + + Fields from ModuleSnapshot: + + * id - id(mod) + * module - mod or a SimpleNamespace with __file__ & __spec__ + * ns - a shallow copy of mod.__dict__ + * ns_id - id(mod.__dict__) + * cached - sys.modules[name] (or None if not there or not snapshotable) + * cached_id - id(sys.modules[name]) (or None if not there) + + Extra fields: + + * summed - the result of calling "mod.sum(1, 2)" + * lookedup - the result of calling "mod.look_up_self()" + * lookedup_id - the object ID of self.lookedup + * state_initialized - the result of calling "mod.state_initialized()" + * init_count - (optional) the result of calling "mod.initialized_count()" + + Overridden methods from ModuleSnapshot: + + * from_module() + * parse() + + Other methods from ModuleSnapshot: + + * build_script() + * from_subinterp() + + ---- + + There are 5 modules in Modules/_testsinglephase.c: + + * _testsinglephase + * has global state + * extra loads skip the init function, copy def.m_base.m_copy + * counts calls to init function + * _testsinglephase_basic_wrapper + * _testsinglephase by another name (and separate init function symbol) + * _testsinglephase_basic_copy + * same as _testsinglephase but with own def (and init func) + * _testsinglephase_with_reinit + * has no global or module state + * mod.state_initialized returns None + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + * _testsinglephase_with_state + * has module state + * an extra load in the main interpreter calls the cached init func + * an extra load in legacy subinterpreters does a full load + + (See Modules/_testsinglephase.c for more info.) + + For all those modules, the snapshot after the initial load (not in + the global extensions cache) would look like the following: + + * initial load + * id: ID of nww module object + * ns: exactly what the module init put there + * ns_id: ID of new module's __dict__ + * cached_id: same as self.id + * summed: 3 (never changes) + * lookedup_id: same as self.id + * state_initialized: a timestamp between the time of the load + and the time of the snapshot + * init_count: 1 (None for _testsinglephase_with_reinit) + + For the other scenarios it varies. + + For the _testsinglephase, _testsinglephase_basic_wrapper, and + _testsinglephase_basic_copy modules, the snapshot should look + like the following: + + * reloaded + * id: no change + * ns: matches what the module init function put there, + including the IDs of all contained objects, + plus any extra attributes added before the reload + * ns_id: no change + * cached_id: no change + * lookedup_id: no change + * state_initialized: no change + * init_count: no change + * already loaded + * (same as initial load except for ns and state_initialized) + * ns: matches the initial load, incl. IDs of contained objects + * state_initialized: no change from initial load + + For _testsinglephase_with_reinit: + + * reloaded: same as initial load (old module & ns is discarded) + * already loaded: same as initial load (old module & ns is discarded) + + For _testsinglephase_with_state: + + * reloaded + * (same as initial load (old module & ns is discarded), + except init_count) + * init_count: increase by 1 + * already loaded: same as reloaded + """ @classmethod def from_module(cls, mod): @@ -2901,17 +3002,18 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # * module's global state was initialized but cleared # Start with an interpreter that gets destroyed right away. - base = self.import_in_subinterp(postscript=''' - # Attrs set after loading are not in m_copy. - mod.spam = 'spam, spam, mash, spam, eggs, and spam' - ''') + base = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') self.check_common(base) self.check_fresh(base) # At this point: # * alive in 0 interpreters # * module def in _PyRuntime.imports.extensions - # * mod init func ran again + # * mod init func ran for the first time (since reset) # * m_copy is NULL (claered when the interpreter was destroyed) # * module's global state was initialized, not reset @@ -2923,7 +3025,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # At this point: # * alive in 1 interpreter (interp1) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran again + # * mod init func ran for the second time (since reset) # * m_copy was copied from interp1 (was NULL) # * module's global state was updated, not reset @@ -2935,7 +3037,7 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # At this point: # * alive in 2 interpreters (interp1, interp2) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran again + # * mod init func did not run again # * m_copy was copied from interp2 (was from interp1) # * module's global state was updated, not reset diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 092673a9ea43e11..ff533e44a82730c 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,6 +1,172 @@ /* Testing module for single-phase initialization of extension modules - */ + +This file contains 5 distinct modules, meaning each as its own name +and its own init function (PyInit_...). The default import system will +only find the one matching the filename: _testsinglephase. To load the +others you must do so manually. For example: + +```python +name = '_testsinglephase_base_wrapper' +filename = _testsinglephase.__file__ +loader = importlib.machinery.ExtensionFileLoader(name, filename) +spec = importlib.util.spec_from_file_location(name, filename, loader=loader) +mod = importlib._bootstrap._load(spec) +``` + +Here are the 5 modules: + +* _testsinglephase + * def: _testsinglephase_basic, + * m_name: "_testsinglephase" + * m_size: -1 + * state + * process-global + * initialized_count (default to -1; will never be 0) + * module (see module state below) + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. clear .module + 3. initialize .module: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions + * (3 common, see below) + * initialized_count() - return .module.initialized_count + * import system + * caches + * global extensions cache: yes + * def.m_base.m_copy: yes + * def.m_base.m_init: no + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + 1. get init function from shared object file + 2. run init function + 3. copy __dict__ into def.m_base.m_copy + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. get module from sys.modules + 3. update module with contents of def.m_base.m_copy + * already loaded in other interpreter (already in global cache) + * same as reload, but create new module and update *it* + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * same as main interpreter + * unload: never (all single-phase init modules) +* _testsinglephase_basic_wrapper + * identical to _testsinglephase except module name +* _testsinglephase_basic_copy + * def: static local variable in init function + * m_name: "_testsinglephase_basic_copy" + * m_size: -1 + * state: same as _testsinglephase + * init function: same as _testsinglephase + * functions: same as _testsinglephase + * import system: same as _testsinglephase +* _testsinglephase_with_reinit + * def: _testsinglephase_with_reinit, + * m_name: "_testsinglephase_with_reinit" + * m_size: 0 + * state + * process-global state: no + * module state: no + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 2. initialize temporary module state (local var): see module state below + 3. initialize module: set initial __dict__ + * functions: see common functions below + * import system + * caches + * global extensions cache: only if loaded in main interpreter + * def.m_base.m_copy: no + * def.m_base.m_init: only if loaded in the main interpreter + * per-interpreter cache: yes (all single-phase init modules) + * load in main interpreter + * initial (not already in global cache) + * (same as _testsinglephase except step 3) + 1. get init function from shared object file + 2. run init function + 3. set def.m_base.m_init to the init function + 4. set entry in global cache + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + 1. get def from global cache + 2. call def->m_base.m_init to get a new module object + 3. replace the existing module in sys.modules + * already loaded in other interpreter (already in global cache) + * same as reload (since will only be in cache for main interp) + * not in any sys.modules, still in global cache + * same as already loaded + * load in legacy (non-isolated) interpreter + * initial (not already in global cache) + * (same as main interpreter except skip steps 3 & 4 there) + 1. get init function from shared object file + 2. run init function + ... + 5. set entry in per-interpreter cache + 6. set entry in sys.modules + * reload (already in sys.modules) + * same as initial (load from scratch) + * already loaded in other interpreter (already in global cache) + * same as initial (load from scratch) + * not in any sys.modules, still in global cache + * same as initial (load from scratch) + * unload: never (all single-phase init modules) +* _testsinglephase_with_state + * def: _testsinglephase_with_state, + * m_name: "_testsinglephase_with_state" + * m_size: sizeof(module_state) + * state + * process-global: no + * module state: see module state below + * initial __dict__: see common initial __dict__ below + * init function + 1. create module + 3. initialize module state: see module state below + 4. initialize module: set initial __dict__ + 5. increment .initialized_count + * functions: see common functions below + * import system: same as _testsinglephase_basic_copy + +Module state: + +* fields + * initialized - when the module was first initialized + * *error + * *int_const + * *str_const +* initialization + 1. set state.initialized to the current time + 2. set state.error to a new exception class + 3. set state->int_const to int(1969) + 4. set state->str_const to "something different" + +Common initial __dict__: + +* error: state.error +* int_const: state.int_const +* str_const: state.str_const +* _module_initialized: state.initialized + +Common functions: + +* look_up_self() - return the module from the per-interpreter "by-index" cache +* sum() - return a + b +* state_initialized() - return state->initialized (or None if m_size == 0) + +See Python/import.c, especially the long comments, for more about +single-phase init modules. +*/ + #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif diff --git a/Python/import.c b/Python/import.c index 0c51ffc6285a9c6..f120a3841b495da 100644 --- a/Python/import.c +++ b/Python/import.c @@ -645,33 +645,33 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) K. PyModule_CreateInitialized() -> PyModule_SetDocString() L. PyModule_CreateInitialized(): set mod->md_def M. : initialize the module, etc. - N. _PyImport_RunModInitFunc(): set def->m_base.m_init - O. import_run_extension() + N. import_run_extension() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - P. import_run_extension(): set __file__ - Q. import_run_extension() -> update_global_state_for_extension() - R. update_global_state_for_extension(): + O. import_run_extension(): set __file__ + P. import_run_extension() -> update_global_state_for_extension() + Q. update_global_state_for_extension(): copy __dict__ into def->m_base.m_copy - S. update_global_state_for_extension(): + R. update_global_state_for_extension(): add it to _PyRuntime.imports.extensions - T. import_run_extension() -> finish_singlephase_extension() - U. finish_singlephase_extension(): + S. import_run_extension() -> finish_singlephase_extension() + T. finish_singlephase_extension(): add it to interp->imports.modules_by_index - V. finish_singlephase_extension(): add it to sys.modules + U. finish_singlephase_extension(): add it to sys.modules Step (Q) is skipped for core modules (sys/builtins). (6). subsequent times (found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. import_find_extension() - -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - C. import_find_extension() -> import_add_module() - D. if name in sys.modules: use that module - E. else: - 1. import_add_module() -> PyModule_NewObject() - 2. import_add_module(): set it on sys.modules - F. import_find_extension(): copy the "m_copy" dict into __dict__ - G. import_find_extension(): add to modules_by_index + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension() -> import_add_module() + E. if name in sys.modules: use that module + F. else: + 1. import_add_module() -> PyModule_NewObject() + 2. import_add_module(): set it on sys.modules + G. reload_singlephase_extension(): copy the "m_copy" dict into __dict__ + H. reload_singlephase_extension(): add to modules_by_index (10). (every time): A. noop @@ -681,21 +681,23 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): A-P. (same as for m_size == -1) - Q-S. (skipped) - T-V. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R. (skipped) + S-U. (same as for m_size == -1) (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): - A-R. (same as for m_size == -1) - S. (skipped) - T-V. (same as for m_size == -1) + A-P. (same as for m_size == -1) + Q. _PyImport_RunModInitFunc(): set def->m_base.m_init + R-U. (same as for m_size == -1) (6). subsequent times (found in _PyRuntime.imports.extensions): A. _imp_create_dynamic_impl() -> import_find_extension() - B. import_find_extension() - -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() - C. import_find_extension(): call def->m_base.m_init (see above) - D. import_find_extension(): add the module to sys.modules - E. import_find_extension(): add to modules_by_index + B. import_find_extension() -> reload_singlephase_extension() + C. reload_singlephase_extension() + -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + D. reload_singlephase_extension(): call def->m_base.m_init (see above) + E. reload_singlephase_extension(): add the module to sys.modules + F. reload_singlephase_extension(): add to modules_by_index (10). every time: A. noop @@ -984,84 +986,103 @@ hashtable_destroy_str(void *ptr) #define HTSEP ':' -static PyModuleDef * -_extensions_cache_get(PyObject *filename, PyObject *name) -{ - PyModuleDef *def = NULL; - void *key = NULL; - extensions_lock_acquire(); - +static int +_extensions_cache_init(void) +{ + _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; + EXTENSIONS.hashtable = _Py_hashtable_new_full( + hashtable_hash_str, + hashtable_compare_str, + hashtable_destroy_str, // key + /* There's no need to decref the def since it's immortal. */ + NULL, // value + &alloc + ); if (EXTENSIONS.hashtable == NULL) { - goto finally; + PyErr_NoMemory(); + return -1; } + return 0; +} - key = hashtable_key_from_2_strings(filename, name, HTSEP); +static _Py_hashtable_entry_t * +_extensions_cache_find_unlocked(PyObject *path, PyObject *name, + void **p_key) +{ + if (EXTENSIONS.hashtable == NULL) { + return NULL; + } + void *key = hashtable_key_from_2_strings(path, name, HTSEP); if (key == NULL) { - goto finally; + return NULL; + } + _Py_hashtable_entry_t *entry = + _Py_hashtable_get_entry(EXTENSIONS.hashtable, key); + if (p_key != NULL) { + *p_key = key; } - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + else { + hashtable_destroy_str(key); + } + return entry; +} + +static PyModuleDef * +_extensions_cache_get(PyObject *path, PyObject *name) +{ + PyModuleDef *def = NULL; + extensions_lock_acquire(); + + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { + /* It was never added. */ goto finally; } def = (PyModuleDef *)entry->value; finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } return def; } static int -_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) +_extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def) { int res = -1; + assert(def != NULL); extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { - _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; - EXTENSIONS.hashtable = _Py_hashtable_new_full( - hashtable_hash_str, - hashtable_compare_str, - hashtable_destroy_str, // key - /* There's no need to decref the def since it's immortal. */ - NULL, // value - &alloc - ); - if (EXTENSIONS.hashtable == NULL) { - PyErr_NoMemory(); + if (_extensions_cache_init() < 0) { goto finally; } } - void *key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { - goto finally; - } - int already_set = 0; - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + void *key = NULL; + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, &key); if (entry == NULL) { + /* It was never added. */ if (_Py_hashtable_set(EXTENSIONS.hashtable, key, def) < 0) { - PyMem_RawFree(key); PyErr_NoMemory(); goto finally; } + /* The hashtable owns the key now. */ + key = NULL; + } + else if (entry->value == NULL) { + /* It was previously deleted. */ + entry->value = def; } else { - if (entry->value == NULL) { - entry->value = def; - } - else { - /* We expect it to be static, so it must be the same pointer. */ - assert((PyModuleDef *)entry->value == def); - already_set = 1; - } - PyMem_RawFree(key); + /* We expect it to be static, so it must be the same pointer. */ + assert((PyModuleDef *)entry->value == def); + /* It was already added. */ + already_set = 1; } + if (!already_set) { /* We assume that all module defs are statically allocated and will never be freed. Otherwise, we would incref here. */ @@ -1071,13 +1092,15 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) finally: extensions_lock_release(); + if (key != NULL) { + hashtable_destroy_str(key); + } return res; } static void -_extensions_cache_delete(PyObject *filename, PyObject *name) +_extensions_cache_delete(PyObject *path, PyObject *name) { - void *key = NULL; extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { @@ -1085,13 +1108,8 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) goto finally; } - key = hashtable_key_from_2_strings(filename, name, HTSEP); - if (key == NULL) { - goto finally; - } - - _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry( - EXTENSIONS.hashtable, key); + _Py_hashtable_entry_t *entry = + _extensions_cache_find_unlocked(path, name, NULL); if (entry == NULL) { /* It was never added. */ goto finally; @@ -1109,9 +1127,6 @@ _extensions_cache_delete(PyObject *filename, PyObject *name) finally: extensions_lock_release(); - if (key != NULL) { - PyMem_RawFree(key); - } } static void @@ -1359,15 +1374,11 @@ finish_singlephase_extension(PyThreadState *tstate, static PyObject * -import_find_extension(PyThreadState *tstate, - struct _Py_ext_module_loader_info *info) +reload_singlephase_extension(PyThreadState *tstate, PyModuleDef *def, + struct _Py_ext_module_loader_info *info) { - /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(info->path, info->name); - if (def == NULL) { - return NULL; - } assert_singlephase(def); + PyObject *mod = NULL; /* It may have been successfully imported previously in an interpreter that allows legacy modules @@ -1378,9 +1389,7 @@ import_find_extension(PyThreadState *tstate, return NULL; } - PyObject *mod, *mdict; PyObject *modules = get_modules_dict(tstate, true); - if (def->m_size == -1) { PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ @@ -1390,6 +1399,7 @@ import_find_extension(PyThreadState *tstate, m_copy = get_core_module_dict( tstate->interp, info->name, info->path); if (m_copy == NULL) { + assert(!PyErr_Occurred()); return NULL; } } @@ -1397,7 +1407,7 @@ import_find_extension(PyThreadState *tstate, if (mod == NULL) { return NULL; } - mdict = PyModule_GetDict(mod); + PyObject *mdict = PyModule_GetDict(mod); if (mdict == NULL) { Py_DECREF(mod); return NULL; @@ -1416,6 +1426,7 @@ import_find_extension(PyThreadState *tstate, } else { if (def->m_base.m_init == NULL) { + assert(!PyErr_Occurred()); return NULL; } struct _Py_ext_module_loader_result res; @@ -1445,12 +1456,41 @@ import_find_extension(PyThreadState *tstate, return NULL; } } + if (_modules_by_index_set(tstate->interp, def, mod) < 0) { PyMapping_DelItem(modules, info->name); Py_DECREF(mod); return NULL; } + return mod; +} + +static PyObject * +import_find_extension(PyThreadState *tstate, + struct _Py_ext_module_loader_info *info) +{ + /* Only single-phase init modules will be in the cache. */ + PyModuleDef *def = _extensions_cache_get(info->path, info->name); + if (def == NULL) { + return NULL; + } + assert_singlephase(def); + + /* It may have been successfully imported previously + in an interpreter that allows legacy modules + but is not allowed in the current interpreter. */ + const char *name_buf = PyUnicode_AsUTF8(info->name); + assert(name_buf != NULL); + if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { + return NULL; + } + + PyObject *mod = reload_singlephase_extension(tstate, def, info); + if (mod == NULL) { + return NULL; + } + int verbose = _PyInterpreterState_GetConfig(tstate->interp)->verbose; if (verbose) { PySys_FormatStderr("import %U # previously loaded (%R)\n", From 852263e1086748492602a90347ecc0a3925e1dda Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Fri, 3 May 2024 15:02:11 +0200 Subject: [PATCH 078/220] gh-117492: Clarify documentation of `typing.Never` (#117678) Co-authored-by: Jelle Zijlstra Co-authored-by: Erlend E. Aasland --- Doc/library/typing.rst | 46 ++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index d816e6368f40d29..573318bd76576b3 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -852,14 +852,25 @@ using ``[]``. .. versionadded:: 3.11 .. data:: Never + NoReturn - The `bottom type `_, + :data:`!Never` and :data:`!NoReturn` represent the + `bottom type `_, a type that has no members. - This can be used to define a function that should never be - called, or a function that never returns:: + They can be used to indicate that a function never returns, + such as :func:`sys.exit`:: - from typing import Never + from typing import Never # or NoReturn + + def stop() -> Never: + raise RuntimeError('no way') + + Or to define a function that should never be + called, as there are no valid arguments, such as + :func:`assert_never`:: + + from typing import Never # or NoReturn def never_call_me(arg: Never) -> None: pass @@ -872,31 +883,18 @@ using ``[]``. case str(): print("It's a str") case _: - never_call_me(arg) # OK, arg is of type Never - - .. versionadded:: 3.11 - - On older Python versions, :data:`NoReturn` may be used to express the - same concept. ``Never`` was added to make the intended meaning more explicit. + never_call_me(arg) # OK, arg is of type Never (or NoReturn) -.. data:: NoReturn + :data:`!Never` and :data:`!NoReturn` have the same meaning in the type system + and static type checkers treat both equivalently. - Special type indicating that a function never returns. - - For example:: + .. versionadded:: 3.6.2 - from typing import NoReturn + Added :data:`NoReturn`. - def stop() -> NoReturn: - raise RuntimeError('no way') - - ``NoReturn`` can also be used as a - `bottom type `_, a type that - has no values. Starting in Python 3.11, the :data:`Never` type should - be used for this concept instead. Type checkers should treat the two - equivalently. + .. versionadded:: 3.11 - .. versionadded:: 3.6.2 + Added :data:`Never`. .. data:: Self From ca269e58c290be8ca11bb728004ea842d9f85e3a Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 3 May 2024 06:17:32 -0700 Subject: [PATCH 079/220] gh-116126: Implement PEP 696 (#116129) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Alex Waygood Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> Co-authored-by: Shantanu <12621235+hauntsaninja@users.noreply.github.com> --- Doc/library/ast.rst | 52 +- Doc/library/typing.rst | 82 +- Doc/reference/compound_stmts.rst | 31 +- Doc/reference/executionmodel.rst | 8 +- Doc/whatsnew/3.13.rst | 6 + Grammar/python.gram | 10 +- Include/internal/pycore_ast.h | 20 +- Include/internal/pycore_ast_state.h | 1 + Include/internal/pycore_intrinsics.h | 3 +- Include/internal/pycore_typevarobject.h | 2 + Lib/ast.py | 9 + Lib/test/test_ast.py | 42 +- Lib/test/test_type_params.py | 114 +- Lib/test/test_typing.py | 190 +++- Lib/test/test_unparse.py | 42 + Lib/typing.py | 95 +- ...-02-29-18-55-45.gh-issue-116129.wsFnIq.rst | 2 + Modules/_typingmodule.c | 3 + Objects/clinic/typevarobject.c.h | 170 ++- Objects/typevarobject.c | 327 +++++- Parser/Python.asdl | 6 +- Parser/parser.c | 991 ++++++++++-------- Python/Python-ast.c | 178 +++- Python/ast.c | 12 +- Python/compile.c | 89 +- Python/intrinsics.c | 1 + Python/symtable.c | 59 +- Tools/c-analyzer/cpython/globals-to-fix.tsv | 2 + 28 files changed, 1924 insertions(+), 623 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-29-18-55-45.gh-issue-116129.wsFnIq.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 09f2a404786fb6b..e954c38c7c55b59 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -1748,15 +1748,17 @@ Type parameters :ref:`Type parameters ` can exist on classes, functions, and type aliases. -.. class:: TypeVar(name, bound) +.. class:: TypeVar(name, bound, default_value) - A :class:`typing.TypeVar`. ``name`` is the name of the type variable. - ``bound`` is the bound or constraints, if any. If ``bound`` is a :class:`Tuple`, - it represents constraints; otherwise it represents the bound. + A :class:`typing.TypeVar`. *name* is the name of the type variable. + *bound* is the bound or constraints, if any. If *bound* is a :class:`Tuple`, + it represents constraints; otherwise it represents the bound. *default_value* + is the default value; if the :class:`!TypeVar` has no default, this + attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[T: int] = list[T]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[T: int = bool] = list[T]"), indent=4)) Module( body=[ TypeAlias( @@ -1764,7 +1766,8 @@ aliases. type_params=[ TypeVar( name='T', - bound=Name(id='int', ctx=Load()))], + bound=Name(id='int', ctx=Load()), + default_value=Name(id='bool', ctx=Load()))], value=Subscript( value=Name(id='list', ctx=Load()), slice=Name(id='T', ctx=Load()), @@ -1772,19 +1775,30 @@ aliases. .. versionadded:: 3.12 -.. class:: ParamSpec(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: ParamSpec(name, default_value) - A :class:`typing.ParamSpec`. ``name`` is the name of the parameter specification. + A :class:`typing.ParamSpec`. *name* is the name of the parameter specification. + *default_value* is the default value; if the :class:`!ParamSpec` has no default, + this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[**P] = Callable[P, int]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[**P = (int, str)] = Callable[P, int]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - ParamSpec(name='P')], + ParamSpec( + name='P', + default_value=Tuple( + elts=[ + Name(id='int', ctx=Load()), + Name(id='str', ctx=Load())], + ctx=Load()))], value=Subscript( value=Name(id='Callable', ctx=Load()), slice=Tuple( @@ -1796,19 +1810,26 @@ aliases. .. versionadded:: 3.12 -.. class:: TypeVarTuple(name) + .. versionchanged:: 3.13 + Added the *default_value* parameter. + +.. class:: TypeVarTuple(name, default_value) - A :class:`typing.TypeVarTuple`. ``name`` is the name of the type variable tuple. + A :class:`typing.TypeVarTuple`. *name* is the name of the type variable tuple. + *default_value* is the default value; if the :class:`!TypeVarTuple` has no + default, this attribute will be set to ``None``. .. doctest:: - >>> print(ast.dump(ast.parse("type Alias[*Ts] = tuple[*Ts]"), indent=4)) + >>> print(ast.dump(ast.parse("type Alias[*Ts = ()] = tuple[*Ts]"), indent=4)) Module( body=[ TypeAlias( name=Name(id='Alias', ctx=Store()), type_params=[ - TypeVarTuple(name='Ts')], + TypeVarTuple( + name='Ts', + default_value=Tuple(ctx=Load()))], value=Subscript( value=Name(id='tuple', ctx=Load()), slice=Tuple( @@ -1821,6 +1842,9 @@ aliases. .. versionadded:: 3.12 + .. versionchanged:: 3.13 + Added the *default_value* parameter. + Function and class definitions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 573318bd76576b3..e06287250641e66 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1614,7 +1614,7 @@ without the dedicated syntax, as documented below. .. _typevar: -.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False) +.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False, infer_variance=False, default=typing.NoDefault) Type variable. @@ -1752,15 +1752,35 @@ without the dedicated syntax, as documented below. the constraints are evaluated only when the attribute is accessed, not when the type variable is created (see :ref:`lazy-evaluation`). + .. attribute:: __default__ + + The default value of the type variable, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the type variable has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionchanged:: 3.12 Type variables can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. The ``infer_variance`` parameter was added. + .. versionchanged:: 3.13 + + Support for default values was added. + .. _typevartuple: -.. class:: TypeVarTuple(name) +.. class:: TypeVarTuple(name, default=typing.NoDefault) Type variable tuple. A specialized form of :ref:`type variable ` that enables *variadic* generics. @@ -1870,6 +1890,22 @@ without the dedicated syntax, as documented below. The name of the type variable tuple. + .. attribute:: __default__ + + The default value of the type variable tuple, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the type variable tuple has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + .. versionadded:: 3.11 .. versionchanged:: 3.12 @@ -1877,7 +1913,11 @@ without the dedicated syntax, as documented below. Type variable tuples can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) + .. versionchanged:: 3.13 + + Support for default values was added. + +.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False, default=typing.NoDefault) Parameter specification variable. A specialized version of :ref:`type variables `. @@ -1946,6 +1986,22 @@ without the dedicated syntax, as documented below. The name of the parameter specification. + .. attribute:: __default__ + + The default value of the parameter specification, or :data:`typing.NoDefault` if it + has no default. + + .. versionadded:: 3.13 + + .. method:: has_default() + + Return whether or not the parameter specification has a default value. This is equivalent + to checking whether :attr:`__default__` is not the :data:`typing.NoDefault` + singleton, except that it does not force evaluation of the + :ref:`lazily evaluated ` default value. + + .. versionadded:: 3.13 + Parameter specification variables created with ``covariant=True`` or ``contravariant=True`` can be used to declare covariant or contravariant generic types. The ``bound`` argument is also accepted, similar to @@ -1959,6 +2015,10 @@ without the dedicated syntax, as documented below. Parameter specifications can now be declared using the :ref:`type parameter ` syntax introduced by :pep:`695`. + .. versionchanged:: 3.13 + + Support for default values was added. + .. note:: Only parameter specification variables defined in global scope can be pickled. @@ -3171,6 +3231,22 @@ Introspection helpers .. versionadded:: 3.7.4 +.. data:: NoDefault + + A sentinel object used to indicate that a type parameter has no default + value. For example: + + .. doctest:: + + >>> T = TypeVar("T") + >>> T.__default__ is typing.NoDefault + True + >>> S = TypeVar("S", default=None) + >>> S.__default__ is None + True + + .. versionadded:: 3.13 + Constant -------- diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 374404bf33abbe2..42cca0664df71d5 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -1620,15 +1620,18 @@ Type parameter lists .. versionadded:: 3.12 +.. versionchanged:: 3.13 + Support for default values was added (see :pep:`696`). + .. index:: single: type parameters .. productionlist:: python-grammar type_params: "[" `type_param` ("," `type_param`)* "]" type_param: `typevar` | `typevartuple` | `paramspec` - typevar: `identifier` (":" `expression`)? - typevartuple: "*" `identifier` - paramspec: "**" `identifier` + typevar: `identifier` (":" `expression`)? ("=" `expression`)? + typevartuple: "*" `identifier` ("=" `expression`)? + paramspec: "**" `identifier` ("=" `expression`)? :ref:`Functions ` (including :ref:`coroutines `), :ref:`classes ` and :ref:`type aliases ` may @@ -1694,19 +1697,31 @@ evaluated in a separate :ref:`annotation scope `. :data:`typing.TypeVarTuple`\ s and :data:`typing.ParamSpec`\ s cannot have bounds or constraints. +All three flavors of type parameters can also have a *default value*, which is used +when the type parameter is not explicitly provided. This is added by appending +a single equals sign (``=``) followed by an expression. Like the bounds and +constraints of type variables, the default value is not evaluated when the +object is created, but only when the type parameter's ``__default__`` attribute +is accessed. To this end, the default value is evaluated in a separate +:ref:`annotation scope `. If no default value is specified +for a type parameter, the ``__default__`` attribute is set to the special +sentinel object :data:`typing.NoDefault`. + The following example indicates the full set of allowed type parameter declarations:: def overly_generic[ SimpleTypeVar, + TypeVarWithDefault = int, TypeVarWithBound: int, TypeVarWithConstraints: (str, bytes), - *SimpleTypeVarTuple, - **SimpleParamSpec, + *SimpleTypeVarTuple = (int, float), + **SimpleParamSpec = (str, bytearray), ]( a: SimpleTypeVar, - b: TypeVarWithBound, - c: Callable[SimpleParamSpec, TypeVarWithConstraints], - *d: SimpleTypeVarTuple, + b: TypeVarWithDefault, + c: TypeVarWithBound, + d: Callable[SimpleParamSpec, TypeVarWithConstraints], + *e: SimpleTypeVarTuple, ): ... .. _generic-functions: diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst index ed50faed6c940d7..f24e1537af39ed5 100644 --- a/Doc/reference/executionmodel.rst +++ b/Doc/reference/executionmodel.rst @@ -205,7 +205,7 @@ Annotation scopes are used in the following contexts: * Type parameter lists for :ref:`generic classes `. A generic class's base classes and keyword arguments are executed within the annotation scope, but its decorators are not. -* The bounds and constraints for type variables +* The bounds, constraints, and default values for type parameters (:ref:`lazily evaluated `). * The value of type aliases (:ref:`lazily evaluated `). @@ -232,13 +232,17 @@ Annotation scopes differ from function scopes in the following ways: .. versionadded:: 3.12 Annotation scopes were introduced in Python 3.12 as part of :pep:`695`. +.. versionchanged:: 3.13 + Annotation scopes are also used for type parameter defaults, as + introduced by :pep:`696`. + .. _lazy-evaluation: Lazy evaluation --------------- The values of type aliases created through the :keyword:`type` statement are -*lazily evaluated*. The same applies to the bounds and constraints of type +*lazily evaluated*. The same applies to the bounds, constraints, and default values of type variables created through the :ref:`type parameter syntax `. This means that they are not evaluated when the type alias or type variable is created. Instead, they are only evaluated when doing so is necessary to resolve diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d59c4ee22e67d3e..d996cf6c4d9b52b 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -89,6 +89,8 @@ Interpreter improvements: New typing features: +* :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, + and :data:`typing.TypeVarTuple`) now support defaults. * :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive type narrowing behavior. @@ -850,6 +852,10 @@ typing an item of a :class:`typing.TypedDict` as read-only for type checkers. See :pep:`705` for more details. +* Add :data:`typing.NoDefault`, a sentinel object used to represent the defaults + of some parameters in the :mod:`typing` module. (Contributed by Jelle Zijlstra in + :gh:`116126`.) + unicodedata ----------- diff --git a/Grammar/python.gram b/Grammar/python.gram index 05d7837e3aa6dba..1c1c53c4b73ace1 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -647,21 +647,25 @@ type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } type_param[type_param_ty] (memo): - | a=NAME b=[type_param_bound] { _PyAST_TypeVar(a->v.Name.id, b, EXTRA) } + | a=NAME b=[type_param_bound] c=[type_param_default] { _PyAST_TypeVar(a->v.Name.id, b, c, EXTRA) } | '*' a=NAME colon=':' e=expression { RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple") } - | '*' a=NAME { _PyAST_TypeVarTuple(a->v.Name.id, EXTRA) } + | '*' a=NAME b=[type_param_starred_default] { _PyAST_TypeVarTuple(a->v.Name.id, b, EXTRA) } | '**' a=NAME colon=':' e=expression { RAISE_SYNTAX_ERROR_STARTING_FROM(colon, e->kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec") } - | '**' a=NAME { _PyAST_ParamSpec(a->v.Name.id, EXTRA) } + | '**' a=NAME b=[type_param_default] { _PyAST_ParamSpec(a->v.Name.id, b, EXTRA) } type_param_bound[expr_ty]: ':' e=expression { e } +type_param_default[expr_ty]: '=' e=expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } +type_param_starred_default[expr_ty]: '=' e=star_expression { + CHECK_VERSION(expr_ty, 13, "Type parameter defaults are", e) } # EXPRESSIONS # ----------- diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h index f222d485e0b54b9..f5bf1205a82be98 100644 --- a/Include/internal/pycore_ast.h +++ b/Include/internal/pycore_ast.h @@ -657,14 +657,17 @@ struct _type_param { struct { identifier name; expr_ty bound; + expr_ty default_value; } TypeVar; struct { identifier name; + expr_ty default_value; } ParamSpec; struct { identifier name; + expr_ty default_value; } TypeVarTuple; } v; @@ -892,14 +895,15 @@ pattern_ty _PyAST_MatchOr(asdl_pattern_seq * patterns, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena); type_ignore_ty _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena); -type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int - col_offset, int end_lineno, int end_col_offset, - PyArena *arena); -type_param_ty _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena); -type_param_ty _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, - int end_lineno, int end_col_offset, PyArena - *arena); +type_param_ty _PyAST_TypeVar(identifier name, expr_ty bound, expr_ty + default_value, int lineno, int col_offset, int + end_lineno, int end_col_offset, PyArena *arena); +type_param_ty _PyAST_ParamSpec(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); +type_param_ty _PyAST_TypeVarTuple(identifier name, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int + end_col_offset, PyArena *arena); PyObject* PyAST_mod2obj(mod_ty t); diff --git a/Include/internal/pycore_ast_state.h b/Include/internal/pycore_ast_state.h index f1b1786264803be..09ae95465495c01 100644 --- a/Include/internal/pycore_ast_state.h +++ b/Include/internal/pycore_ast_state.h @@ -184,6 +184,7 @@ struct ast_state { PyObject *conversion; PyObject *ctx; PyObject *decorator_list; + PyObject *default_value; PyObject *defaults; PyObject *elt; PyObject *elts; diff --git a/Include/internal/pycore_intrinsics.h b/Include/internal/pycore_intrinsics.h index 8fa88ea3f74caa8..39c2a30f6e979de 100644 --- a/Include/internal/pycore_intrinsics.h +++ b/Include/internal/pycore_intrinsics.h @@ -28,8 +28,9 @@ #define INTRINSIC_TYPEVAR_WITH_BOUND 2 #define INTRINSIC_TYPEVAR_WITH_CONSTRAINTS 3 #define INTRINSIC_SET_FUNCTION_TYPE_PARAMS 4 +#define INTRINSIC_SET_TYPEPARAM_DEFAULT 5 -#define MAX_INTRINSIC_2 4 +#define MAX_INTRINSIC_2 5 typedef PyObject *(*intrinsic_func1)(PyThreadState* tstate, PyObject *value); typedef PyObject *(*intrinsic_func2)(PyThreadState* tstate, PyObject *value1, PyObject *value2); diff --git a/Include/internal/pycore_typevarobject.h b/Include/internal/pycore_typevarobject.h index c9fa97d68207571..80a2daf4efc16a2 100644 --- a/Include/internal/pycore_typevarobject.h +++ b/Include/internal/pycore_typevarobject.h @@ -13,10 +13,12 @@ extern PyObject *_Py_make_paramspec(PyThreadState *, PyObject *); extern PyObject *_Py_make_typevartuple(PyThreadState *, PyObject *); extern PyObject *_Py_make_typealias(PyThreadState *, PyObject *); extern PyObject *_Py_subscript_generic(PyThreadState *, PyObject *); +extern PyObject *_Py_set_typeparam_default(PyThreadState *, PyObject *, PyObject *); extern int _Py_initialize_generic(PyInterpreterState *); extern void _Py_clear_generic_types(PyInterpreterState *); extern PyTypeObject _PyTypeAlias_Type; +extern PyObject _Py_NoDefaultStruct; #ifdef __cplusplus } diff --git a/Lib/ast.py b/Lib/ast.py index 9f386051659e76c..d7e51aba5957062 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1124,12 +1124,21 @@ def visit_TypeVar(self, node): if node.bound: self.write(": ") self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 68024304dfb7466..6d05c8f8f47c50d 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -195,16 +195,19 @@ def to_tuple(t): "type X[T, *Ts, **P] = (T, Ts, P)", "type X[T: int, *Ts, **P] = (T, Ts, P)", "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", # Generic classes "class X[T]: pass", "class X[T, *Ts, **P]: pass", "class X[T: int, *Ts, **P]: pass", "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", # Generic functions "def f[T](): pass", "def f[T, *Ts, **P](): pass", "def f[T: int, *Ts, **P](): pass", "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", ] # These are compiled through "single" @@ -1108,6 +1111,18 @@ def test_type_params_feature_version(self): with self.assertRaises(SyntaxError): ast.parse(sample, feature_version=(3, 11)) + def test_type_params_default_feature_version(self): + samples = [ + "type X[*Ts=int] = int", + "class X[T=int]: pass", + "def f[**P=int](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 12)) + def test_invalid_major_feature_version(self): with self.assertRaises(ValueError): ast.parse('pass', feature_version=(2, 7)) @@ -3261,18 +3276,21 @@ def main(): ('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), ('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), ('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), ] single_results = [ ('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 4b86395ee74f753..4c5bf6bfd33c75a 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -6,7 +6,7 @@ import weakref from test.support import requires_working_socket, check_syntax_error, run_code -from typing import Generic, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args +from typing import Generic, NoDefault, Sequence, TypeVar, TypeVarTuple, ParamSpec, get_args class TypeParamsInvalidTest(unittest.TestCase): @@ -412,6 +412,14 @@ def test_comprehension_02(self): func, = T.__bound__ self.assertEqual(func(), 1) + def test_comprehension_03(self): + def F[T: [lambda: T for T in (T, [1])[1]]](): return [lambda: T for T in T.__name__] + func, = F() + self.assertEqual(func(), "T") + T, = F.__type_params__ + func, = T.__bound__ + self.assertEqual(func(), 1) + def test_gen_exp_in_nested_class(self): code = """ from test.test_type_params import make_base @@ -591,10 +599,12 @@ class Foo[T: Foo, U: (Foo, Foo)]: self.assertEqual(type_params[0].__name__, "T") self.assertIs(type_params[0].__bound__, Foo) self.assertEqual(type_params[0].__constraints__, ()) + self.assertIs(type_params[0].__default__, NoDefault) self.assertEqual(type_params[1].__name__, "U") self.assertIs(type_params[1].__bound__, None) self.assertEqual(type_params[1].__constraints__, (Foo, Foo)) + self.assertIs(type_params[1].__default__, NoDefault) def test_evaluation_error(self): class Foo[T: Undefined, U: (Undefined,)]: @@ -605,6 +615,8 @@ class Foo[T: Undefined, U: (Undefined,)]: type_params[0].__bound__ self.assertEqual(type_params[0].__constraints__, ()) self.assertIs(type_params[1].__bound__, None) + self.assertIs(type_params[0].__default__, NoDefault) + self.assertIs(type_params[1].__default__, NoDefault) with self.assertRaises(NameError): type_params[1].__constraints__ @@ -1158,3 +1170,103 @@ class Inner[U](T): """ with self.assertRaises(RuntimeError): run_code(code) + + +class DefaultsTest(unittest.TestCase): + def test_defaults_on_func(self): + ns = run_code(""" + def func[T=int, **U=float, *V=None](): + pass + """) + + T, U, V = ns["func"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_class(self): + ns = run_code(""" + class C[T=int, **U=float, *V=None]: + pass + """) + + T, U, V = ns["C"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_defaults_on_type_alias(self): + ns = run_code(""" + type Alias[T = int, **U = float, *V = None] = int + """) + + T, U, V = ns["Alias"].__type_params__ + self.assertIs(T.__default__, int) + self.assertIs(U.__default__, float) + self.assertIs(V.__default__, None) + + def test_starred_invalid(self): + check_syntax_error(self, "type Alias[T = *int] = int") + check_syntax_error(self, "type Alias[**P = *int] = int") + + def test_starred_typevartuple(self): + ns = run_code(""" + default = tuple[int, str] + type Alias[*Ts = *default] = Ts + """) + + Ts, = ns["Alias"].__type_params__ + self.assertEqual(Ts.__default__, next(iter(ns["default"]))) + + def test_nondefault_after_default(self): + check_syntax_error(self, "def func[T=int, U](): pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "class C[T=int, U]: pass", "non-default type parameter 'U' follows default type parameter") + check_syntax_error(self, "type A[T=int, U] = int", "non-default type parameter 'U' follows default type parameter") + + def test_lazy_evaluation(self): + ns = run_code(""" + type Alias[T = Undefined, *U = Undefined, **V = Undefined] = int + """) + + T, U, V = ns["Alias"].__type_params__ + + with self.assertRaises(NameError): + T.__default__ + with self.assertRaises(NameError): + U.__default__ + with self.assertRaises(NameError): + V.__default__ + + ns["Undefined"] = "defined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + # Now it is cached + ns["Undefined"] = "redefined" + self.assertEqual(T.__default__, "defined") + self.assertEqual(U.__default__, "defined") + self.assertEqual(V.__default__, "defined") + + def test_symtable_key_regression_default(self): + # Test against the bugs that would happen if we used .default_ + # as the key in the symtable. + ns = run_code(""" + type X[T = [T for T in [T]]] = T + """) + + T, = ns["X"].__type_params__ + self.assertEqual(T.__default__, [T]) + + def test_symtable_key_regression_name(self): + # Test against the bugs that would happen if we used .name + # as the key in the symtable. + ns = run_code(""" + type X1[T = A] = T + type X2[T = B] = T + A = "A" + B = "B" + """) + + self.assertEqual(ns["X1"].__type_params__[0].__default__, "A") + self.assertEqual(ns["X2"].__type_params__[0].__default__, "B") diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 703fe84f3aad107..112db03ae878875 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -38,7 +38,7 @@ from typing import Self, LiteralString from typing import TypeAlias from typing import ParamSpec, Concatenate, ParamSpecArgs, ParamSpecKwargs -from typing import TypeGuard, TypeIs +from typing import TypeGuard, TypeIs, NoDefault import abc import textwrap import typing @@ -580,6 +580,174 @@ def test_constructor(self): self.assertIs(T.__bound__, None) +class TypeParameterDefaultsTests(BaseTestCase): + def test_typevar(self): + T = TypeVar('T', default=int) + self.assertEqual(T.__default__, int) + self.assertTrue(T.has_default()) + self.assertIsInstance(T, TypeVar) + + class A(Generic[T]): ... + Alias = Optional[T] + + def test_typevar_none(self): + U = TypeVar('U') + U_None = TypeVar('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[T]: ... + T, = X.__type_params__ + self.assertIs(T.__default__, NoDefault) + self.assertFalse(T.has_default()) + + def test_paramspec(self): + P = ParamSpec('P', default=(str, int)) + self.assertEqual(P.__default__, (str, int)) + self.assertTrue(P.has_default()) + self.assertIsInstance(P, ParamSpec) + + class A(Generic[P]): ... + Alias = typing.Callable[P, None] + + P_default = ParamSpec('P_default', default=...) + self.assertIs(P_default.__default__, ...) + + def test_paramspec_none(self): + U = ParamSpec('U') + U_None = ParamSpec('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**P]: ... + P, = X.__type_params__ + self.assertIs(P.__default__, NoDefault) + self.assertFalse(P.has_default()) + + def test_typevartuple(self): + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + self.assertTrue(Ts.has_default()) + self.assertIsInstance(Ts, TypeVarTuple) + + class A(Generic[Unpack[Ts]]): ... + Alias = Optional[Unpack[Ts]] + + def test_typevartuple_specialization(self): + T = TypeVar("T") + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, Unpack[Ts]]): ... + self.assertEqual(A[float].__args__, (float, str, int)) + self.assertEqual(A[float, range].__args__, (float, range)) + self.assertEqual(A[float, *tuple[int, ...]].__args__, (float, *tuple[int, ...])) + + def test_typevar_and_typevartuple_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + Ts = TypeVarTuple('Ts', default=Unpack[Tuple[str, int]]) + self.assertEqual(Ts.__default__, Unpack[Tuple[str, int]]) + class A(Generic[T, U, Unpack[Ts]]): ... + self.assertEqual(A[int].__args__, (int, float, str, int)) + self.assertEqual(A[int, str].__args__, (int, str, str, int)) + self.assertEqual(A[int, str, range].__args__, (int, str, range)) + self.assertEqual(A[int, str, *tuple[int, ...]].__args__, (int, str, *tuple[int, ...])) + + def test_no_default_after_typevar_tuple(self): + T = TypeVar("T", default=int) + Ts = TypeVarTuple("Ts") + Ts_default = TypeVarTuple("Ts_default", default=Unpack[Tuple[str, int]]) + + with self.assertRaises(TypeError): + class X(Generic[*Ts, T]): ... + + with self.assertRaises(TypeError): + class Y(Generic[*Ts_default, T]): ... + + def test_paramspec_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P]): ... + self.assertEqual(A[float].__args__, (float, (str, int))) + self.assertEqual(A[float, [range]].__args__, (float, (range,))) + + def test_typevar_and_paramspec_specialization(self): + T = TypeVar("T") + U = TypeVar("U", default=float) + P = ParamSpec('P', default=[str, int]) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, U, P]): ... + self.assertEqual(A[float].__args__, (float, float, (str, int))) + self.assertEqual(A[float, int].__args__, (float, int, (str, int))) + self.assertEqual(A[float, int, [range]].__args__, (float, int, (range,))) + + def test_paramspec_and_typevar_specialization(self): + T = TypeVar("T") + P = ParamSpec('P', default=[str, int]) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, [str, int]) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, (str, int), float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + + def test_typevartuple_none(self): + U = TypeVarTuple('U') + U_None = TypeVarTuple('U_None', default=None) + self.assertIs(U.__default__, NoDefault) + self.assertFalse(U.has_default()) + self.assertIs(U_None.__default__, None) + self.assertTrue(U_None.has_default()) + + class X[**Ts]: ... + Ts, = X.__type_params__ + self.assertIs(Ts.__default__, NoDefault) + self.assertFalse(Ts.has_default()) + + def test_no_default_after_non_default(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + + with self.assertRaisesRegex( + TypeError, r"Type parameter ~T without a default follows type parameter with a default" + ): + Test = Generic[DefaultStrT, T] + + def test_need_more_params(self): + DefaultStrT = TypeVar('DefaultStrT', default=str) + T = TypeVar('T') + U = TypeVar('U') + + class A(Generic[T, U, DefaultStrT]): ... + A[int, bool] + A[int, bool, str] + + with self.assertRaisesRegex( + TypeError, r"Too few arguments for .+; actual 1, expected at least 2" + ): + Test = A[int] + + def test_pickle(self): + global U, U_co, U_contra, U_default # pickle wants to reference the class by name + U = TypeVar('U') + U_co = TypeVar('U_co', covariant=True) + U_contra = TypeVar('U_contra', contravariant=True) + U_default = TypeVar('U_default', default=int) + for proto in range(pickle.HIGHEST_PROTOCOL): + for typevar in (U, U_co, U_contra, U_default): + z = pickle.loads(pickle.dumps(typevar, proto)) + self.assertEqual(z.__name__, typevar.__name__) + self.assertEqual(z.__covariant__, typevar.__covariant__) + self.assertEqual(z.__contravariant__, typevar.__contravariant__) + self.assertEqual(z.__bound__, typevar.__bound__) + self.assertEqual(z.__default__, typevar.__default__) + + def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: """Renders templates with possible combinations of replacements. @@ -10001,6 +10169,26 @@ class CustomerModel(ModelBase, init=False): self.assertIsInstance(CustomerModel, Decorated) +class NoDefaultTests(BaseTestCase): + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + s = pickle.dumps(NoDefault, proto) + loaded = pickle.loads(s) + self.assertIs(NoDefault, loaded) + + def test_constructor(self): + self.assertIs(NoDefault, type(NoDefault)()) + with self.assertRaises(TypeError): + NoDefault(1) + + def test_repr(self): + self.assertEqual(repr(NoDefault), 'typing.NoDefault') + + def test_no_call(self): + with self.assertRaises(TypeError): + NoDefault() + + class AllTests(BaseTestCase): """Tests for __all__.""" diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index bb15f64c59dbd13..35394f29fbe49df 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -673,6 +673,20 @@ def test_quote_in_format_spec(self): self.check_ast_roundtrip("""f'\\'{x:\\"}' """) self.check_ast_roundtrip("""f'\\'{x:\\\\"}' """) + def test_type_params(self): + self.check_ast_roundtrip("type A = int") + self.check_ast_roundtrip("type A[T] = int") + self.check_ast_roundtrip("type A[T: int] = int") + self.check_ast_roundtrip("type A[T = int] = int") + self.check_ast_roundtrip("type A[T: int = int] = int") + self.check_ast_roundtrip("type A[**P] = int") + self.check_ast_roundtrip("type A[**P = int] = int") + self.check_ast_roundtrip("type A[*Ts] = int") + self.check_ast_roundtrip("type A[*Ts = int] = int") + self.check_ast_roundtrip("type A[*Ts = *int] = int") + self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") + self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -723,6 +737,20 @@ def test_function_with_type_params_and_bound(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "def f[T: int]():\n pass") + def test_function_with_type_params_and_default(self): + node = ast.FunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "def f[T = 1, *Ts = *1, **P = 1]():\n pass") + def test_async_function(self): node = ast.AsyncFunctionDef( name="f", @@ -746,6 +774,20 @@ def test_async_function_with_type_params(self): ast.fix_missing_locations(node) self.assertEqual(ast.unparse(node), "async def f[T]():\n pass") + def test_async_function_with_type_params_and_default(self): + node = ast.AsyncFunctionDef( + name="f", + args=ast.arguments(), + body=[ast.Pass()], + type_params=[ + ast.TypeVar("T", default_value=ast.Constant(value=1)), + ast.TypeVarTuple("Ts", default_value=ast.Starred(value=ast.Constant(value=1), ctx=ast.Load())), + ast.ParamSpec("P", default_value=ast.Constant(value=1)), + ], + ) + ast.fix_missing_locations(node) + self.assertEqual(ast.unparse(node), "async def f[T = 1, *Ts = *1, **P = 1]():\n pass") + class DirectoryTestCase(ASTTestCase): """Test roundtrip behaviour on all files in Lib and Lib/test.""" diff --git a/Lib/typing.py b/Lib/typing.py index eff65cfb68b8666..3f6ff491e7b918c 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -38,6 +38,7 @@ ParamSpecKwargs, TypeAliasType, Generic, + NoDefault, ) # Please keep __all__ alphabetized within each category. @@ -138,6 +139,7 @@ 'NewType', 'no_type_check', 'no_type_check_decorator', + 'NoDefault', 'NoReturn', 'NotRequired', 'overload', @@ -266,6 +268,10 @@ def _collect_parameters(args): >>> _collect_parameters((T, Callable[P, T])) (~T, ~P) """ + # required type parameter cannot appear after parameter with default + default_encountered = False + # or after TypeVarTuple + type_var_tuple_encountered = False parameters = [] for t in args: if isinstance(t, type): @@ -280,27 +286,58 @@ def _collect_parameters(args): parameters.append(collected) elif hasattr(t, '__typing_subst__'): if t not in parameters: + if type_var_tuple_encountered and t.has_default(): + raise TypeError('Type parameter with a default' + ' follows TypeVarTuple') + + if t.has_default(): + default_encountered = True + elif default_encountered: + raise TypeError(f'Type parameter {t!r} without a default' + ' follows type parameter with a default') + parameters.append(t) else: + if _is_unpacked_typevartuple(t): + type_var_tuple_encountered = True for x in getattr(t, '__parameters__', ()): if x not in parameters: parameters.append(x) return tuple(parameters) -def _check_generic(cls, parameters, elen): +def _check_generic_specialization(cls, arguments): """Check correct count for parameters of a generic cls (internal helper). This gives a nice error message in case of count mismatch. """ - if not elen: + expected_len = len(cls.__parameters__) + if not expected_len: raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments for {cls};" - f" actual {alen}, expected {elen}") + actual_len = len(arguments) + if actual_len != expected_len: + # deal with defaults + if actual_len < expected_len: + # If the parameter at index `actual_len` in the parameters list + # has a default, then all parameters after it must also have + # one, because we validated as much in _collect_parameters(). + # That means that no error needs to be raised here, despite + # the number of arguments being passed not matching the number + # of parameters: all parameters that aren't explicitly + # specialized in this call are parameters with default values. + if cls.__parameters__[actual_len].has_default(): + return + + expected_len -= sum(p.has_default() for p in cls.__parameters__) + expect_val = f"at least {expected_len}" + else: + expect_val = expected_len + + raise TypeError(f"Too {'many' if actual_len > expected_len else 'few'} arguments" + f" for {cls}; actual {actual_len}, expected {expect_val}") -def _unpack_args(args): + +def _unpack_args(*args): newargs = [] for arg in args: subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) @@ -1089,11 +1126,15 @@ def _typevartuple_prepare_subst(self, alias, args): elif left + right > alen: raise TypeError(f"Too few arguments for {alias};" f" actual {alen}, expected at least {plen-1}") + if left == alen - right and self.has_default(): + replacement = _unpack_args(self.__default__) + else: + replacement = args[left: alen - right] return ( *args[:left], *([fillarg]*(typevartuple_index - left)), - tuple(args[left: alen - right]), + replacement, *([fillarg]*(plen - right - left - typevartuple_index - 1)), *args[alen - right:], ) @@ -1111,6 +1152,8 @@ def _paramspec_subst(self, arg): def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) + if i == len(args) and self.has_default(): + args = [*args, self.__default__] if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1124,33 +1167,33 @@ def _paramspec_prepare_subst(self, alias, args): @_tp_cache -def _generic_class_getitem(cls, params): +def _generic_class_getitem(cls, args): """Parameterizes a generic class. At least, parameterizing a generic class is the *main* thing this method does. For example, for some generic class `Foo`, this is called when we - do `Foo[int]` - there, with `cls=Foo` and `params=int`. + do `Foo[int]` - there, with `cls=Foo` and `args=int`. However, note that this method is also called when defining generic classes in the first place with `class Foo(Generic[T]): ...`. """ - if not isinstance(params, tuple): - params = (params,) + if not isinstance(args, tuple): + args = (args,) - params = tuple(_type_convert(p) for p in params) + args = tuple(_type_convert(p) for p in args) is_generic_or_protocol = cls in (Generic, Protocol) if is_generic_or_protocol: # Generic and Protocol can only be subscripted with unique type variables. - if not params: + if not args: raise TypeError( f"Parameter list to {cls.__qualname__}[...] cannot be empty" ) - if not all(_is_typevar_like(p) for p in params): + if not all(_is_typevar_like(p) for p in args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be type variables " f"or parameter specification variables.") - if len(set(params)) != len(params): + if len(set(args)) != len(args): raise TypeError( f"Parameters to {cls.__name__}[...] must all be unique") else: @@ -1158,18 +1201,18 @@ def _generic_class_getitem(cls, params): for param in cls.__parameters__: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: - params = prepare(cls, params) - _check_generic(cls, params, len(cls.__parameters__)) + args = prepare(cls, args) + _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, params): + for param, new_arg in zip(cls.__parameters__, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: new_args.append(new_arg) - params = tuple(new_args) + args = tuple(new_args) - return _GenericAlias(cls, params) + return _GenericAlias(cls, args) def _generic_init_subclass(cls, *args, **kwargs): @@ -1390,8 +1433,7 @@ def __getitem__(self, args): # Preprocess `args`. if not isinstance(args, tuple): args = (args,) - args = tuple(_type_convert(p) for p in args) - args = _unpack_args(args) + args = _unpack_args(*(_type_convert(p) for p in args)) new_args = self._determine_new_args(args) r = self.copy_with(new_args) return r @@ -1552,7 +1594,12 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) - _check_generic(self, params, self._nparams) + actual_len = len(params) + if actual_len != self._nparams: + if not self._nparams: + raise TypeError(f"{self} is not a generic class") + raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" + f" actual {actual_len}, expected {self._nparams}") return self.copy_with(params) def copy_with(self, params): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-29-18-55-45.gh-issue-116129.wsFnIq.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-29-18-55-45.gh-issue-116129.wsFnIq.rst new file mode 100644 index 000000000000000..e632ad58d6ded5f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-29-18-55-45.gh-issue-116129.wsFnIq.rst @@ -0,0 +1,2 @@ +Implement :pep:`696`, adding support for defaults on type parameters. +Patch by Jelle Zijlstra. diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 9ea72bf89ce0b29..180f3d7eb01794c 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -63,6 +63,9 @@ _typing_exec(PyObject *m) if (PyModule_AddObjectRef(m, "TypeAliasType", (PyObject *)&_PyTypeAlias_Type) < 0) { return -1; } + if (PyModule_AddObjectRef(m, "NoDefault", (PyObject *)&_Py_NoDefaultStruct) < 0) { + return -1; + } return 0; } diff --git a/Objects/clinic/typevarobject.c.h b/Objects/clinic/typevarobject.c.h index 2bb0a98a2ed64c3..0ba4ff48bc88044 100644 --- a/Objects/clinic/typevarobject.c.h +++ b/Objects/clinic/typevarobject.c.h @@ -9,16 +9,16 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywordsWithVararg() PyDoc_STRVAR(typevar_new__doc__, -"typevar(name, *constraints, bound=None, covariant=False,\n" -" contravariant=False, infer_variance=False)\n" +"typevar(name, *constraints, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a TypeVar."); static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance); + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance); static PyObject * typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -26,14 +26,14 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,20 +42,21 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevar", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[6]; + PyObject *argsbuf[7]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = Py_MIN(nargs, 1) + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *constraints = NULL; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -80,7 +81,13 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[3]) { - covariant = PyObject_IsTrue(fastargs[3]); + default_value = fastargs[3]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[4]) { + covariant = PyObject_IsTrue(fastargs[4]); if (covariant < 0) { goto exit; } @@ -88,8 +95,8 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[4]) { - contravariant = PyObject_IsTrue(fastargs[4]); + if (fastargs[5]) { + contravariant = PyObject_IsTrue(fastargs[5]); if (contravariant < 0) { goto exit; } @@ -97,12 +104,12 @@ typevar_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[5]); + infer_variance = PyObject_IsTrue(fastargs[6]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = typevar_new_impl(type, name, constraints, bound, covariant, contravariant, infer_variance); + return_value = typevar_new_impl(type, name, constraints, bound, default_value, covariant, contravariant, infer_variance); exit: Py_XDECREF(constraints); @@ -117,6 +124,36 @@ PyDoc_STRVAR(typevar_typing_subst__doc__, #define TYPEVAR_TYPING_SUBST_METHODDEF \ {"__typing_subst__", (PyCFunction)typevar_typing_subst, METH_O, typevar_typing_subst__doc__}, +PyDoc_STRVAR(typevar_typing_prepare_subst__doc__, +"__typing_prepare_subst__($self, alias, args, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF \ + {"__typing_prepare_subst__", _PyCFunction_CAST(typevar_typing_prepare_subst), METH_FASTCALL, typevar_typing_prepare_subst__doc__}, + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args); + +static PyObject * +typevar_typing_prepare_subst(typevarobject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *alias; + PyObject *__clinic_args; + + if (!_PyArg_CheckPositional("__typing_prepare_subst__", nargs, 2, 2)) { + goto exit; + } + alias = args[0]; + __clinic_args = args[1]; + return_value = typevar_typing_prepare_subst_impl(self, alias, __clinic_args); + +exit: + return return_value; +} + PyDoc_STRVAR(typevar_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -134,6 +171,23 @@ typevar_reduce(typevarobject *self, PyObject *Py_UNUSED(ignored)) return typevar_reduce_impl(self); } +PyDoc_STRVAR(typevar_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVAR_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevar_has_default, METH_NOARGS, typevar_has_default__doc__}, + +static PyObject * +typevar_has_default_impl(typevarobject *self); + +static PyObject * +typevar_has_default(typevarobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevar_has_default_impl(self); +} + PyDoc_STRVAR(paramspecargs_new__doc__, "paramspecargs(origin)\n" "--\n" @@ -243,15 +297,16 @@ paramspeckwargs_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(paramspec_new__doc__, -"paramspec(name, *, bound=None, covariant=False, contravariant=False,\n" -" infer_variance=False)\n" +"paramspec(name, *, bound=None, default=typing.NoDefault,\n" +" covariant=False, contravariant=False, infer_variance=False)\n" "--\n" "\n" "Create a ParamSpec object."); static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance); + PyObject *default_value, int covariant, int contravariant, + int infer_variance); static PyObject * paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -259,14 +314,14 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 5 + #define NUM_KEYWORDS 6 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, + .ob_item = { &_Py_ID(name), &_Py_ID(bound), &_Py_ID(default), &_Py_ID(covariant), &_Py_ID(contravariant), &_Py_ID(infer_variance), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -275,19 +330,20 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "bound", "covariant", "contravariant", "infer_variance", NULL}; + static const char * const _keywords[] = {"name", "bound", "default", "covariant", "contravariant", "infer_variance", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "paramspec", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[5]; + PyObject *argsbuf[6]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; PyObject *bound = Py_None; + PyObject *default_value = &_Py_NoDefaultStruct; int covariant = 0; int contravariant = 0; int infer_variance = 0; @@ -311,7 +367,13 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } } if (fastargs[2]) { - covariant = PyObject_IsTrue(fastargs[2]); + default_value = fastargs[2]; + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[3]) { + covariant = PyObject_IsTrue(fastargs[3]); if (covariant < 0) { goto exit; } @@ -319,8 +381,8 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - if (fastargs[3]) { - contravariant = PyObject_IsTrue(fastargs[3]); + if (fastargs[4]) { + contravariant = PyObject_IsTrue(fastargs[4]); if (contravariant < 0) { goto exit; } @@ -328,12 +390,12 @@ paramspec_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - infer_variance = PyObject_IsTrue(fastargs[4]); + infer_variance = PyObject_IsTrue(fastargs[5]); if (infer_variance < 0) { goto exit; } skip_optional_kwonly: - return_value = paramspec_new_impl(type, name, bound, covariant, contravariant, infer_variance); + return_value = paramspec_new_impl(type, name, bound, default_value, covariant, contravariant, infer_variance); exit: return return_value; @@ -394,14 +456,32 @@ paramspec_reduce(paramspecobject *self, PyObject *Py_UNUSED(ignored)) return paramspec_reduce_impl(self); } +PyDoc_STRVAR(paramspec_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define PARAMSPEC_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)paramspec_has_default, METH_NOARGS, paramspec_has_default__doc__}, + +static PyObject * +paramspec_has_default_impl(paramspecobject *self); + +static PyObject * +paramspec_has_default(paramspecobject *self, PyObject *Py_UNUSED(ignored)) +{ + return paramspec_has_default_impl(self); +} + PyDoc_STRVAR(typevartuple__doc__, -"typevartuple(name)\n" +"typevartuple(name, *, default=typing.NoDefault)\n" "--\n" "\n" "Create a new TypeVarTuple with the given name."); static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name); +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value); static PyObject * typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -409,14 +489,14 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 2 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD PyObject *ob_item[NUM_KEYWORDS]; } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(name), }, + .ob_item = { &_Py_ID(name), &_Py_ID(default), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -425,17 +505,19 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", NULL}; + static const char * const _keywords[] = {"name", "default", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "typevartuple", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[1]; + PyObject *argsbuf[2]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *name; + PyObject *default_value = &_Py_NoDefaultStruct; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, 1, 1, 0, argsbuf); if (!fastargs) { @@ -446,7 +528,12 @@ typevartuple(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto exit; } name = fastargs[0]; - return_value = typevartuple_impl(type, name); + if (!noptargs) { + goto skip_optional_kwonly; + } + default_value = fastargs[1]; +skip_optional_kwonly: + return_value = typevartuple_impl(type, name, default_value); exit: return return_value; @@ -507,6 +594,23 @@ typevartuple_reduce(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) return typevartuple_reduce_impl(self); } +PyDoc_STRVAR(typevartuple_has_default__doc__, +"has_default($self, /)\n" +"--\n" +"\n"); + +#define TYPEVARTUPLE_HAS_DEFAULT_METHODDEF \ + {"has_default", (PyCFunction)typevartuple_has_default, METH_NOARGS, typevartuple_has_default__doc__}, + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self); + +static PyObject * +typevartuple_has_default(typevartupleobject *self, PyObject *Py_UNUSED(ignored)) +{ + return typevartuple_has_default_impl(self); +} + PyDoc_STRVAR(typealias_reduce__doc__, "__reduce__($self, /)\n" "--\n" @@ -591,4 +695,4 @@ typealias_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=5a582d9d89ad787b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=73b39e550e4e336c input=a9049054013a1b77]*/ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 7f80c9c61b8abcf..cc916045266aea5 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -23,6 +23,8 @@ typedef struct { PyObject *evaluate_bound; PyObject *constraints; PyObject *evaluate_constraints; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -31,12 +33,16 @@ typedef struct { typedef struct { PyObject_HEAD PyObject *name; + PyObject *default_value; + PyObject *evaluate_default; } typevartupleobject; typedef struct { PyObject_HEAD PyObject *name; PyObject *bound; + PyObject *default_value; + PyObject *evaluate_default; bool covariant; bool contravariant; bool infer_variance; @@ -53,6 +59,64 @@ typedef struct { #include "clinic/typevarobject.c.h" +/* NoDefault is a marker object to indicate that a parameter has no default. */ + +static PyObject * +NoDefault_repr(PyObject *op) +{ + return PyUnicode_FromString("typing.NoDefault"); +} + +static PyObject * +NoDefault_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("NoDefault"); +} + +static PyMethodDef notimplemented_methods[] = { + {"__reduce__", NoDefault_reduce, METH_NOARGS, NULL}, + {NULL, NULL} +}; + +static PyObject * +nodefault_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + if (PyTuple_GET_SIZE(args) || (kwargs && PyDict_GET_SIZE(kwargs))) { + PyErr_SetString(PyExc_TypeError, "NoDefaultType takes no arguments"); + return NULL; + } + return &_Py_NoDefaultStruct; +} + +static void +nodefault_dealloc(PyObject *nodefault) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref NoDefault out of existence. Instead, + * since NoDefault is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(nodefault); +} + +PyDoc_STRVAR(notimplemented_doc, +"NoDefaultType()\n" +"--\n\n" +"The type of the NoDefault singleton."); + +PyTypeObject _PyNoDefault_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "NoDefaultType", + .tp_dealloc = nodefault_dealloc, + .tp_repr = NoDefault_repr, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_doc = notimplemented_doc, + .tp_methods = notimplemented_methods, + .tp_new = nodefault_new, +}; + +PyObject _Py_NoDefaultStruct = _PyObject_HEAD_INIT(&_PyNoDefault_Type); + + static PyObject * call_typing_func_object(const char *name, PyObject **args, size_t nargs) { @@ -200,6 +264,8 @@ typevar_dealloc(PyObject *self) Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); Py_XDECREF(tv->evaluate_constraints); + Py_XDECREF(tv->default_value); + Py_XDECREF(tv->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -216,6 +282,8 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); + Py_VISIT(tv->default_value); + Py_VISIT(tv->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -227,6 +295,8 @@ typevar_clear(typevarobject *self) Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); Py_CLEAR(self->evaluate_constraints); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -266,6 +336,20 @@ typevar_bound(typevarobject *self, void *Py_UNUSED(ignored)) return bound; } +static PyObject * +typevar_default(typevarobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyObject * typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) { @@ -283,12 +367,14 @@ typevar_constraints(typevarobject *self, void *Py_UNUSED(ignored)) static PyGetSetDef typevar_getset[] = { {"__bound__", (getter)typevar_bound, NULL, NULL, NULL}, {"__constraints__", (getter)typevar_constraints, NULL, NULL, NULL}, + {"__default__", (getter)typevar_default, NULL, NULL, NULL}, {0} }; static typevarobject * typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, PyObject *constraints, PyObject *evaluate_constraints, + PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { @@ -305,6 +391,8 @@ typevar_alloc(PyObject *name, PyObject *bound, PyObject *evaluate_bound, tv->evaluate_bound = Py_XNewRef(evaluate_bound); tv->constraints = Py_XNewRef(constraints); tv->evaluate_constraints = Py_XNewRef(evaluate_constraints); + tv->default_value = Py_XNewRef(default_value); + tv->evaluate_default = NULL; tv->covariant = covariant; tv->contravariant = contravariant; @@ -328,6 +416,7 @@ typevar.__new__ as typevar_new name: object(subclass_of="&PyUnicode_Type") *constraints: object bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -337,9 +426,9 @@ Create a TypeVar. static PyObject * typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, - PyObject *bound, int covariant, int contravariant, - int infer_variance) -/*[clinic end generated code: output=1d200450ee99226d input=41ae33a916bfe76f]*/ + PyObject *bound, PyObject *default_value, int covariant, + int contravariant, int infer_variance) +/*[clinic end generated code: output=d2b248ff074eaab6 input=836f97f631d7293a]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, @@ -386,6 +475,7 @@ typevar_new_impl(PyTypeObject *type, PyObject *name, PyObject *constraints, PyObject *tv = (PyObject *)typevar_alloc(name, bound, NULL, constraints, NULL, + default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); @@ -410,6 +500,66 @@ typevar_typing_subst(typevarobject *self, PyObject *arg) return result; } +/*[clinic input] +typevar.__typing_prepare_subst__ as typevar_typing_prepare_subst + + alias: object + args: object + / + +[clinic start generated code]*/ + +static PyObject * +typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, + PyObject *args) +/*[clinic end generated code: output=82c3f4691e0ded22 input=201a750415d14ffb]*/ +{ + PyObject *params = PyObject_GetAttrString(alias, "__parameters__"); + if (params == NULL) { + return NULL; + } + Py_ssize_t i = PySequence_Index(params, (PyObject *)self); + if (i == -1) { + Py_DECREF(params); + return NULL; + } + Py_ssize_t args_len = PySequence_Length(args); + if (args_len == -1) { + Py_DECREF(params); + return NULL; + } + if (i < args_len) { + // We already have a value for our TypeVar + Py_DECREF(params); + return Py_NewRef(args); + } + else if (i == args_len) { + // If the TypeVar has a default, use it. + PyObject *dflt = typevar_default(self, NULL); + if (dflt == NULL) { + Py_DECREF(params); + return NULL; + } + if (dflt != &_Py_NoDefaultStruct) { + PyObject *new_args = PyTuple_Pack(1, dflt); + Py_DECREF(dflt); + if (new_args == NULL) { + Py_DECREF(params); + return NULL; + } + PyObject *result = PySequence_Concat(args, new_args); + Py_DECREF(params); + Py_DECREF(new_args); + return result; + } + } + Py_DECREF(params); + PyErr_Format(PyExc_TypeError, + "Too few arguments for %S; actual %d, expected at least %d", + alias, args_len, i + 1); + return NULL; +} + /*[clinic input] typevar.__reduce__ as typevar_reduce @@ -422,6 +572,23 @@ typevar_reduce_impl(typevarobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevar.has_default as typevar_has_default + +[clinic start generated code]*/ + +static PyObject * +typevar_has_default_impl(typevarobject *self) +/*[clinic end generated code: output=76bf0b8dc98b97dd input=31024aa030761cf6]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevar_mro_entries(PyObject *self, PyObject *args) { @@ -432,7 +599,9 @@ typevar_mro_entries(PyObject *self, PyObject *args) static PyMethodDef typevar_methods[] = { TYPEVAR_TYPING_SUBST_METHODDEF + TYPEVAR_TYPING_PREPARE_SUBST_METHODDEF TYPEVAR_REDUCE_METHODDEF + TYPEVAR_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevar_mro_entries, METH_O}, {0} }; @@ -457,12 +626,18 @@ variables::\n\ class StrOrBytesSequence[A: (str, bytes)]:\n\ ...\n\ \n\ +Type variables can also have defaults:\n\ +\n\ + class IntDefault[T = int]:\n\ + ...\n\ +\n\ However, if desired, reusable type variables can also be constructed\n\ manually, like so::\n\ \n\ T = TypeVar('T') # Can be anything\n\ S = TypeVar('S', bound=str) # Can be any subtype of str\n\ A = TypeVar('A', str, bytes) # Must be exactly str or bytes\n\ + D = TypeVar('D', default=int) # Defaults to int\n\ \n\ Type variables exist primarily for the benefit of static type\n\ checkers. They serve as the parameters for generic types as well\n\ @@ -739,6 +914,8 @@ paramspec_dealloc(PyObject *self) Py_DECREF(ps->name); Py_XDECREF(ps->bound); + Py_XDECREF(ps->default_value); + Py_XDECREF(ps->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -752,6 +929,8 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) Py_VISIT(Py_TYPE(self)); paramspecobject *ps = (paramspecobject *)self; Py_VISIT(ps->bound); + Py_VISIT(ps->default_value); + Py_VISIT(ps->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -760,6 +939,8 @@ static int paramspec_clear(paramspecobject *self) { Py_CLEAR(self->bound); + Py_CLEAR(self->default_value); + Py_CLEAR(self->evaluate_default); PyObject_ClearManagedDict((PyObject *)self); return 0; } @@ -800,14 +981,29 @@ paramspec_kwargs(PyObject *self, void *unused) return (PyObject *)paramspecattr_new(tp, self); } +static PyObject * +paramspec_default(paramspecobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + static PyGetSetDef paramspec_getset[] = { {"args", (getter)paramspec_args, NULL, PyDoc_STR("Represents positional arguments."), NULL}, {"kwargs", (getter)paramspec_kwargs, NULL, PyDoc_STR("Represents keyword arguments."), NULL}, + {"__default__", (getter)paramspec_default, NULL, "The default value for this ParamSpec.", NULL}, {0}, }; static paramspecobject * -paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, +paramspec_alloc(PyObject *name, PyObject *bound, PyObject *default_value, bool covariant, bool contravariant, bool infer_variance, PyObject *module) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.paramspec_type; @@ -820,6 +1016,8 @@ paramspec_alloc(PyObject *name, PyObject *bound, bool covariant, ps->covariant = covariant; ps->contravariant = contravariant; ps->infer_variance = infer_variance; + ps->default_value = Py_XNewRef(default_value); + ps->evaluate_default = NULL; _PyObject_GC_TRACK(ps); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)ps, "__module__", module) < 0) { @@ -837,6 +1035,7 @@ paramspec.__new__ as paramspec_new name: object(subclass_of="&PyUnicode_Type") * bound: object = None + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault covariant: bool = False contravariant: bool = False infer_variance: bool = False @@ -846,8 +1045,9 @@ Create a ParamSpec object. static PyObject * paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, - int covariant, int contravariant, int infer_variance) -/*[clinic end generated code: output=fd2daab79cba62da input=57c49c581979b952]*/ + PyObject *default_value, int covariant, int contravariant, + int infer_variance) +/*[clinic end generated code: output=47ca9d63fa5a094d input=495e1565bc067ab9]*/ { if (covariant && contravariant) { PyErr_SetString(PyExc_ValueError, "Bivariant types are not supported."); @@ -869,7 +1069,7 @@ paramspec_new_impl(PyTypeObject *type, PyObject *name, PyObject *bound, return NULL; } PyObject *ps = (PyObject *)paramspec_alloc( - name, bound, covariant, contravariant, infer_variance, module); + name, bound, default_value, covariant, contravariant, infer_variance, module); Py_XDECREF(bound); Py_DECREF(module); return ps; @@ -925,6 +1125,22 @@ paramspec_reduce_impl(paramspecobject *self) return Py_NewRef(self->name); } +/*[clinic input] +paramspec.has_default as paramspec_has_default + +[clinic start generated code]*/ + +static PyObject * +paramspec_has_default_impl(paramspecobject *self) +/*[clinic end generated code: output=daaae7467a6a4368 input=2112e97eeb76cd59]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * paramspec_mro_entries(PyObject *self, PyObject *args) { @@ -936,6 +1152,7 @@ paramspec_mro_entries(PyObject *self, PyObject *args) static PyMethodDef paramspec_methods[] = { PARAMSPEC_TYPING_SUBST_METHODDEF PARAMSPEC_TYPING_PREPARE_SUBST_METHODDEF + PARAMSPEC_HAS_DEFAULT_METHODDEF PARAMSPEC_REDUCE_METHODDEF {"__mro_entries__", paramspec_mro_entries, METH_O}, {0} @@ -950,10 +1167,17 @@ where the use of '**' creates a parameter specification::\n\ \n\ type IntFunc[**P] = Callable[P, int]\n\ \n\ +The following syntax creates a parameter specification that defaults\n\ +to a callable accepting two positional-only arguments of types int\n\ +and str:\n\ +\n\ + type IntFuncDefault[**P = (int, str)] = Callable[P, int]\n\ +\n\ For compatibility with Python 3.11 and earlier, ParamSpec objects\n\ can also be created as follows::\n\ \n\ P = ParamSpec('P')\n\ + DefaultP = ParamSpec('DefaultP', default=(int, str))\n\ \n\ Parameter specification variables exist primarily for the benefit of\n\ static type checkers. They are used to forward the parameter types of\n\ @@ -1021,6 +1245,8 @@ typevartuple_dealloc(PyObject *self) typevartupleobject *tvt = (typevartupleobject *)self; Py_DECREF(tvt->name); + Py_XDECREF(tvt->default_value); + Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); PyObject_ClearWeakRefs(self); @@ -1060,7 +1286,7 @@ static PyMemberDef typevartuple_members[] = { }; static typevartupleobject * -typevartuple_alloc(PyObject *name, PyObject *module) +typevartuple_alloc(PyObject *name, PyObject *module, PyObject *default_value) { PyTypeObject *tp = _PyInterpreterState_GET()->cached_objects.typevartuple_type; typevartupleobject *tvt = PyObject_GC_New(typevartupleobject, tp); @@ -1068,6 +1294,8 @@ typevartuple_alloc(PyObject *name, PyObject *module) return NULL; } tvt->name = Py_NewRef(name); + tvt->default_value = Py_XNewRef(default_value); + tvt->evaluate_default = NULL; _PyObject_GC_TRACK(tvt); if (module != NULL) { if (PyObject_SetAttrString((PyObject *)tvt, "__module__", module) < 0) { @@ -1083,19 +1311,22 @@ typevartuple_alloc(PyObject *name, PyObject *module) typevartuple.__new__ name: object(subclass_of="&PyUnicode_Type") + * + default as default_value: object(c_default="&_Py_NoDefaultStruct") = typing.NoDefault Create a new TypeVarTuple with the given name. [clinic start generated code]*/ static PyObject * -typevartuple_impl(PyTypeObject *type, PyObject *name) -/*[clinic end generated code: output=09d417a28f976202 input=00d28abcf1fc96bb]*/ +typevartuple_impl(PyTypeObject *type, PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=9d6b76dfe95aae51 input=e149739929a866d0]*/ { PyObject *module = caller(); if (module == NULL) { return NULL; } - PyObject *result = (PyObject *)typevartuple_alloc(name, module); + PyObject *result = (PyObject *)typevartuple_alloc(name, module, default_value); Py_DECREF(module); return result; } @@ -1148,6 +1379,23 @@ typevartuple_reduce_impl(typevartupleobject *self) return Py_NewRef(self->name); } + +/*[clinic input] +typevartuple.has_default as typevartuple_has_default + +[clinic start generated code]*/ + +static PyObject * +typevartuple_has_default_impl(typevartupleobject *self) +/*[clinic end generated code: output=4895f602f56a5e29 input=9ef3250ddb2c1851]*/ +{ + if (self->evaluate_default != NULL || + (self->default_value != &_Py_NoDefaultStruct && self->default_value != NULL)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} + static PyObject * typevartuple_mro_entries(PyObject *self, PyObject *args) { @@ -1160,6 +1408,8 @@ static int typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); + Py_VISIT(((typevartupleobject *)self)->default_value); + Py_VISIT(((typevartupleobject *)self)->evaluate_default); PyObject_VisitManagedDict(self, visit, arg); return 0; } @@ -1167,14 +1417,36 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) static int typevartuple_clear(PyObject *self) { + Py_CLEAR(((typevartupleobject *)self)->default_value); + Py_CLEAR(((typevartupleobject *)self)->evaluate_default); PyObject_ClearManagedDict(self); return 0; } +static PyObject * +typevartuple_default(typevartupleobject *self, void *unused) +{ + if (self->default_value != NULL) { + return Py_NewRef(self->default_value); + } + if (self->evaluate_default == NULL) { + return &_Py_NoDefaultStruct; + } + PyObject *default_value = PyObject_CallNoArgs(self->evaluate_default); + self->default_value = Py_XNewRef(default_value); + return default_value; +} + +static PyGetSetDef typevartuple_getset[] = { + {"__default__", (getter)typevartuple_default, NULL, "The default value for this TypeVarTuple.", NULL}, + {0}, +}; + static PyMethodDef typevartuple_methods[] = { TYPEVARTUPLE_TYPING_SUBST_METHODDEF TYPEVARTUPLE_TYPING_PREPARE_SUBST_METHODDEF TYPEVARTUPLE_REDUCE_METHODDEF + TYPEVARTUPLE_HAS_DEFAULT_METHODDEF {"__mro_entries__", typevartuple_mro_entries, METH_O}, {0} }; @@ -1190,10 +1462,15 @@ where a single '*' indicates a type variable tuple::\n\ def move_first_element_to_last[T, *Ts](tup: tuple[T, *Ts]) -> tuple[*Ts, T]:\n\ return (*tup[1:], tup[0])\n\ \n\ +Type variables tuples can have default values:\n\ +\n\ + type AliasWithDefault[*Ts = (str, int)] = tuple[*Ts]\n\ +\n\ For compatibility with Python 3.11 and earlier, TypeVarTuple objects\n\ can also be created as follows::\n\ \n\ Ts = TypeVarTuple('Ts') # Can be given any name\n\ + DefaultTs = TypeVarTuple('Ts', default=(str, int))\n\ \n\ Just as a TypeVar (type variable) is a placeholder for a single type,\n\ a TypeVarTuple is a placeholder for an *arbitrary* number of types. For\n\ @@ -1218,6 +1495,7 @@ PyType_Slot typevartuple_slots[] = { {Py_tp_doc, (void *)typevartuple_doc}, {Py_tp_members, typevartuple_members}, {Py_tp_methods, typevartuple_methods}, + {Py_tp_getset, typevartuple_getset}, {Py_tp_new, typevartuple}, {Py_tp_iter, typevartuple_iter}, {Py_tp_repr, typevartuple_repr}, @@ -1241,21 +1519,21 @@ PyObject * _Py_make_typevar(PyObject *name, PyObject *evaluate_bound, PyObject *evaluate_constraints) { return (PyObject *)typevar_alloc(name, NULL, evaluate_bound, NULL, evaluate_constraints, - false, false, true, NULL); + NULL, false, false, true, NULL); } PyObject * _Py_make_paramspec(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)paramspec_alloc(v, NULL, false, false, true, NULL); + return (PyObject *)paramspec_alloc(v, NULL, NULL, false, false, true, NULL); } PyObject * _Py_make_typevartuple(PyThreadState *Py_UNUSED(ignored), PyObject *v) { assert(PyUnicode_Check(v)); - return (PyObject *)typevartuple_alloc(v, NULL); + return (PyObject *)typevartuple_alloc(v, NULL, NULL); } static void @@ -1687,3 +1965,24 @@ void _Py_clear_generic_types(PyInterpreterState *interp) Py_CLEAR(interp->cached_objects.paramspecargs_type); Py_CLEAR(interp->cached_objects.paramspeckwargs_type); } + +PyObject * +_Py_set_typeparam_default(PyThreadState *ts, PyObject *typeparam, PyObject *evaluate_default) +{ + if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevar_type)) { + Py_XSETREF(((typevarobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.paramspec_type)) { + Py_XSETREF(((paramspecobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else if (Py_IS_TYPE(typeparam, ts->interp->cached_objects.typevartuple_type)) { + Py_XSETREF(((typevartupleobject *)typeparam)->evaluate_default, Py_NewRef(evaluate_default)); + return Py_NewRef(typeparam); + } + else { + PyErr_Format(PyExc_TypeError, "Expected a type param, got %R", typeparam); + return NULL; + } +} diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 0d154867276c363..80776ffe4493930 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -145,8 +145,8 @@ module Python type_ignore = TypeIgnore(int lineno, string tag) - type_param = TypeVar(identifier name, expr? bound) - | ParamSpec(identifier name) - | TypeVarTuple(identifier name) + type_param = TypeVar(identifier name, expr? bound, expr? default_value) + | ParamSpec(identifier name, expr? default_value) + | TypeVarTuple(identifier name, expr? default_value) attributes (int lineno, int col_offset, int end_lineno, int end_col_offset) } diff --git a/Parser/parser.c b/Parser/parser.c index 0715e9775a8f065..e34fcada15ebf50 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -186,431 +186,433 @@ static char *soft_keywords[] = { #define type_param_seq_type 1099 #define type_param_type 1100 #define type_param_bound_type 1101 -#define expressions_type 1102 -#define expression_type 1103 -#define yield_expr_type 1104 -#define star_expressions_type 1105 -#define star_expression_type 1106 -#define star_named_expressions_type 1107 -#define star_named_expression_type 1108 -#define assignment_expression_type 1109 -#define named_expression_type 1110 -#define disjunction_type 1111 -#define conjunction_type 1112 -#define inversion_type 1113 -#define comparison_type 1114 -#define compare_op_bitwise_or_pair_type 1115 -#define eq_bitwise_or_type 1116 -#define noteq_bitwise_or_type 1117 -#define lte_bitwise_or_type 1118 -#define lt_bitwise_or_type 1119 -#define gte_bitwise_or_type 1120 -#define gt_bitwise_or_type 1121 -#define notin_bitwise_or_type 1122 -#define in_bitwise_or_type 1123 -#define isnot_bitwise_or_type 1124 -#define is_bitwise_or_type 1125 -#define bitwise_or_type 1126 // Left-recursive -#define bitwise_xor_type 1127 // Left-recursive -#define bitwise_and_type 1128 // Left-recursive -#define shift_expr_type 1129 // Left-recursive -#define sum_type 1130 // Left-recursive -#define term_type 1131 // Left-recursive -#define factor_type 1132 -#define power_type 1133 -#define await_primary_type 1134 -#define primary_type 1135 // Left-recursive -#define slices_type 1136 -#define slice_type 1137 -#define atom_type 1138 -#define group_type 1139 -#define lambdef_type 1140 -#define lambda_params_type 1141 -#define lambda_parameters_type 1142 -#define lambda_slash_no_default_type 1143 -#define lambda_slash_with_default_type 1144 -#define lambda_star_etc_type 1145 -#define lambda_kwds_type 1146 -#define lambda_param_no_default_type 1147 -#define lambda_param_with_default_type 1148 -#define lambda_param_maybe_default_type 1149 -#define lambda_param_type 1150 -#define fstring_middle_type 1151 -#define fstring_replacement_field_type 1152 -#define fstring_conversion_type 1153 -#define fstring_full_format_spec_type 1154 -#define fstring_format_spec_type 1155 -#define fstring_type 1156 -#define string_type 1157 -#define strings_type 1158 -#define list_type 1159 -#define tuple_type 1160 -#define set_type 1161 -#define dict_type 1162 -#define double_starred_kvpairs_type 1163 -#define double_starred_kvpair_type 1164 -#define kvpair_type 1165 -#define for_if_clauses_type 1166 -#define for_if_clause_type 1167 -#define listcomp_type 1168 -#define setcomp_type 1169 -#define genexp_type 1170 -#define dictcomp_type 1171 -#define arguments_type 1172 -#define args_type 1173 -#define kwargs_type 1174 -#define starred_expression_type 1175 -#define kwarg_or_starred_type 1176 -#define kwarg_or_double_starred_type 1177 -#define star_targets_type 1178 -#define star_targets_list_seq_type 1179 -#define star_targets_tuple_seq_type 1180 -#define star_target_type 1181 -#define target_with_star_atom_type 1182 -#define star_atom_type 1183 -#define single_target_type 1184 -#define single_subscript_attribute_target_type 1185 -#define t_primary_type 1186 // Left-recursive -#define t_lookahead_type 1187 -#define del_targets_type 1188 -#define del_target_type 1189 -#define del_t_atom_type 1190 -#define type_expressions_type 1191 -#define func_type_comment_type 1192 -#define invalid_arguments_type 1193 -#define invalid_kwarg_type 1194 -#define expression_without_invalid_type 1195 -#define invalid_legacy_expression_type 1196 -#define invalid_expression_type 1197 -#define invalid_named_expression_type 1198 -#define invalid_assignment_type 1199 -#define invalid_ann_assign_target_type 1200 -#define invalid_del_stmt_type 1201 -#define invalid_block_type 1202 -#define invalid_comprehension_type 1203 -#define invalid_dict_comprehension_type 1204 -#define invalid_parameters_type 1205 -#define invalid_default_type 1206 -#define invalid_star_etc_type 1207 -#define invalid_kwds_type 1208 -#define invalid_parameters_helper_type 1209 -#define invalid_lambda_parameters_type 1210 -#define invalid_lambda_parameters_helper_type 1211 -#define invalid_lambda_star_etc_type 1212 -#define invalid_lambda_kwds_type 1213 -#define invalid_double_type_comments_type 1214 -#define invalid_with_item_type 1215 -#define invalid_for_target_type 1216 -#define invalid_group_type 1217 -#define invalid_import_type 1218 -#define invalid_import_from_targets_type 1219 -#define invalid_compound_stmt_type 1220 -#define invalid_with_stmt_type 1221 -#define invalid_with_stmt_indent_type 1222 -#define invalid_try_stmt_type 1223 -#define invalid_except_stmt_type 1224 -#define invalid_finally_stmt_type 1225 -#define invalid_except_stmt_indent_type 1226 -#define invalid_except_star_stmt_indent_type 1227 -#define invalid_match_stmt_type 1228 -#define invalid_case_block_type 1229 -#define invalid_as_pattern_type 1230 -#define invalid_class_pattern_type 1231 -#define invalid_class_argument_pattern_type 1232 -#define invalid_if_stmt_type 1233 -#define invalid_elif_stmt_type 1234 -#define invalid_else_stmt_type 1235 -#define invalid_while_stmt_type 1236 -#define invalid_for_stmt_type 1237 -#define invalid_def_raw_type 1238 -#define invalid_class_def_raw_type 1239 -#define invalid_double_starred_kvpairs_type 1240 -#define invalid_kvpair_type 1241 -#define invalid_starred_expression_type 1242 -#define invalid_replacement_field_type 1243 -#define invalid_conversion_character_type 1244 -#define invalid_arithmetic_type 1245 -#define invalid_factor_type 1246 -#define _loop0_1_type 1247 -#define _loop0_2_type 1248 -#define _loop1_3_type 1249 -#define _loop0_5_type 1250 -#define _gather_4_type 1251 -#define _tmp_6_type 1252 -#define _tmp_7_type 1253 -#define _tmp_8_type 1254 -#define _tmp_9_type 1255 -#define _tmp_10_type 1256 -#define _tmp_11_type 1257 -#define _tmp_12_type 1258 -#define _tmp_13_type 1259 -#define _loop1_14_type 1260 -#define _tmp_15_type 1261 -#define _tmp_16_type 1262 -#define _tmp_17_type 1263 -#define _loop0_19_type 1264 -#define _gather_18_type 1265 -#define _loop0_21_type 1266 -#define _gather_20_type 1267 -#define _tmp_22_type 1268 -#define _tmp_23_type 1269 -#define _loop0_24_type 1270 -#define _loop1_25_type 1271 -#define _loop0_27_type 1272 -#define _gather_26_type 1273 -#define _tmp_28_type 1274 -#define _loop0_30_type 1275 -#define _gather_29_type 1276 -#define _tmp_31_type 1277 -#define _loop1_32_type 1278 -#define _tmp_33_type 1279 -#define _tmp_34_type 1280 -#define _tmp_35_type 1281 -#define _loop0_36_type 1282 -#define _loop0_37_type 1283 -#define _loop0_38_type 1284 -#define _loop1_39_type 1285 -#define _loop0_40_type 1286 -#define _loop1_41_type 1287 -#define _loop1_42_type 1288 -#define _loop1_43_type 1289 -#define _loop0_44_type 1290 -#define _loop1_45_type 1291 -#define _loop0_46_type 1292 -#define _loop1_47_type 1293 -#define _loop0_48_type 1294 -#define _loop0_49_type 1295 -#define _loop1_50_type 1296 -#define _loop0_52_type 1297 -#define _gather_51_type 1298 -#define _loop0_54_type 1299 -#define _gather_53_type 1300 -#define _loop0_56_type 1301 -#define _gather_55_type 1302 -#define _loop0_58_type 1303 -#define _gather_57_type 1304 -#define _tmp_59_type 1305 -#define _loop1_60_type 1306 -#define _loop1_61_type 1307 -#define _tmp_62_type 1308 -#define _tmp_63_type 1309 -#define _loop1_64_type 1310 -#define _loop0_66_type 1311 -#define _gather_65_type 1312 -#define _tmp_67_type 1313 -#define _tmp_68_type 1314 -#define _tmp_69_type 1315 -#define _tmp_70_type 1316 -#define _loop0_72_type 1317 -#define _gather_71_type 1318 -#define _loop0_74_type 1319 -#define _gather_73_type 1320 -#define _tmp_75_type 1321 -#define _loop0_77_type 1322 -#define _gather_76_type 1323 -#define _loop0_79_type 1324 -#define _gather_78_type 1325 -#define _loop0_81_type 1326 -#define _gather_80_type 1327 -#define _loop1_82_type 1328 -#define _loop1_83_type 1329 -#define _loop0_85_type 1330 -#define _gather_84_type 1331 -#define _loop1_86_type 1332 -#define _loop1_87_type 1333 -#define _loop1_88_type 1334 -#define _tmp_89_type 1335 -#define _loop0_91_type 1336 -#define _gather_90_type 1337 -#define _tmp_92_type 1338 -#define _tmp_93_type 1339 -#define _tmp_94_type 1340 -#define _tmp_95_type 1341 -#define _tmp_96_type 1342 -#define _tmp_97_type 1343 -#define _loop0_98_type 1344 -#define _loop0_99_type 1345 -#define _loop0_100_type 1346 -#define _loop1_101_type 1347 -#define _loop0_102_type 1348 -#define _loop1_103_type 1349 -#define _loop1_104_type 1350 -#define _loop1_105_type 1351 -#define _loop0_106_type 1352 -#define _loop1_107_type 1353 -#define _loop0_108_type 1354 -#define _loop1_109_type 1355 -#define _loop0_110_type 1356 -#define _loop1_111_type 1357 -#define _loop0_112_type 1358 -#define _loop0_113_type 1359 -#define _loop1_114_type 1360 -#define _tmp_115_type 1361 -#define _loop0_117_type 1362 -#define _gather_116_type 1363 -#define _loop1_118_type 1364 -#define _loop0_119_type 1365 -#define _loop0_120_type 1366 -#define _tmp_121_type 1367 -#define _tmp_122_type 1368 -#define _loop0_124_type 1369 -#define _gather_123_type 1370 -#define _tmp_125_type 1371 -#define _loop0_127_type 1372 -#define _gather_126_type 1373 -#define _loop0_129_type 1374 -#define _gather_128_type 1375 -#define _loop0_131_type 1376 -#define _gather_130_type 1377 -#define _loop0_133_type 1378 -#define _gather_132_type 1379 -#define _loop0_134_type 1380 -#define _loop0_136_type 1381 -#define _gather_135_type 1382 -#define _loop1_137_type 1383 -#define _tmp_138_type 1384 -#define _loop0_140_type 1385 -#define _gather_139_type 1386 -#define _loop0_142_type 1387 -#define _gather_141_type 1388 -#define _loop0_144_type 1389 -#define _gather_143_type 1390 -#define _loop0_146_type 1391 -#define _gather_145_type 1392 -#define _loop0_148_type 1393 -#define _gather_147_type 1394 -#define _tmp_149_type 1395 -#define _tmp_150_type 1396 -#define _loop0_152_type 1397 -#define _gather_151_type 1398 -#define _tmp_153_type 1399 -#define _tmp_154_type 1400 -#define _tmp_155_type 1401 -#define _tmp_156_type 1402 -#define _tmp_157_type 1403 -#define _tmp_158_type 1404 -#define _tmp_159_type 1405 -#define _tmp_160_type 1406 -#define _tmp_161_type 1407 -#define _tmp_162_type 1408 -#define _loop0_163_type 1409 -#define _loop0_164_type 1410 -#define _loop0_165_type 1411 -#define _tmp_166_type 1412 -#define _tmp_167_type 1413 -#define _tmp_168_type 1414 -#define _tmp_169_type 1415 -#define _loop0_170_type 1416 -#define _loop0_171_type 1417 -#define _loop0_172_type 1418 -#define _loop1_173_type 1419 -#define _tmp_174_type 1420 -#define _loop0_175_type 1421 -#define _tmp_176_type 1422 -#define _loop0_177_type 1423 -#define _loop1_178_type 1424 -#define _tmp_179_type 1425 -#define _tmp_180_type 1426 -#define _tmp_181_type 1427 -#define _loop0_182_type 1428 -#define _tmp_183_type 1429 -#define _tmp_184_type 1430 -#define _loop1_185_type 1431 -#define _tmp_186_type 1432 -#define _loop0_187_type 1433 -#define _loop0_188_type 1434 -#define _loop0_189_type 1435 -#define _loop0_191_type 1436 -#define _gather_190_type 1437 -#define _tmp_192_type 1438 -#define _loop0_193_type 1439 -#define _tmp_194_type 1440 -#define _loop0_195_type 1441 -#define _loop1_196_type 1442 -#define _loop1_197_type 1443 -#define _tmp_198_type 1444 -#define _tmp_199_type 1445 -#define _loop0_200_type 1446 -#define _tmp_201_type 1447 -#define _tmp_202_type 1448 -#define _tmp_203_type 1449 -#define _loop0_205_type 1450 -#define _gather_204_type 1451 -#define _loop0_207_type 1452 -#define _gather_206_type 1453 -#define _loop0_209_type 1454 -#define _gather_208_type 1455 -#define _loop0_211_type 1456 -#define _gather_210_type 1457 -#define _loop0_213_type 1458 -#define _gather_212_type 1459 -#define _tmp_214_type 1460 -#define _loop0_215_type 1461 -#define _loop1_216_type 1462 -#define _tmp_217_type 1463 -#define _loop0_218_type 1464 -#define _loop1_219_type 1465 -#define _tmp_220_type 1466 -#define _tmp_221_type 1467 -#define _tmp_222_type 1468 -#define _tmp_223_type 1469 -#define _tmp_224_type 1470 -#define _tmp_225_type 1471 -#define _tmp_226_type 1472 -#define _tmp_227_type 1473 -#define _tmp_228_type 1474 -#define _tmp_229_type 1475 -#define _loop0_231_type 1476 -#define _gather_230_type 1477 -#define _tmp_232_type 1478 -#define _tmp_233_type 1479 -#define _tmp_234_type 1480 -#define _tmp_235_type 1481 -#define _tmp_236_type 1482 -#define _tmp_237_type 1483 -#define _tmp_238_type 1484 -#define _loop0_239_type 1485 -#define _tmp_240_type 1486 -#define _tmp_241_type 1487 -#define _tmp_242_type 1488 -#define _tmp_243_type 1489 -#define _tmp_244_type 1490 -#define _tmp_245_type 1491 -#define _tmp_246_type 1492 -#define _tmp_247_type 1493 -#define _tmp_248_type 1494 -#define _tmp_249_type 1495 -#define _tmp_250_type 1496 -#define _tmp_251_type 1497 -#define _tmp_252_type 1498 -#define _tmp_253_type 1499 -#define _tmp_254_type 1500 -#define _tmp_255_type 1501 -#define _loop0_256_type 1502 -#define _tmp_257_type 1503 -#define _tmp_258_type 1504 -#define _tmp_259_type 1505 -#define _tmp_260_type 1506 -#define _tmp_261_type 1507 -#define _tmp_262_type 1508 -#define _tmp_263_type 1509 -#define _tmp_264_type 1510 -#define _tmp_265_type 1511 -#define _tmp_266_type 1512 -#define _tmp_267_type 1513 -#define _tmp_268_type 1514 -#define _tmp_269_type 1515 -#define _tmp_270_type 1516 -#define _tmp_271_type 1517 -#define _tmp_272_type 1518 -#define _loop0_274_type 1519 -#define _gather_273_type 1520 -#define _tmp_275_type 1521 -#define _tmp_276_type 1522 -#define _tmp_277_type 1523 -#define _tmp_278_type 1524 -#define _tmp_279_type 1525 -#define _tmp_280_type 1526 +#define type_param_default_type 1102 +#define type_param_starred_default_type 1103 +#define expressions_type 1104 +#define expression_type 1105 +#define yield_expr_type 1106 +#define star_expressions_type 1107 +#define star_expression_type 1108 +#define star_named_expressions_type 1109 +#define star_named_expression_type 1110 +#define assignment_expression_type 1111 +#define named_expression_type 1112 +#define disjunction_type 1113 +#define conjunction_type 1114 +#define inversion_type 1115 +#define comparison_type 1116 +#define compare_op_bitwise_or_pair_type 1117 +#define eq_bitwise_or_type 1118 +#define noteq_bitwise_or_type 1119 +#define lte_bitwise_or_type 1120 +#define lt_bitwise_or_type 1121 +#define gte_bitwise_or_type 1122 +#define gt_bitwise_or_type 1123 +#define notin_bitwise_or_type 1124 +#define in_bitwise_or_type 1125 +#define isnot_bitwise_or_type 1126 +#define is_bitwise_or_type 1127 +#define bitwise_or_type 1128 // Left-recursive +#define bitwise_xor_type 1129 // Left-recursive +#define bitwise_and_type 1130 // Left-recursive +#define shift_expr_type 1131 // Left-recursive +#define sum_type 1132 // Left-recursive +#define term_type 1133 // Left-recursive +#define factor_type 1134 +#define power_type 1135 +#define await_primary_type 1136 +#define primary_type 1137 // Left-recursive +#define slices_type 1138 +#define slice_type 1139 +#define atom_type 1140 +#define group_type 1141 +#define lambdef_type 1142 +#define lambda_params_type 1143 +#define lambda_parameters_type 1144 +#define lambda_slash_no_default_type 1145 +#define lambda_slash_with_default_type 1146 +#define lambda_star_etc_type 1147 +#define lambda_kwds_type 1148 +#define lambda_param_no_default_type 1149 +#define lambda_param_with_default_type 1150 +#define lambda_param_maybe_default_type 1151 +#define lambda_param_type 1152 +#define fstring_middle_type 1153 +#define fstring_replacement_field_type 1154 +#define fstring_conversion_type 1155 +#define fstring_full_format_spec_type 1156 +#define fstring_format_spec_type 1157 +#define fstring_type 1158 +#define string_type 1159 +#define strings_type 1160 +#define list_type 1161 +#define tuple_type 1162 +#define set_type 1163 +#define dict_type 1164 +#define double_starred_kvpairs_type 1165 +#define double_starred_kvpair_type 1166 +#define kvpair_type 1167 +#define for_if_clauses_type 1168 +#define for_if_clause_type 1169 +#define listcomp_type 1170 +#define setcomp_type 1171 +#define genexp_type 1172 +#define dictcomp_type 1173 +#define arguments_type 1174 +#define args_type 1175 +#define kwargs_type 1176 +#define starred_expression_type 1177 +#define kwarg_or_starred_type 1178 +#define kwarg_or_double_starred_type 1179 +#define star_targets_type 1180 +#define star_targets_list_seq_type 1181 +#define star_targets_tuple_seq_type 1182 +#define star_target_type 1183 +#define target_with_star_atom_type 1184 +#define star_atom_type 1185 +#define single_target_type 1186 +#define single_subscript_attribute_target_type 1187 +#define t_primary_type 1188 // Left-recursive +#define t_lookahead_type 1189 +#define del_targets_type 1190 +#define del_target_type 1191 +#define del_t_atom_type 1192 +#define type_expressions_type 1193 +#define func_type_comment_type 1194 +#define invalid_arguments_type 1195 +#define invalid_kwarg_type 1196 +#define expression_without_invalid_type 1197 +#define invalid_legacy_expression_type 1198 +#define invalid_expression_type 1199 +#define invalid_named_expression_type 1200 +#define invalid_assignment_type 1201 +#define invalid_ann_assign_target_type 1202 +#define invalid_del_stmt_type 1203 +#define invalid_block_type 1204 +#define invalid_comprehension_type 1205 +#define invalid_dict_comprehension_type 1206 +#define invalid_parameters_type 1207 +#define invalid_default_type 1208 +#define invalid_star_etc_type 1209 +#define invalid_kwds_type 1210 +#define invalid_parameters_helper_type 1211 +#define invalid_lambda_parameters_type 1212 +#define invalid_lambda_parameters_helper_type 1213 +#define invalid_lambda_star_etc_type 1214 +#define invalid_lambda_kwds_type 1215 +#define invalid_double_type_comments_type 1216 +#define invalid_with_item_type 1217 +#define invalid_for_target_type 1218 +#define invalid_group_type 1219 +#define invalid_import_type 1220 +#define invalid_import_from_targets_type 1221 +#define invalid_compound_stmt_type 1222 +#define invalid_with_stmt_type 1223 +#define invalid_with_stmt_indent_type 1224 +#define invalid_try_stmt_type 1225 +#define invalid_except_stmt_type 1226 +#define invalid_finally_stmt_type 1227 +#define invalid_except_stmt_indent_type 1228 +#define invalid_except_star_stmt_indent_type 1229 +#define invalid_match_stmt_type 1230 +#define invalid_case_block_type 1231 +#define invalid_as_pattern_type 1232 +#define invalid_class_pattern_type 1233 +#define invalid_class_argument_pattern_type 1234 +#define invalid_if_stmt_type 1235 +#define invalid_elif_stmt_type 1236 +#define invalid_else_stmt_type 1237 +#define invalid_while_stmt_type 1238 +#define invalid_for_stmt_type 1239 +#define invalid_def_raw_type 1240 +#define invalid_class_def_raw_type 1241 +#define invalid_double_starred_kvpairs_type 1242 +#define invalid_kvpair_type 1243 +#define invalid_starred_expression_type 1244 +#define invalid_replacement_field_type 1245 +#define invalid_conversion_character_type 1246 +#define invalid_arithmetic_type 1247 +#define invalid_factor_type 1248 +#define _loop0_1_type 1249 +#define _loop0_2_type 1250 +#define _loop1_3_type 1251 +#define _loop0_5_type 1252 +#define _gather_4_type 1253 +#define _tmp_6_type 1254 +#define _tmp_7_type 1255 +#define _tmp_8_type 1256 +#define _tmp_9_type 1257 +#define _tmp_10_type 1258 +#define _tmp_11_type 1259 +#define _tmp_12_type 1260 +#define _tmp_13_type 1261 +#define _loop1_14_type 1262 +#define _tmp_15_type 1263 +#define _tmp_16_type 1264 +#define _tmp_17_type 1265 +#define _loop0_19_type 1266 +#define _gather_18_type 1267 +#define _loop0_21_type 1268 +#define _gather_20_type 1269 +#define _tmp_22_type 1270 +#define _tmp_23_type 1271 +#define _loop0_24_type 1272 +#define _loop1_25_type 1273 +#define _loop0_27_type 1274 +#define _gather_26_type 1275 +#define _tmp_28_type 1276 +#define _loop0_30_type 1277 +#define _gather_29_type 1278 +#define _tmp_31_type 1279 +#define _loop1_32_type 1280 +#define _tmp_33_type 1281 +#define _tmp_34_type 1282 +#define _tmp_35_type 1283 +#define _loop0_36_type 1284 +#define _loop0_37_type 1285 +#define _loop0_38_type 1286 +#define _loop1_39_type 1287 +#define _loop0_40_type 1288 +#define _loop1_41_type 1289 +#define _loop1_42_type 1290 +#define _loop1_43_type 1291 +#define _loop0_44_type 1292 +#define _loop1_45_type 1293 +#define _loop0_46_type 1294 +#define _loop1_47_type 1295 +#define _loop0_48_type 1296 +#define _loop0_49_type 1297 +#define _loop1_50_type 1298 +#define _loop0_52_type 1299 +#define _gather_51_type 1300 +#define _loop0_54_type 1301 +#define _gather_53_type 1302 +#define _loop0_56_type 1303 +#define _gather_55_type 1304 +#define _loop0_58_type 1305 +#define _gather_57_type 1306 +#define _tmp_59_type 1307 +#define _loop1_60_type 1308 +#define _loop1_61_type 1309 +#define _tmp_62_type 1310 +#define _tmp_63_type 1311 +#define _loop1_64_type 1312 +#define _loop0_66_type 1313 +#define _gather_65_type 1314 +#define _tmp_67_type 1315 +#define _tmp_68_type 1316 +#define _tmp_69_type 1317 +#define _tmp_70_type 1318 +#define _loop0_72_type 1319 +#define _gather_71_type 1320 +#define _loop0_74_type 1321 +#define _gather_73_type 1322 +#define _tmp_75_type 1323 +#define _loop0_77_type 1324 +#define _gather_76_type 1325 +#define _loop0_79_type 1326 +#define _gather_78_type 1327 +#define _loop0_81_type 1328 +#define _gather_80_type 1329 +#define _loop1_82_type 1330 +#define _loop1_83_type 1331 +#define _loop0_85_type 1332 +#define _gather_84_type 1333 +#define _loop1_86_type 1334 +#define _loop1_87_type 1335 +#define _loop1_88_type 1336 +#define _tmp_89_type 1337 +#define _loop0_91_type 1338 +#define _gather_90_type 1339 +#define _tmp_92_type 1340 +#define _tmp_93_type 1341 +#define _tmp_94_type 1342 +#define _tmp_95_type 1343 +#define _tmp_96_type 1344 +#define _tmp_97_type 1345 +#define _loop0_98_type 1346 +#define _loop0_99_type 1347 +#define _loop0_100_type 1348 +#define _loop1_101_type 1349 +#define _loop0_102_type 1350 +#define _loop1_103_type 1351 +#define _loop1_104_type 1352 +#define _loop1_105_type 1353 +#define _loop0_106_type 1354 +#define _loop1_107_type 1355 +#define _loop0_108_type 1356 +#define _loop1_109_type 1357 +#define _loop0_110_type 1358 +#define _loop1_111_type 1359 +#define _loop0_112_type 1360 +#define _loop0_113_type 1361 +#define _loop1_114_type 1362 +#define _tmp_115_type 1363 +#define _loop0_117_type 1364 +#define _gather_116_type 1365 +#define _loop1_118_type 1366 +#define _loop0_119_type 1367 +#define _loop0_120_type 1368 +#define _tmp_121_type 1369 +#define _tmp_122_type 1370 +#define _loop0_124_type 1371 +#define _gather_123_type 1372 +#define _tmp_125_type 1373 +#define _loop0_127_type 1374 +#define _gather_126_type 1375 +#define _loop0_129_type 1376 +#define _gather_128_type 1377 +#define _loop0_131_type 1378 +#define _gather_130_type 1379 +#define _loop0_133_type 1380 +#define _gather_132_type 1381 +#define _loop0_134_type 1382 +#define _loop0_136_type 1383 +#define _gather_135_type 1384 +#define _loop1_137_type 1385 +#define _tmp_138_type 1386 +#define _loop0_140_type 1387 +#define _gather_139_type 1388 +#define _loop0_142_type 1389 +#define _gather_141_type 1390 +#define _loop0_144_type 1391 +#define _gather_143_type 1392 +#define _loop0_146_type 1393 +#define _gather_145_type 1394 +#define _loop0_148_type 1395 +#define _gather_147_type 1396 +#define _tmp_149_type 1397 +#define _tmp_150_type 1398 +#define _loop0_152_type 1399 +#define _gather_151_type 1400 +#define _tmp_153_type 1401 +#define _tmp_154_type 1402 +#define _tmp_155_type 1403 +#define _tmp_156_type 1404 +#define _tmp_157_type 1405 +#define _tmp_158_type 1406 +#define _tmp_159_type 1407 +#define _tmp_160_type 1408 +#define _tmp_161_type 1409 +#define _tmp_162_type 1410 +#define _loop0_163_type 1411 +#define _loop0_164_type 1412 +#define _loop0_165_type 1413 +#define _tmp_166_type 1414 +#define _tmp_167_type 1415 +#define _tmp_168_type 1416 +#define _tmp_169_type 1417 +#define _loop0_170_type 1418 +#define _loop0_171_type 1419 +#define _loop0_172_type 1420 +#define _loop1_173_type 1421 +#define _tmp_174_type 1422 +#define _loop0_175_type 1423 +#define _tmp_176_type 1424 +#define _loop0_177_type 1425 +#define _loop1_178_type 1426 +#define _tmp_179_type 1427 +#define _tmp_180_type 1428 +#define _tmp_181_type 1429 +#define _loop0_182_type 1430 +#define _tmp_183_type 1431 +#define _tmp_184_type 1432 +#define _loop1_185_type 1433 +#define _tmp_186_type 1434 +#define _loop0_187_type 1435 +#define _loop0_188_type 1436 +#define _loop0_189_type 1437 +#define _loop0_191_type 1438 +#define _gather_190_type 1439 +#define _tmp_192_type 1440 +#define _loop0_193_type 1441 +#define _tmp_194_type 1442 +#define _loop0_195_type 1443 +#define _loop1_196_type 1444 +#define _loop1_197_type 1445 +#define _tmp_198_type 1446 +#define _tmp_199_type 1447 +#define _loop0_200_type 1448 +#define _tmp_201_type 1449 +#define _tmp_202_type 1450 +#define _tmp_203_type 1451 +#define _loop0_205_type 1452 +#define _gather_204_type 1453 +#define _loop0_207_type 1454 +#define _gather_206_type 1455 +#define _loop0_209_type 1456 +#define _gather_208_type 1457 +#define _loop0_211_type 1458 +#define _gather_210_type 1459 +#define _loop0_213_type 1460 +#define _gather_212_type 1461 +#define _tmp_214_type 1462 +#define _loop0_215_type 1463 +#define _loop1_216_type 1464 +#define _tmp_217_type 1465 +#define _loop0_218_type 1466 +#define _loop1_219_type 1467 +#define _tmp_220_type 1468 +#define _tmp_221_type 1469 +#define _tmp_222_type 1470 +#define _tmp_223_type 1471 +#define _tmp_224_type 1472 +#define _tmp_225_type 1473 +#define _tmp_226_type 1474 +#define _tmp_227_type 1475 +#define _tmp_228_type 1476 +#define _tmp_229_type 1477 +#define _loop0_231_type 1478 +#define _gather_230_type 1479 +#define _tmp_232_type 1480 +#define _tmp_233_type 1481 +#define _tmp_234_type 1482 +#define _tmp_235_type 1483 +#define _tmp_236_type 1484 +#define _tmp_237_type 1485 +#define _tmp_238_type 1486 +#define _loop0_239_type 1487 +#define _tmp_240_type 1488 +#define _tmp_241_type 1489 +#define _tmp_242_type 1490 +#define _tmp_243_type 1491 +#define _tmp_244_type 1492 +#define _tmp_245_type 1493 +#define _tmp_246_type 1494 +#define _tmp_247_type 1495 +#define _tmp_248_type 1496 +#define _tmp_249_type 1497 +#define _tmp_250_type 1498 +#define _tmp_251_type 1499 +#define _tmp_252_type 1500 +#define _tmp_253_type 1501 +#define _tmp_254_type 1502 +#define _tmp_255_type 1503 +#define _loop0_256_type 1504 +#define _tmp_257_type 1505 +#define _tmp_258_type 1506 +#define _tmp_259_type 1507 +#define _tmp_260_type 1508 +#define _tmp_261_type 1509 +#define _tmp_262_type 1510 +#define _tmp_263_type 1511 +#define _tmp_264_type 1512 +#define _tmp_265_type 1513 +#define _tmp_266_type 1514 +#define _tmp_267_type 1515 +#define _tmp_268_type 1516 +#define _tmp_269_type 1517 +#define _tmp_270_type 1518 +#define _tmp_271_type 1519 +#define _tmp_272_type 1520 +#define _loop0_274_type 1521 +#define _gather_273_type 1522 +#define _tmp_275_type 1523 +#define _tmp_276_type 1524 +#define _tmp_277_type 1525 +#define _tmp_278_type 1526 +#define _tmp_279_type 1527 +#define _tmp_280_type 1528 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -714,6 +716,8 @@ static asdl_type_param_seq* type_params_rule(Parser *p); static asdl_type_param_seq* type_param_seq_rule(Parser *p); static type_param_ty type_param_rule(Parser *p); static expr_ty type_param_bound_rule(Parser *p); +static expr_ty type_param_default_rule(Parser *p); +static expr_ty type_param_starred_default_rule(Parser *p); static expr_ty expressions_rule(Parser *p); static expr_ty expression_rule(Parser *p); static expr_ty yield_expr_rule(Parser *p); @@ -10646,11 +10650,11 @@ type_param_seq_rule(Parser *p) } // type_param: -// | NAME type_param_bound? +// | NAME type_param_bound? type_param_default? // | '*' NAME ':' expression -// | '*' NAME +// | '*' NAME type_param_starred_default? // | '**' NAME ':' expression -// | '**' NAME +// | '**' NAME type_param_default? static type_param_ty type_param_rule(Parser *p) { @@ -10676,21 +10680,24 @@ type_param_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // NAME type_param_bound? + { // NAME type_param_bound? type_param_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound?")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound? type_param_default?")); expr_ty a; void *b; + void *c; if ( (a = _PyPegen_name_token(p)) // NAME && (b = type_param_bound_rule(p), !p->error_indicator) // type_param_bound? + && + (c = type_param_default_rule(p), !p->error_indicator) // type_param_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound?")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME type_param_bound? type_param_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10700,7 +10707,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_TypeVar ( a -> v . Name . id , b , EXTRA ); + _res = _PyAST_TypeVar ( a -> v . Name . id , b , c , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10710,7 +10717,7 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound?")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME type_param_bound? type_param_default?")); } { // '*' NAME ':' expression if (p->error_indicator) { @@ -10745,21 +10752,24 @@ type_param_rule(Parser *p) D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME ':' expression")); } - { // '*' NAME + { // '*' NAME type_param_starred_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' NAME type_param_starred_default?")); Token * _literal; expr_ty a; + void *b; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && (a = _PyPegen_name_token(p)) // NAME + && + (b = type_param_starred_default_rule(p), !p->error_indicator) // type_param_starred_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME type_param_starred_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10769,7 +10779,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_TypeVarTuple ( a -> v . Name . id , EXTRA ); + _res = _PyAST_TypeVarTuple ( a -> v . Name . id , b , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10779,7 +10789,7 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*' NAME type_param_starred_default?")); } { // '**' NAME ':' expression if (p->error_indicator) { @@ -10814,21 +10824,24 @@ type_param_rule(Parser *p) D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME ':' expression")); } - { // '**' NAME + { // '**' NAME type_param_default? if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME")); + D(fprintf(stderr, "%*c> type_param[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**' NAME type_param_default?")); Token * _literal; expr_ty a; + void *b; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' && (a = _PyPegen_name_token(p)) // NAME + && + (b = type_param_default_rule(p), !p->error_indicator) // type_param_default? ) { - D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME")); + D(fprintf(stderr, "%*c+ type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME type_param_default?")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -10838,7 +10851,7 @@ type_param_rule(Parser *p) UNUSED(_end_lineno); // Only used by EXTRA macro int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro - _res = _PyAST_ParamSpec ( a -> v . Name . id , EXTRA ); + _res = _PyAST_ParamSpec ( a -> v . Name . id , b , EXTRA ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; p->level--; @@ -10848,7 +10861,7 @@ type_param_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s type_param[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**' NAME type_param_default?")); } _res = NULL; done: @@ -10903,6 +10916,98 @@ type_param_bound_rule(Parser *p) return _res; } +// type_param_default: '=' expression +static expr_ty +type_param_default_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // '=' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_param_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' expression")); + Token * _literal; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (e = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ type_param_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' expression")); + _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_param_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// type_param_starred_default: '=' star_expression +static expr_ty +type_param_starred_default_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + expr_ty _res = NULL; + int _mark = p->mark; + { // '=' star_expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_param_starred_default[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); + Token * _literal; + expr_ty e; + if ( + (_literal = _PyPegen_expect_token(p, 22)) // token='=' + && + (e = star_expression_rule(p)) // star_expression + ) + { + D(fprintf(stderr, "%*c+ type_param_starred_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); + _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_param_starred_default[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'=' star_expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // expressions: expression ((',' expression))+ ','? | expression ',' | expression static expr_ty expressions_rule(Parser *p) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 60b46263a0d3299..cc7734e0dbbf9f6 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -203,6 +203,7 @@ void _PyAST_Fini(PyInterpreterState *interp) Py_CLEAR(state->conversion); Py_CLEAR(state->ctx); Py_CLEAR(state->decorator_list); + Py_CLEAR(state->default_value); Py_CLEAR(state->defaults); Py_CLEAR(state->elt); Py_CLEAR(state->elts); @@ -311,6 +312,7 @@ static int init_identifiers(struct ast_state *state) if ((state->conversion = PyUnicode_InternFromString("conversion")) == NULL) return -1; if ((state->ctx = PyUnicode_InternFromString("ctx")) == NULL) return -1; if ((state->decorator_list = PyUnicode_InternFromString("decorator_list")) == NULL) return -1; + if ((state->default_value = PyUnicode_InternFromString("default_value")) == NULL) return -1; if ((state->defaults = PyUnicode_InternFromString("defaults")) == NULL) return -1; if ((state->elt = PyUnicode_InternFromString("elt")) == NULL) return -1; if ((state->elts = PyUnicode_InternFromString("elts")) == NULL) return -1; @@ -809,12 +811,15 @@ static PyObject* ast2obj_type_param(struct ast_state *state, struct validator static const char * const TypeVar_fields[]={ "name", "bound", + "default_value", }; static const char * const ParamSpec_fields[]={ "name", + "default_value", }; static const char * const TypeVarTuple_fields[]={ "name", + "default_value", }; @@ -4913,6 +4918,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVar_annotations, "default_value", type) + == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVar_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->TypeVar_type, "_field_types", TypeVar_annotations) == 0; if (!cond) { @@ -4938,6 +4959,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + cond = PyDict_SetItemString(ParamSpec_annotations, "default_value", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(ParamSpec_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->ParamSpec_type, "_field_types", ParamSpec_annotations) == 0; if (!cond) { @@ -4964,6 +5001,22 @@ add_ast_annotations(struct ast_state *state) return 0; } } + { + PyObject *type = state->expr_type; + type = _Py_union_type_or(type, Py_None); + cond = type != NULL; + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + cond = PyDict_SetItemString(TypeVarTuple_annotations, "default_value", + type) == 0; + Py_DECREF(type); + if (!cond) { + Py_DECREF(TypeVarTuple_annotations); + return 0; + } + } cond = PyObject_SetAttrString(state->TypeVarTuple_type, "_field_types", TypeVarTuple_annotations) == 0; if (!cond) { @@ -6243,28 +6296,37 @@ init_types(struct ast_state *state) if (!state->TypeIgnore_type) return -1; state->type_param_type = make_type(state, "type_param", state->AST_type, NULL, 0, - "type_param = TypeVar(identifier name, expr? bound)\n" - " | ParamSpec(identifier name)\n" - " | TypeVarTuple(identifier name)"); + "type_param = TypeVar(identifier name, expr? bound, expr? default_value)\n" + " | ParamSpec(identifier name, expr? default_value)\n" + " | TypeVarTuple(identifier name, expr? default_value)"); if (!state->type_param_type) return -1; if (add_attributes(state, state->type_param_type, type_param_attributes, 4) < 0) return -1; state->TypeVar_type = make_type(state, "TypeVar", state->type_param_type, - TypeVar_fields, 2, - "TypeVar(identifier name, expr? bound)"); + TypeVar_fields, 3, + "TypeVar(identifier name, expr? bound, expr? default_value)"); if (!state->TypeVar_type) return -1; if (PyObject_SetAttr(state->TypeVar_type, state->bound, Py_None) == -1) return -1; + if (PyObject_SetAttr(state->TypeVar_type, state->default_value, Py_None) == + -1) + return -1; state->ParamSpec_type = make_type(state, "ParamSpec", state->type_param_type, ParamSpec_fields, - 1, - "ParamSpec(identifier name)"); + 2, + "ParamSpec(identifier name, expr? default_value)"); if (!state->ParamSpec_type) return -1; + if (PyObject_SetAttr(state->ParamSpec_type, state->default_value, Py_None) + == -1) + return -1; state->TypeVarTuple_type = make_type(state, "TypeVarTuple", state->type_param_type, - TypeVarTuple_fields, 1, - "TypeVarTuple(identifier name)"); + TypeVarTuple_fields, 2, + "TypeVarTuple(identifier name, expr? default_value)"); if (!state->TypeVarTuple_type) return -1; + if (PyObject_SetAttr(state->TypeVarTuple_type, state->default_value, + Py_None) == -1) + return -1; if (!add_ast_annotations(state)) { return -1; @@ -8055,8 +8117,9 @@ _PyAST_TypeIgnore(int lineno, string tag, PyArena *arena) } type_param_ty -_PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_TypeVar(identifier name, expr_ty bound, expr_ty default_value, int + lineno, int col_offset, int end_lineno, int end_col_offset, + PyArena *arena) { type_param_ty p; if (!name) { @@ -8070,6 +8133,7 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int p->kind = TypeVar_kind; p->v.TypeVar.name = name; p->v.TypeVar.bound = bound; + p->v.TypeVar.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -8078,8 +8142,8 @@ _PyAST_TypeVar(identifier name, expr_ty bound, int lineno, int col_offset, int } type_param_ty -_PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, - int end_col_offset, PyArena *arena) +_PyAST_ParamSpec(identifier name, expr_ty default_value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena *arena) { type_param_ty p; if (!name) { @@ -8092,6 +8156,7 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, return NULL; p->kind = ParamSpec_kind; p->v.ParamSpec.name = name; + p->v.ParamSpec.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -8100,8 +8165,9 @@ _PyAST_ParamSpec(identifier name, int lineno, int col_offset, int end_lineno, } type_param_ty -_PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int - end_lineno, int end_col_offset, PyArena *arena) +_PyAST_TypeVarTuple(identifier name, expr_ty default_value, int lineno, int + col_offset, int end_lineno, int end_col_offset, PyArena + *arena) { type_param_ty p; if (!name) { @@ -8114,6 +8180,7 @@ _PyAST_TypeVarTuple(identifier name, int lineno, int col_offset, int return NULL; p->kind = TypeVarTuple_kind; p->v.TypeVarTuple.name = name; + p->v.TypeVarTuple.default_value = default_value; p->lineno = lineno; p->col_offset = col_offset; p->end_lineno = end_lineno; @@ -10079,6 +10146,11 @@ ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) if (PyObject_SetAttr(result, state->bound, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.TypeVar.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; case ParamSpec_kind: tp = (PyTypeObject *)state->ParamSpec_type; @@ -10089,6 +10161,11 @@ ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.ParamSpec.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; case TypeVarTuple_kind: tp = (PyTypeObject *)state->TypeVarTuple_type; @@ -10099,6 +10176,11 @@ ast2obj_type_param(struct ast_state *state, struct validator *vstate, void* _o) if (PyObject_SetAttr(result, state->name, value) == -1) goto failed; Py_DECREF(value); + value = ast2obj_expr(state, vstate, o->v.TypeVarTuple.default_value); + if (!value) goto failed; + if (PyObject_SetAttr(result, state->default_value, value) == -1) + goto failed; + Py_DECREF(value); break; } value = ast2obj_int(state, vstate, o->lineno); @@ -16935,6 +17017,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (isinstance) { identifier name; expr_ty bound; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { return -1; @@ -16970,8 +17053,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeVar(name, bound, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'TypeVar' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeVar(name, bound, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -16982,6 +17082,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, } if (isinstance) { identifier name; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { return -1; @@ -17000,8 +17101,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_ParamSpec(name, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'ParamSpec' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_ParamSpec(name, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } @@ -17012,6 +17130,7 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, } if (isinstance) { identifier name; + expr_ty default_value; if (PyObject_GetOptionalAttr(obj, state->name, &tmp) < 0) { return -1; @@ -17030,8 +17149,25 @@ obj2ast_type_param(struct ast_state *state, PyObject* obj, type_param_ty* out, if (res != 0) goto failed; Py_CLEAR(tmp); } - *out = _PyAST_TypeVarTuple(name, lineno, col_offset, end_lineno, - end_col_offset, arena); + if (PyObject_GetOptionalAttr(obj, state->default_value, &tmp) < 0) { + return -1; + } + if (tmp == NULL || tmp == Py_None) { + Py_CLEAR(tmp); + default_value = NULL; + } + else { + int res; + if (_Py_EnterRecursiveCall(" while traversing 'TypeVarTuple' node")) { + goto failed; + } + res = obj2ast_expr(state, tmp, &default_value, arena); + _Py_LeaveRecursiveCall(); + if (res != 0) goto failed; + Py_CLEAR(tmp); + } + *out = _PyAST_TypeVarTuple(name, default_value, lineno, col_offset, + end_lineno, end_col_offset, arena); if (*out == NULL) goto failed; return 0; } diff --git a/Python/ast.c b/Python/ast.c index 71b09d889f17c19..1d1a48ec885686f 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -1011,13 +1011,19 @@ validate_typeparam(struct validator *state, type_param_ty tp) case TypeVar_kind: ret = validate_name(tp->v.TypeVar.name) && (!tp->v.TypeVar.bound || - validate_expr(state, tp->v.TypeVar.bound, Load)); + validate_expr(state, tp->v.TypeVar.bound, Load)) && + (!tp->v.TypeVar.default_value || + validate_expr(state, tp->v.TypeVar.default_value, Load)); break; case ParamSpec_kind: - ret = validate_name(tp->v.ParamSpec.name); + ret = validate_name(tp->v.ParamSpec.name) && + (!tp->v.ParamSpec.default_value || + validate_expr(state, tp->v.ParamSpec.default_value, Load)); break; case TypeVarTuple_kind: - ret = validate_name(tp->v.TypeVarTuple.name); + ret = validate_name(tp->v.TypeVarTuple.name) && + (!tp->v.TypeVarTuple.default_value || + validate_expr(state, tp->v.TypeVarTuple.default_value, Load)); break; } return ret; diff --git a/Python/compile.c b/Python/compile.c index feedd9888343970..ec47af151f99580 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2116,6 +2116,36 @@ wrap_in_stopiteration_handler(struct compiler *c) return SUCCESS; } +static int +compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, + identifier name, void *key, + bool allow_starred) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS, + key, e->lineno) == -1) { + return ERROR; + } + if (allow_starred && e->kind == Starred_kind) { + VISIT(c, expr, e->v.Starred.value); + ADDOP_I(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); + } + else { + VISIT(c, expr, e); + } + ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, LOC(e), co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return SUCCESS; +} + static int compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) { @@ -2123,6 +2153,7 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) return SUCCESS; } Py_ssize_t n = asdl_seq_LEN(type_params); + bool seen_default = false; for (Py_ssize_t i = 0; i < n; i++) { type_param_ty typeparam = asdl_seq_GET(type_params, i); @@ -2132,22 +2163,10 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVar.name); if (typeparam->v.TypeVar.bound) { expr_ty bound = typeparam->v.TypeVar.bound; - if (compiler_enter_scope(c, typeparam->v.TypeVar.name, COMPILER_SCOPE_TYPEPARAMS, - (void *)typeparam, bound->lineno) == -1) { + if (compiler_type_param_bound_or_default(c, bound, typeparam->v.TypeVar.name, + (void *)typeparam, false) < 0) { return ERROR; } - VISIT_IN_SCOPE(c, expr, bound); - ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); - PyCodeObject *co = optimize_and_assemble(c, 1); - compiler_exit_scope(c); - if (co == NULL) { - return ERROR; - } - if (compiler_make_closure(c, loc, co, 0) < 0) { - Py_DECREF(co); - return ERROR; - } - Py_DECREF(co); int intrinsic = bound->kind == Tuple_kind ? INTRINSIC_TYPEVAR_WITH_CONSTRAINTS @@ -2157,18 +2176,60 @@ compiler_type_params(struct compiler *c, asdl_type_param_seq *type_params) else { ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVAR); } + if (typeparam->v.TypeVar.default_value) { + seen_default = true; + expr_ty default_ = typeparam->v.TypeVar.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVar.name, + (void *)((uintptr_t)typeparam + 1), false) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVar.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVar.name, Store)); break; case TypeVarTuple_kind: ADDOP_LOAD_CONST(c, loc, typeparam->v.TypeVarTuple.name); ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_TYPEVARTUPLE); + if (typeparam->v.TypeVarTuple.default_value) { + expr_ty default_ = typeparam->v.TypeVarTuple.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.TypeVarTuple.name, + (void *)typeparam, true) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.TypeVarTuple.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.TypeVarTuple.name, Store)); break; case ParamSpec_kind: ADDOP_LOAD_CONST(c, loc, typeparam->v.ParamSpec.name); ADDOP_I(c, loc, CALL_INTRINSIC_1, INTRINSIC_PARAMSPEC); + if (typeparam->v.ParamSpec.default_value) { + expr_ty default_ = typeparam->v.ParamSpec.default_value; + if (compiler_type_param_bound_or_default(c, default_, typeparam->v.ParamSpec.name, + (void *)typeparam, false) < 0) { + return ERROR; + } + ADDOP_I(c, loc, CALL_INTRINSIC_2, INTRINSIC_SET_TYPEPARAM_DEFAULT); + seen_default = true; + } + else if (seen_default) { + return compiler_error(c, loc, "non-default type parameter '%U' " + "follows default type parameter", + typeparam->v.ParamSpec.name); + } ADDOP_I(c, loc, COPY, 1); RETURN_IF_ERROR(compiler_nameop(c, loc, typeparam->v.ParamSpec.name, Store)); break; diff --git a/Python/intrinsics.c b/Python/intrinsics.c index d3146973b75178d..5b10c3ce7d5d775 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -265,6 +265,7 @@ _PyIntrinsics_BinaryFunctions[] = { INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_BOUND, make_typevar_with_bound) INTRINSIC_FUNC_ENTRY(INTRINSIC_TYPEVAR_WITH_CONSTRAINTS, make_typevar_with_constraints) INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_FUNCTION_TYPE_PARAMS, _Py_set_function_type_params) + INTRINSIC_FUNC_ENTRY(INTRINSIC_SET_TYPEPARAM_DEFAULT, _Py_set_typeparam_default) }; #undef INTRINSIC_FUNC_ENTRY diff --git a/Python/symtable.c b/Python/symtable.c index eecd159b2c3f174..2ec21a2d376da27 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -2275,6 +2275,24 @@ symtable_visit_expr(struct symtable *st, expr_ty e) VISIT_QUIT(st, 1); } +static int +symtable_visit_type_param_bound_or_default(struct symtable *st, expr_ty e, identifier name, void *key) +{ + if (e) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + if (!symtable_enter_block(st, name, TypeVarBoundBlock, key, LOCATION(e))) + return 0; + st->st_cur->ste_can_see_class_scope = is_in_class; + if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(e))) { + VISIT_QUIT(st, 0); + } + VISIT(st, expr, e); + if (!symtable_exit_block(st)) + return 0; + } + return 1; +} + static int symtable_visit_type_param(struct symtable *st, type_param_ty tp) { @@ -2287,28 +2305,39 @@ symtable_visit_type_param(struct symtable *st, type_param_ty tp) case TypeVar_kind: if (!symtable_add_def(st, tp->v.TypeVar.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) VISIT_QUIT(st, 0); - if (tp->v.TypeVar.bound) { - int is_in_class = st->st_cur->ste_can_see_class_scope; - if (!symtable_enter_block(st, tp->v.TypeVar.name, - TypeVarBoundBlock, (void *)tp, - LOCATION(tp))) - VISIT_QUIT(st, 0); - st->st_cur->ste_can_see_class_scope = is_in_class; - if (is_in_class && !symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(tp->v.TypeVar.bound))) { - VISIT_QUIT(st, 0); - } - VISIT(st, expr, tp->v.TypeVar.bound); - if (!symtable_exit_block(st)) - VISIT_QUIT(st, 0); + + // We must use a different key for the bound and default. The obvious choice would be to + // use the .bound and .default_value pointers, but that fails when the expression immediately + // inside the bound or default is a comprehension: we would reuse the same key for + // the comprehension scope. Therefore, use the address + 1 as the second key. + // The only requirement for the key is that it is unique and it matches the logic in + // compile.c where the scope is retrieved. + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.bound, tp->v.TypeVar.name, + (void *)tp)) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVar.default_value, tp->v.TypeVar.name, + (void *)((uintptr_t)tp + 1))) { + VISIT_QUIT(st, 0); } break; case TypeVarTuple_kind: - if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.TypeVarTuple.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { VISIT_QUIT(st, 0); + } + if (!symtable_visit_type_param_bound_or_default(st, tp->v.TypeVarTuple.default_value, tp->v.TypeVarTuple.name, + (void *)tp)) { + VISIT_QUIT(st, 0); + } break; case ParamSpec_kind: - if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) + if (!symtable_add_def(st, tp->v.ParamSpec.name, DEF_TYPE_PARAM | DEF_LOCAL, LOCATION(tp))) { VISIT_QUIT(st, 0); + } + if (!symtable_visit_type_param_bound_or_default(st, tp->v.ParamSpec.default_value, tp->v.ParamSpec.name, + (void *)tp)) { + VISIT_QUIT(st, 0); + } break; } VISIT_QUIT(st, 1); diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 79a8f850ec14512..b58e9d9fae380fe 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -92,6 +92,7 @@ Objects/typeobject.c - PyBaseObject_Type - Objects/typeobject.c - PySuper_Type - Objects/typeobject.c - PyType_Type - Objects/typevarobject.c - _PyTypeAlias_Type - +Objects/typevarobject.c - _PyNoDefault_Type - Objects/unicodeobject.c - PyUnicodeIter_Type - Objects/unicodeobject.c - PyUnicode_Type - Objects/weakrefobject.c - _PyWeakref_CallableProxyType - @@ -310,6 +311,7 @@ Objects/object.c - _Py_NotImplementedStruct - Objects/setobject.c - _dummy_struct - Objects/setobject.c - _PySet_Dummy - Objects/sliceobject.c - _Py_EllipsisObject - +Objects/typevarobject.c - _Py_NoDefaultStruct - Python/instrumentation.c - _PyInstrumentation_DISABLE - Python/instrumentation.c - _PyInstrumentation_MISSING - From 37ccf167869d101c4021c435868b7f89ccda8148 Mon Sep 17 00:00:00 2001 From: Alexander Kanavin Date: Fri, 3 May 2024 15:34:05 +0200 Subject: [PATCH 080/220] gh-101732: Modules/_ssl.c: use Y2038 compatible openssl function when available (GH-118425) --- .../Library/2024-04-30-12-59-04.gh-issue-101732.29zUDu.rst | 1 + Modules/_ssl.c | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-04-30-12-59-04.gh-issue-101732.29zUDu.rst diff --git a/Misc/NEWS.d/next/Library/2024-04-30-12-59-04.gh-issue-101732.29zUDu.rst b/Misc/NEWS.d/next/Library/2024-04-30-12-59-04.gh-issue-101732.29zUDu.rst new file mode 100644 index 000000000000000..354dfc463620620 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-30-12-59-04.gh-issue-101732.29zUDu.rst @@ -0,0 +1 @@ +Use a Y2038 compatible openssl time function when available. diff --git a/Modules/_ssl.c b/Modules/_ssl.c index f7fdbf4b6f90cbc..885a9c25967d2d7 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -5329,7 +5329,11 @@ PySSLSession_clear(PySSLSession *self) static PyObject * PySSLSession_get_time(PySSLSession *self, void *closure) { +#if OPENSSL_VERSION_NUMBER >= 0x30300000L + return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); +#else return PyLong_FromLong(SSL_SESSION_get_time(self->session)); +#endif } PyDoc_STRVAR(PySSLSession_get_time_doc, From c8deb1e4b495bf97ab00c710dfd63f227e1fb645 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Fri, 3 May 2024 08:05:19 -0600 Subject: [PATCH 081/220] gh-118513: Fix sibling comprehensions with a name bound in one and global in the other (#118526) Co-authored-by: Jelle Zijlstra Co-authored-by: Kirill Podoprigora --- Lib/test/test_listcomps.py | 14 ++++ ...-05-02-21-19-35.gh-issue-118513.qHODjb.rst | 1 + Python/compile.c | 81 ++++++++++--------- 3 files changed, 57 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-02-21-19-35.gh-issue-118513.qHODjb.rst diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index 2868dd01545b95f..df1debf35210caf 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -666,6 +666,20 @@ def test_code_replace_extended_arg(self): self._check_in_scopes(code, expected) self._check_in_scopes(code, expected, exec_func=self._replacing_exec) + def test_multiple_comprehension_name_reuse(self): + code = """ + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"y": [3]}, ns={"x": 3}) + + code = """ + x = 2 + [x for x in [1]] + y = [x for _ in [1]] + """ + self._check_in_scopes(code, {"x": 2, "y": [3]}, ns={"x": 3}, scopes=["class"]) + self._check_in_scopes(code, {"x": 2, "y": [2]}, ns={"x": 3}, scopes=["function", "module"]) __test__ = {'doctests' : doctests} diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-21-19-35.gh-issue-118513.qHODjb.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-21-19-35.gh-issue-118513.qHODjb.rst new file mode 100644 index 000000000000000..b7155b4474d140a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-21-19-35.gh-issue-118513.qHODjb.rst @@ -0,0 +1 @@ +Fix incorrect :exc:`UnboundLocalError` when two comprehensions in the same function both reference the same name, and in one comprehension the name is bound while in the other it's an implicit global. diff --git a/Python/compile.c b/Python/compile.c index ec47af151f99580..35a7848f021c314 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -5561,10 +5561,48 @@ push_inlined_comprehension_state(struct compiler *c, location loc, while (PyDict_Next(entry->ste_symbols, &pos, &k, &v)) { assert(PyLong_Check(v)); long symbol = PyLong_AS_LONG(v); - // only values bound in the comprehension (DEF_LOCAL) need to be handled - // at all; DEF_LOCAL | DEF_NONLOCAL can occur in the case of an - // assignment expression to a nonlocal in the comprehension, these don't - // need handling here since they shouldn't be isolated + long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; + PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); + if (outv == NULL) { + if (PyErr_Occurred()) { + return ERROR; + } + outv = _PyLong_GetZero(); + } + assert(PyLong_CheckExact(outv)); + long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; + // If a name has different scope inside than outside the comprehension, + // we need to temporarily handle it with the right scope while + // compiling the comprehension. If it's free in the comprehension + // scope, no special handling; it should be handled the same as the + // enclosing scope. (If it's free in outer scope and cell in inner + // scope, we can't treat it as both cell and free in the same function, + // but treating it as free throughout is fine; it's *_DEREF + // either way.) + if ((scope != outsc && scope != FREE && !(scope == CELL && outsc == FREE)) + || in_class_block) { + if (state->temp_symbols == NULL) { + state->temp_symbols = PyDict_New(); + if (state->temp_symbols == NULL) { + return ERROR; + } + } + // update the symbol to the in-comprehension version and save + // the outer version; we'll restore it after running the + // comprehension + Py_INCREF(outv); + if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { + Py_DECREF(outv); + return ERROR; + } + if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { + Py_DECREF(outv); + return ERROR; + } + Py_DECREF(outv); + } + // locals handling for names bound in comprehension (DEF_LOCAL | + // DEF_NONLOCAL occurs in assignment expression to nonlocal) if ((symbol & DEF_LOCAL && !(symbol & DEF_NONLOCAL)) || in_class_block) { if (!_PyST_IsFunctionLike(c->u->u_ste)) { // non-function scope: override this name to use fast locals @@ -5589,41 +5627,6 @@ push_inlined_comprehension_state(struct compiler *c, location loc, } } } - long scope = (symbol >> SCOPE_OFFSET) & SCOPE_MASK; - PyObject *outv = PyDict_GetItemWithError(c->u->u_ste->ste_symbols, k); - if (outv == NULL) { - outv = _PyLong_GetZero(); - } - assert(PyLong_Check(outv)); - long outsc = (PyLong_AS_LONG(outv) >> SCOPE_OFFSET) & SCOPE_MASK; - if (scope != outsc && !(scope == CELL && outsc == FREE)) { - // If a name has different scope inside than outside the - // comprehension, we need to temporarily handle it with the - // right scope while compiling the comprehension. (If it's free - // in outer scope and cell in inner scope, we can't treat it as - // both cell and free in the same function, but treating it as - // free throughout is fine; it's *_DEREF either way.) - - if (state->temp_symbols == NULL) { - state->temp_symbols = PyDict_New(); - if (state->temp_symbols == NULL) { - return ERROR; - } - } - // update the symbol to the in-comprehension version and save - // the outer version; we'll restore it after running the - // comprehension - Py_INCREF(outv); - if (PyDict_SetItem(c->u->u_ste->ste_symbols, k, v) < 0) { - Py_DECREF(outv); - return ERROR; - } - if (PyDict_SetItem(state->temp_symbols, k, outv) < 0) { - Py_DECREF(outv); - return ERROR; - } - Py_DECREF(outv); - } // local names bound in comprehension must be isolated from // outer scope; push existing value (which may be NULL if // not defined) on stack From 24e643d4ef024a3561c927dc07c59c435bb27bcc Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 3 May 2024 11:05:30 -0400 Subject: [PATCH 082/220] gh-118527: Use `_Py_ID(__main__)` for main module name (#118528) Most module names are interned and immortalized, but the main module was not. This partially addresses a scaling bottleneck in the free-threaded when creating closure concurrently in the main module. --- Python/pylifecycle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 7726ccc979dbcc0..9dc6e3f31128c1b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -2348,7 +2348,7 @@ static PyStatus add_main_module(PyInterpreterState *interp) { PyObject *m, *d, *ann_dict; - m = PyImport_AddModule("__main__"); + m = PyImport_AddModuleObject(&_Py_ID(__main__)); if (m == NULL) return _PyStatus_ERR("can't create __main__ module"); From 2dae505e87e3815f087d4b07a71bb2c5cce22304 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 3 May 2024 11:09:57 -0400 Subject: [PATCH 083/220] gh-117514: Add `sys._is_gil_enabled()` function (#118514) The function returns `True` or `False` depending on whether the GIL is currently enabled. In the default build, it always returns `True` because the GIL is always enabled. --- Doc/library/sys.rst | 8 +++++ Lib/test/test_sys.py | 6 ++++ ...-05-02-16-04-51.gh-issue-117514.CJiuC0.rst | 4 +++ Python/clinic/sysmodule.c.h | 30 ++++++++++++++++++- Python/sysmodule.c | 20 +++++++++++++ 5 files changed, 67 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-02-16-04-51.gh-issue-117514.CJiuC0.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 19d6856efe5d097..91afa9d58e87e6d 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1199,6 +1199,14 @@ always available. return value of :func:`intern` around to benefit from it. +.. function:: _is_gil_enabled() + + Return :const:`True` if the :term:`GIL` is enabled and :const:`False` if + it is disabled. + + .. versionadded:: 3.13 + + .. function:: is_finalizing() Return :const:`True` if the main Python interpreter is diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 14ec51eb757e006..df0a6f09c09c082 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1053,6 +1053,12 @@ def test_getallocatedblocks(self): c = sys.getallocatedblocks() self.assertIn(c, range(b - 50, b + 50)) + def test_is_gil_enabled(self): + if support.Py_GIL_DISABLED: + self.assertIs(type(sys._is_gil_enabled()), bool) + else: + self.assertTrue(sys._is_gil_enabled()) + def test_is_finalizing(self): self.assertIs(sys.is_finalizing(), False) # Don't use the atexit module because _Py_Finalizing is only set diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-16-04-51.gh-issue-117514.CJiuC0.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-16-04-51.gh-issue-117514.CJiuC0.rst new file mode 100644 index 000000000000000..fc162afb2750c6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-16-04-51.gh-issue-117514.CJiuC0.rst @@ -0,0 +1,4 @@ +Add ``sys._is_gil_enabled()`` function that returns whether the GIL is +currently enabled. In the default build it always returns ``True`` because +the GIL is always enabled. In the free-threaded build, it may return +``True`` or ``False``. diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 31f66e807a8547e..0a8704c71e488cb 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1485,6 +1485,34 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(sys__is_gil_enabled__doc__, +"_is_gil_enabled($module, /)\n" +"--\n" +"\n" +"Return True if the GIL is currently enabled and False otherwise."); + +#define SYS__IS_GIL_ENABLED_METHODDEF \ + {"_is_gil_enabled", (PyCFunction)sys__is_gil_enabled, METH_NOARGS, sys__is_gil_enabled__doc__}, + +static int +sys__is_gil_enabled_impl(PyObject *module); + +static PyObject * +sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + int _return_value; + + _return_value = sys__is_gil_enabled_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyBool_FromLong((long)_return_value); + +exit: + return return_value; +} + #ifndef SYS_GETWINDOWSVERSION_METHODDEF #define SYS_GETWINDOWSVERSION_METHODDEF #endif /* !defined(SYS_GETWINDOWSVERSION_METHODDEF) */ @@ -1528,4 +1556,4 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=518424ee03e353b0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=352ac7a0085e8a1f input=a9049054013a1b77]*/ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d3fbfcd3e79636b..f469f165e7fb84d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2393,6 +2393,25 @@ sys__get_cpu_count_config_impl(PyObject *module) return config->cpu_count; } +/*[clinic input] +sys._is_gil_enabled -> bool + +Return True if the GIL is currently enabled and False otherwise. +[clinic start generated code]*/ + +static int +sys__is_gil_enabled_impl(PyObject *module) +/*[clinic end generated code: output=57732cf53f5b9120 input=7e9c47f15a00e809]*/ +{ +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + return interp->ceval.gil->enabled; +#else + return 1; +#endif +} + + static PerfMapState perf_map_state; PyAPI_FUNC(int) PyUnstable_PerfMapState_Init(void) { @@ -2565,6 +2584,7 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF + SYS__IS_GIL_ENABLED_METHODDEF {NULL, NULL} // sentinel }; From 3e818afb9b7c557aa633aeb3d5c4959750feeab0 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 3 May 2024 08:14:26 -0700 Subject: [PATCH 084/220] gh-118495: Skip test using threads after forking when running with TSAN (#118530) This is unsupported. Note that `skip_unless_reliable_fork()` checks for the conditions used by the decorators that were removed, along with checking for TSAN. --- Lib/test/test_threading.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 0047e8a8798d4e0..329767aa82e336b 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -49,7 +49,7 @@ def skip_unless_reliable_fork(test): if support.HAVE_ASAN_FORK_BUG: return unittest.skip("libasan has a pthread_create() dead lock related to thread+fork")(test) if support.check_sanitizer(thread=True): - return unittest.skip("TSAN doesn't support threads after fork") + return unittest.skip("TSAN doesn't support threads after fork")(test) return test @@ -781,8 +781,7 @@ def func(): "current is main True\n" ) - @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - @support.requires_fork() + @skip_unless_reliable_fork @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") def test_main_thread_after_fork_from_foreign_thread(self, create_dummy=False): code = """if 1: From c2627d6eea924daf80f374c18a5fd73ef61283fa Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Fri, 3 May 2024 08:30:55 -0700 Subject: [PATCH 085/220] gh-116322: Add Py_mod_gil module slot (#116882) This PR adds the ability to enable the GIL if it was disabled at interpreter startup, and modifies the multi-phase module initialization path to enable the GIL when loading a module, unless that module's spec includes a slot indicating it can run safely without the GIL. PEP 703 called the constant for the slot `Py_mod_gil_not_used`; I went with `Py_MOD_GIL_NOT_USED` for consistency with gh-104148. A warning will be issued up to once per interpreter for the first GIL-using module that is loaded. If `-v` is given, a shorter message will be printed to stderr every time a GIL-using module is loaded (including the first one that issues a warning). --- Doc/c-api/module.rst | 38 ++++++++++++++++ Include/internal/pycore_moduleobject.h | 3 ++ Include/moduleobject.h | 16 ++++++- .../extension/_test_nonmodule_cases.py | 44 +++++++++++++++++++ .../test_importlib/extension/test_loader.py | 35 +++++---------- Lib/test/test_sys.py | 5 ++- ...-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst | 5 +++ Modules/_abc.c | 1 + Modules/_asynciomodule.c | 1 + Modules/_bisectmodule.c | 1 + Modules/_blake2/blake2module.c | 1 + Modules/_bz2module.c | 1 + Modules/_codecsmodule.c | 1 + Modules/_collectionsmodule.c | 1 + Modules/_contextvarsmodule.c | 1 + Modules/_csv.c | 1 + Modules/_ctypes/_ctypes.c | 1 + Modules/_ctypes/_ctypes_test.c | 5 ++- Modules/_curses_panel.c | 1 + Modules/_cursesmodule.c | 3 ++ Modules/_datetimemodule.c | 3 ++ Modules/_dbmmodule.c | 1 + Modules/_decimal/_decimal.c | 1 + Modules/_elementtree.c | 1 + Modules/_functoolsmodule.c | 1 + Modules/_gdbmmodule.c | 1 + Modules/_hashopenssl.c | 1 + Modules/_heapqmodule.c | 1 + Modules/_interpchannelsmodule.c | 1 + Modules/_interpqueuesmodule.c | 1 + Modules/_interpretersmodule.c | 1 + Modules/_io/_iomodule.c | 1 + Modules/_json.c | 1 + Modules/_localemodule.c | 1 + Modules/_lsprof.c | 1 + Modules/_lzmamodule.c | 1 + Modules/_multiprocessing/multiprocessing.c | 1 + Modules/_multiprocessing/posixshmem.c | 5 ++- Modules/_opcode.c | 1 + Modules/_operator.c | 1 + Modules/_pickle.c | 1 + Modules/_posixsubprocess.c | 1 + Modules/_queuemodule.c | 1 + Modules/_randommodule.c | 1 + Modules/_scproxy.c | 5 ++- Modules/_sqlite/module.c | 1 + Modules/_sre/sre.c | 1 + Modules/_ssl.c | 1 + Modules/_stat.c | 1 + Modules/_statisticsmodule.c | 5 ++- Modules/_struct.c | 1 + Modules/_suggestions.c | 14 ++++-- Modules/_sysconfig.c | 1 + Modules/_testbuffer.c | 3 ++ Modules/_testcapimodule.c | 3 ++ Modules/_testclinic.c | 3 ++ Modules/_testclinic_limited.c | 3 ++ Modules/_testexternalinspection.c | 6 +++ Modules/_testimportmultiple.c | 26 ++++++----- Modules/_testinternalcapi.c | 1 + Modules/_testlimitedcapi.c | 3 ++ Modules/_testmultiphase.c | 21 ++++++++- Modules/_testsinglephase.c | 9 ++++ Modules/_threadmodule.c | 1 + Modules/_tkinter.c | 3 ++ Modules/_tracemalloc.c | 3 ++ Modules/_typingmodule.c | 1 + Modules/_uuidmodule.c | 5 ++- Modules/_weakref.c | 1 + Modules/_winapi.c | 1 + Modules/_xxtestfuzz/_xxtestfuzz.c | 9 +++- Modules/_zoneinfo.c | 1 + Modules/arraymodule.c | 1 + Modules/atexitmodule.c | 1 + Modules/binascii.c | 1 + Modules/cjkcodecs/cjkcodecs.h | 1 + Modules/cjkcodecs/multibytecodec.c | 1 + Modules/cmathmodule.c | 1 + Modules/errnomodule.c | 5 ++- Modules/faulthandler.c | 1 + Modules/fcntlmodule.c | 1 + Modules/gcmodule.c | 1 + Modules/grpmodule.c | 1 + Modules/itertoolsmodule.c | 1 + Modules/mathmodule.c | 1 + Modules/md5module.c | 1 + Modules/mmapmodule.c | 1 + Modules/overlapped.c | 1 + Modules/posixmodule.c | 1 + Modules/pwdmodule.c | 1 + Modules/pyexpat.c | 1 + Modules/readline.c | 3 ++ Modules/resource.c | 1 + Modules/selectmodule.c | 1 + Modules/sha1module.c | 1 + Modules/sha2module.c | 1 + Modules/sha3module.c | 1 + Modules/signalmodule.c | 1 + Modules/socketmodule.c | 1 + Modules/symtablemodule.c | 1 + Modules/syslogmodule.c | 1 + Modules/termios.c | 1 + Modules/timemodule.c | 1 + Modules/unicodedata.c | 1 + Modules/xxlimited.c | 5 ++- Modules/xxlimited_35.c | 4 ++ Modules/xxmodule.c | 1 + Modules/xxsubtype.c | 1 + Modules/zlibmodule.c | 1 + Objects/moduleobject.c | 35 +++++++++++++++ Objects/unicodeobject.c | 1 + PC/_testconsole.c | 5 ++- PC/msvcrtmodule.c | 1 + PC/winreg.c | 1 + PC/winsound.c | 5 ++- Parser/asdl_c.py | 1 + Python/Python-ast.c | 1 + Python/Python-tokenize.c | 1 + Python/_warnings.c | 1 + Python/bltinmodule.c | 3 ++ Python/import.c | 1 + Python/marshal.c | 1 + Python/sysmodule.c | 3 ++ 123 files changed, 376 insertions(+), 62 deletions(-) create mode 100644 Lib/test/test_importlib/extension/_test_nonmodule_cases.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 979b22261efa3b3..86308d921f47f2f 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -411,6 +411,31 @@ The available slot types are: .. versionadded:: 3.12 +.. c:macro:: Py_mod_gil + + Specifies one of the following values: + + .. c:macro:: Py_MOD_GIL_USED + + The module depends on the presence of the global interpreter lock (GIL), + and may access global state without synchronization. + + .. c:macro:: Py_MOD_GIL_NOT_USED + + The module is safe to run without an active GIL. + + This slot is ignored by Python builds not configured with + :option:`--disable-gil`. Otherwise, it determines whether or not importing + this module will cause the GIL to be automatically enabled. See + :envvar:`PYTHON_GIL` and :option:`-X gil <-X>` for more detail. + + Multiple ``Py_mod_gil`` slots may not be specified in one module definition. + + If ``Py_mod_gil`` is not specified, the import machinery defaults to + ``Py_MOD_GIL_USED``. + + .. versionadded: 3.13 + See :PEP:`489` for more details on multi-phase initialization. Low-level module creation functions @@ -609,6 +634,19 @@ state: .. versionadded:: 3.9 +.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil) + + Indicate that *module* does or does not support running without the global + interpreter lock (GIL), using one of the values from + :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization + function. If this function is not called during module initialization, the + import machinery assumes the module does not support running without the + GIL. This function is only available in Python builds configured with + :option:`--disable-gil`. + Return ``-1`` on error, ``0`` on success. + + .. versionadded:: 3.13 + Module lookup ^^^^^^^^^^^^^ diff --git a/Include/internal/pycore_moduleobject.h b/Include/internal/pycore_moduleobject.h index 5644bbe5e0552bd..049677b292e2352 100644 --- a/Include/internal/pycore_moduleobject.h +++ b/Include/internal/pycore_moduleobject.h @@ -22,6 +22,9 @@ typedef struct { PyObject *md_weaklist; // for logging purposes after md_dict is cleared PyObject *md_name; +#ifdef Py_GIL_DISABLED + void *md_gil; +#endif } PyModuleObject; static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) { diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 83f8c2030dbb8ff..6afa3c7be37ee7c 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -76,9 +76,13 @@ struct PyModuleDef_Slot { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000 # define Py_mod_multiple_interpreters 3 #endif +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_mod_gil 4 +#endif + #ifndef Py_LIMITED_API -#define _Py_mod_LAST_SLOT 3 +#define _Py_mod_LAST_SLOT 4 #endif #endif /* New in 3.5 */ @@ -90,6 +94,16 @@ struct PyModuleDef_Slot { # define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2) #endif +/* for Py_mod_gil: */ +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000 +# define Py_MOD_GIL_USED ((void *)0) +# define Py_MOD_GIL_NOT_USED ((void *)1) +#endif + +#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) +PyAPI_FUNC(int) PyModule_ExperimentalSetGIL(PyObject *module, void *gil); +#endif + struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; diff --git a/Lib/test/test_importlib/extension/_test_nonmodule_cases.py b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py new file mode 100644 index 000000000000000..8ffd18d221df4fa --- /dev/null +++ b/Lib/test/test_importlib/extension/_test_nonmodule_cases.py @@ -0,0 +1,44 @@ +import types +import unittest +from test.test_importlib import util + +machinery = util.import_importlib('importlib.machinery') + +from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests + + +class NonModuleExtensionTests: + setUp = MultiPhaseExtensionModuleTests.setUp + load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name + + def _test_nonmodule(self): + # Test returning a non-module object from create works. + name = self.name + '_nonmodule' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + + # issue 27782 + def test_nonmodule_with_methods(self): + # Test creating a non-module object with methods defined. + name = self.name + '_nonmodule_with_methods' + mod = self.load_module_by_name(name) + self.assertNotEqual(type(mod), type(unittest)) + self.assertEqual(mod.three, 3) + self.assertEqual(mod.bar(10, 1), 9) + + def test_null_slots(self): + # Test that NULL slots aren't a problem. + name = self.name + '_null_slots' + module = self.load_module_by_name(name) + self.assertIsInstance(module, types.ModuleType) + self.assertEqual(module.__name__, name) + + +(Frozen_NonModuleExtensionTests, + Source_NonModuleExtensionTests + ) = util.test_both(NonModuleExtensionTests, machinery=machinery) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_importlib/extension/test_loader.py b/Lib/test/test_importlib/extension/test_loader.py index 7607f0e08575959..0dd21e079eba224 100644 --- a/Lib/test/test_importlib/extension/test_loader.py +++ b/Lib/test/test_importlib/extension/test_loader.py @@ -10,7 +10,8 @@ import warnings import importlib.util import importlib -from test.support import MISSING_C_DOCSTRINGS +from test import support +from test.support import MISSING_C_DOCSTRINGS, script_helper class LoaderTests: @@ -325,29 +326,6 @@ def test_unloadable_nonascii(self): self.load_module_by_name(name) self.assertEqual(cm.exception.name, name) - def test_nonmodule(self): - # Test returning a non-module object from create works. - name = self.name + '_nonmodule' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - - # issue 27782 - def test_nonmodule_with_methods(self): - # Test creating a non-module object with methods defined. - name = self.name + '_nonmodule_with_methods' - mod = self.load_module_by_name(name) - self.assertNotEqual(type(mod), type(unittest)) - self.assertEqual(mod.three, 3) - self.assertEqual(mod.bar(10, 1), 9) - - def test_null_slots(self): - # Test that NULL slots aren't a problem. - name = self.name + '_null_slots' - module = self.load_module_by_name(name) - self.assertIsInstance(module, types.ModuleType) - self.assertEqual(module.__name__, name) - def test_bad_modules(self): # Test SystemError is raised for misbehaving extensions. for name_base in [ @@ -401,5 +379,14 @@ def test_nonascii(self): ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery) +class NonModuleExtensionTests(unittest.TestCase): + def test_nonmodule_cases(self): + # The test cases in this file cause the GIL to be enabled permanently + # in free-threaded builds, so they are run in a subprocess to isolate + # this effect. + script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py") + script_helper.run_test_script(script) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index df0a6f09c09c082..73912767ae25b74 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1606,7 +1606,10 @@ def get_gen(): yield 1 check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit) check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit) # module - check(unittest, size('PnPPP')) + if support.Py_GIL_DISABLED: + check(unittest, size('PPPPPP')) + else: + check(unittest, size('PPPPP')) # None check(None, size('')) # NotImplementedType diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst new file mode 100644 index 000000000000000..2d3bf411a5a1628 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst @@ -0,0 +1,5 @@ +Extension modules may indicate to the runtime that they can run without the +GIL. Multi-phase init modules do so by calling providing +``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init +modules call ``PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED)`` from +their init function. diff --git a/Modules/_abc.c b/Modules/_abc.c index f2a523e6f2fc273..4f4b24b035db4af 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -970,6 +970,7 @@ _abcmodule_free(void *module) static PyModuleDef_Slot _abcmodule_slots[] = { {Py_mod_exec, _abcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 0873d32a9ec1a5b..a26714f9755df5f 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -3795,6 +3795,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9e0fd336419b446..56322c48b7cd354 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -462,6 +462,7 @@ bisect_modexec(PyObject *m) static PyModuleDef_Slot bisect_slots[] = { {Py_mod_exec, bisect_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_blake2/blake2module.c b/Modules/_blake2/blake2module.c index 5df9fd3df493eeb..78242214764f2bb 100644 --- a/Modules/_blake2/blake2module.c +++ b/Modules/_blake2/blake2module.c @@ -137,6 +137,7 @@ blake2_exec(PyObject *m) static PyModuleDef_Slot _blake2_slots[] = { {Py_mod_exec, blake2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 3d0d4ee5e79c2bb..661847ad26702ee 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -802,6 +802,7 @@ _bz2_free(void *module) static struct PyModuleDef_Slot _bz2_slots[] = { {Py_mod_exec, _bz2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index c31c1b6d6f2bbc0..32373f0799bfeb8 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -1050,6 +1050,7 @@ static PyMethodDef _codecs_functions[] = { static PyModuleDef_Slot _codecs_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 309d63c9bf7cbe7..b865351c93d2d8b 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2817,6 +2817,7 @@ collections_exec(PyObject *module) { static struct PyModuleDef_Slot collections_slots[] = { {Py_mod_exec, collections_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index f621c1de6d42d6b..3f96f07909b69a6 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -45,6 +45,7 @@ _contextvars_exec(PyObject *m) static struct PyModuleDef_Slot _contextvars_slots[] = { {Py_mod_exec, _contextvars_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_csv.c b/Modules/_csv.c index ac948f417cebf51..9d6b66d4938687c 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -1796,6 +1796,7 @@ csv_exec(PyObject *module) { static PyModuleDef_Slot csv_slots[] = { {Py_mod_exec, csv_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 3cb0b24668eb2aa..1b1a0ea549f1e16 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -5948,6 +5948,7 @@ module_free(void *module) static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _ctypes_mod_exec}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index 1dd3ef190524701..f46f6362ddd03b1 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -1,7 +1,7 @@ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif // gh-85283: On Windows, Py_LIMITED_API requires Py_BUILD_CORE to not attempt @@ -1167,6 +1167,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item) static struct PyModuleDef_Slot _ctypes_test_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 2ec8f34c5c220b0..125c72dbbe77121 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -697,6 +697,7 @@ static PyModuleDef_Slot _curses_slots[] = { // XXX gh-103092: fix isolation. {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index d04d1e973af0303..8bf6824b6d83ffa 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -4743,6 +4743,9 @@ PyInit__curses(void) m = PyModule_Create(&_cursesmodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 06004e258b2eff9..00015c5d8c23cbc 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6984,6 +6984,9 @@ PyInit__datetime(void) PyObject *mod = PyModule_Create(&datetimemodule); if (mod == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_datetime_exec(mod) < 0) { Py_DECREF(mod); diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index ee33fe625be3d70..1be4234aad3291b 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -616,6 +616,7 @@ _dbm_module_free(void *module) static PyModuleDef_Slot _dbmmodule_slots[] = { {Py_mod_exec, _dbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index fe6711143b845b1..2daa24c823a5428 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -6157,6 +6157,7 @@ decimal_free(void *module) static struct PyModuleDef_Slot _decimal_slots[] = { {Py_mod_exec, _decimal_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index aaa0cad76ae5c4b..b11983d2caa2d1c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -4463,6 +4463,7 @@ module_exec(PyObject *m) static struct PyModuleDef_Slot elementtree_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index e37473a582b55fb..9dee7bf30627100 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -1559,6 +1559,7 @@ _functools_free(void *module) static struct PyModuleDef_Slot _functools_slots[] = { {Py_mod_exec, _functools_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index db868c18160fdad..df7fba67810ed04 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -825,6 +825,7 @@ _gdbm_module_free(void *module) static PyModuleDef_Slot _gdbm_module_slots[] = { {Py_mod_exec, _gdbm_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index d0b46810dc14895..14d9c186151232c 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2289,6 +2289,7 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_constructors}, {Py_mod_exec, hashlib_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 9d4ec256ee9e3ef..695ce22f8049df8 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -681,6 +681,7 @@ heapq_exec(PyObject *m) static struct PyModuleDef_Slot heapq_slots[] = { {Py_mod_exec, heapq_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 43c96584790fa05..ff8dacf5bd1ad0e 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -3326,6 +3326,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 46801bd416495a6..556953db6b80394 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1830,6 +1830,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 8fea56977ef3fe1..86a4113dcc16f1d 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -1519,6 +1519,7 @@ module_exec(PyObject *mod) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 173f5b55e5f732f..269070fe2b0a424 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -720,6 +720,7 @@ iomodule_exec(PyObject *m) static struct PyModuleDef_Slot iomodule_slots[] = { {Py_mod_exec, iomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_json.c b/Modules/_json.c index c55299899e77fe5..fc39f624b723f50 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1780,6 +1780,7 @@ _json_exec(PyObject *module) static PyModuleDef_Slot _json_slots[] = { {Py_mod_exec, _json_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index fe8e4c5e30035b3..d4923442478b3ec 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -860,6 +860,7 @@ _locale_exec(PyObject *module) static struct PyModuleDef_Slot _locale_slots[] = { {Py_mod_exec, _locale_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index a76c3dea5557839..18be01df5c13777 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -1006,6 +1006,7 @@ _lsprof_exec(PyObject *module) static PyModuleDef_Slot _lsprofslots[] = { {Py_mod_exec, _lsprof_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f6bfbfa62687b8b..97f3a8f03da9a86 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -1604,6 +1604,7 @@ static PyMethodDef lzma_methods[] = { static PyModuleDef_Slot lzma_slots[] = { {Py_mod_exec, lzma_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 1f6ab718a36984a..cee8cf7b9a83c06 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -277,6 +277,7 @@ multiprocessing_exec(PyObject *module) static PyModuleDef_Slot multiprocessing_slots[] = { {Py_mod_exec, multiprocessing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index d332a4e9d9ea0b8..aeb2d79de6f9ed5 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -2,10 +2,10 @@ posixshmem - A Python extension that provides shm_open() and shm_unlink() */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -128,6 +128,7 @@ static PyMethodDef module_methods[ ] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index 85e0ffec900e89f..cc72cb170ceaed4 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -406,6 +406,7 @@ _opcode_exec(PyObject *m) { static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _opcode_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_operator.c b/Modules/_operator.c index 306d4508f52a686..5d3f88327d19ad7 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1928,6 +1928,7 @@ operator_exec(PyObject *module) static struct PyModuleDef_Slot operator_slots[] = { {Py_mod_exec, operator_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d7ffb04c28c2ac3..754a326822e0f02 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -7863,6 +7863,7 @@ _pickle_exec(PyObject *m) static PyModuleDef_Slot pickle_slots[] = { {Py_mod_exec, _pickle_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b160cd78177a175..daec4ad708dea46 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1317,6 +1317,7 @@ static PyMethodDef module_methods[] = { static PyModuleDef_Slot _posixsubprocess_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 5db9b645849fcd8..aee8db802d8c3f5 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -594,6 +594,7 @@ queuemodule_exec(PyObject *module) static PyModuleDef_Slot queuemodule_slots[] = { {Py_mod_exec, queuemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 56b891dfe0f85f8..140640ae8fbf3a6 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -642,6 +642,7 @@ _random_exec(PyObject *module) static PyModuleDef_Slot _random_slots[] = { {Py_mod_exec, _random_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_scproxy.c b/Modules/_scproxy.c index 042738b4ab83a2e..e9170f2ce1ae872 100644 --- a/Modules/_scproxy.c +++ b/Modules/_scproxy.c @@ -3,10 +3,10 @@ * using the SystemConfiguration framework. */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -239,6 +239,7 @@ static PyMethodDef mod_methods[] = { static PyModuleDef_Slot _scproxy_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 46fed9f13281f39..2c25ee32e58189c 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -758,6 +758,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 00fbd9674b8cdda..c1eff63d921de94 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -3272,6 +3272,7 @@ sre_exec(PyObject *m) static PyModuleDef_Slot sre_slots[] = { {Py_mod_exec, sre_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 885a9c25967d2d7..9d50b576ba337f4 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -6515,6 +6515,7 @@ static PyModuleDef_Slot sslmodule_slots[] = { {Py_mod_exec, sslmodule_init_strings}, {Py_mod_exec, sslmodule_init_lock}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_stat.c b/Modules/_stat.c index 8059ec2f1f066db..a4f15e8e65e8947 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -679,6 +679,7 @@ stat_exec(PyObject *module) static PyModuleDef_Slot stat_slots[] = { {Py_mod_exec, stat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_statisticsmodule.c b/Modules/_statisticsmodule.c index 78a6552c4c9ec0c..b84f731ad6a1da8 100644 --- a/Modules/_statisticsmodule.c +++ b/Modules/_statisticsmodule.c @@ -1,9 +1,9 @@ /* statistics accelerator C extension: _statistics module. */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -136,6 +136,7 @@ PyDoc_STRVAR(statistics_doc, static struct PyModuleDef_Slot _statisticsmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_struct.c b/Modules/_struct.c index fa2cd37e003e0a2..905dcbdeeddc5a9 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -2593,6 +2593,7 @@ _structmodule_exec(PyObject *m) static PyModuleDef_Slot _structmodule_slots[] = { {Py_mod_exec, _structmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_suggestions.c b/Modules/_suggestions.c index 30b524d70c12114..80c7179c4c251c8 100644 --- a/Modules/_suggestions.c +++ b/Modules/_suggestions.c @@ -49,15 +49,21 @@ static PyMethodDef module_methods[] = { {NULL, NULL, 0, NULL} // Sentinel }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef suggestions_module = { PyModuleDef_HEAD_INIT, "_suggestions", NULL, - -1, - module_methods + 0, + module_methods, + module_slots, }; PyMODINIT_FUNC PyInit__suggestions(void) { - return PyModule_Create(&suggestions_module); + return PyModuleDef_Init(&suggestions_module); } - diff --git a/Modules/_sysconfig.c b/Modules/_sysconfig.c index c76b9e6b3ebafab..c50c5cfabc2f1f0 100644 --- a/Modules/_sysconfig.c +++ b/Modules/_sysconfig.c @@ -80,6 +80,7 @@ static struct PyMethodDef sysconfig_methods[] = { static PyModuleDef_Slot sysconfig_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 9e77744efad728a..35d4ffecad6b15d 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -2901,6 +2901,9 @@ PyInit__testbuffer(void) if (mod == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_testbuffer_exec(mod) < 0) { Py_DECREF(mod); return NULL; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f5892fc5ed2a2c1..beae13cd74c7319 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3935,6 +3935,9 @@ PyInit__testcapi(void) m = PyModule_Create(&_testcapimodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); if (PyType_Ready(&_HashInheritanceTester_Type) < 0) { diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 2e9d00ac2189eb7..c7af552f029b811 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1955,6 +1955,9 @@ PyInit__testclinic(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddType(m, &TestClass) < 0) { goto error; } diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index 29f1b7c13e4c505..d5f98085f842255 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -146,5 +146,8 @@ PyInit__testclinic_limited(void) if (m == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif return m; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index e2f96cdad5c58ef..d9c65fe253f8d3b 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -627,6 +627,12 @@ PyMODINIT_FUNC PyInit__testexternalinspection(void) { PyObject* mod = PyModule_Create(&module); + if (mod == NULL) { + return NULL; + } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { Py_DECREF(mod); diff --git a/Modules/_testimportmultiple.c b/Modules/_testimportmultiple.c index a65ca513a12516c..c147596f88a3a86 100644 --- a/Modules/_testimportmultiple.c +++ b/Modules/_testimportmultiple.c @@ -6,18 +6,24 @@ #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x03020000 +# define Py_LIMITED_API 0x030d0000 #endif #include +static PyModuleDef_Slot shared_slots[] = { + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _testimportmultiple = { PyModuleDef_HEAD_INIT, "_testimportmultiple", "_testimportmultiple doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -25,16 +31,16 @@ static struct PyModuleDef _testimportmultiple = { PyMODINIT_FUNC PyInit__testimportmultiple(void) { - return PyModule_Create(&_testimportmultiple); + return PyModuleDef_Init(&_testimportmultiple); } static struct PyModuleDef _foomodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_foo", "_testimportmultiple_foo doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL @@ -42,21 +48,21 @@ static struct PyModuleDef _foomodule = { PyMODINIT_FUNC PyInit__testimportmultiple_foo(void) { - return PyModule_Create(&_foomodule); + return PyModuleDef_Init(&_foomodule); } static struct PyModuleDef _barmodule = { PyModuleDef_HEAD_INIT, "_testimportmultiple_bar", "_testimportmultiple_bar doc", - -1, - NULL, + 0, NULL, + shared_slots, NULL, NULL, NULL }; PyMODINIT_FUNC PyInit__testimportmultiple_bar(void){ - return PyModule_Create(&_barmodule); + return PyModuleDef_Init(&_barmodule); } diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index f7952a119f0bd9c..de98af32b5dff73 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2142,6 +2142,7 @@ module_exec(PyObject *module) static struct PyModuleDef_Slot module_slots[] = { {Py_mod_exec, module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 598071fe0ddbad8..f88476f4be20546 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -25,6 +25,9 @@ PyInit__testlimitedcapi(void) if (mod == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { return NULL; diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 21c5f696a4f2ec4..ca3d83233c5dd03 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -431,6 +431,7 @@ static int execfunc(PyObject *m) static PyModuleDef_Slot main_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -519,13 +520,18 @@ PyInit__testmultiphase_nonmodule_with_methods(void) /**** Non-ASCII-named modules ****/ +static PyModuleDef_Slot nonascii_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef def_nonascii_latin = { \ PyModuleDef_HEAD_INIT, /* m_base */ "_testmultiphase_nonascii_latin", /* m_name */ PyDoc_STR("Module named in Czech"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -543,7 +549,7 @@ static PyModuleDef def_nonascii_kana = { \ PyDoc_STR("Module named in Japanese"), /* m_doc */ 0, /* m_size */ NULL, /* m_methods */ - NULL, /* m_slots */ + nonascii_slots, /* m_slots */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ @@ -757,6 +763,7 @@ static PyModuleDef_Slot slots_nonmodule_with_exec_slots[] = { {Py_mod_create, createfunc_nonmodule}, {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -778,6 +785,7 @@ execfunc_err(PyObject *mod) static PyModuleDef_Slot slots_exec_err[] = { {Py_mod_exec, execfunc_err}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -800,6 +808,7 @@ execfunc_raise(PyObject *spec) static PyModuleDef_Slot slots_exec_raise[] = { {Py_mod_exec, execfunc_raise}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -822,6 +831,7 @@ execfunc_unreported_exception(PyObject *mod) static PyModuleDef_Slot slots_exec_unreported_exception[] = { {Py_mod_exec, execfunc_unreported_exception}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -857,6 +867,7 @@ meth_state_access_exec(PyObject *m) static PyModuleDef_Slot meth_state_access_slots[] = { {Py_mod_exec, meth_state_access_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; @@ -889,6 +900,9 @@ PyInit__test_module_state_shared(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { Py_DECREF(module); @@ -903,6 +917,7 @@ PyInit__test_module_state_shared(void) static PyModuleDef_Slot slots_multiple_multiple_interpreters_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -920,6 +935,7 @@ PyInit__testmultiphase_multiple_multiple_interpreters_slots(void) static PyModuleDef_Slot non_isolated_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; @@ -940,6 +956,7 @@ static PyModuleDef_Slot shared_gil_only_slots[] = { We put it here explicitly to draw attention to the contrast with Py_MOD_PER_INTERPRETER_GIL_SUPPORTED. */ {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index ff533e44a82730c..c0eb266751e9e6e 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -471,6 +471,9 @@ init__testsinglephase_basic(PyModuleDef *def) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = &global_state.module; // It may have been set by a previous run or under a different name. @@ -562,6 +565,9 @@ PyInit__testsinglephase_with_reinit(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); +#endif assert(get_module_state(module) == NULL); @@ -624,6 +630,9 @@ PyInit__testsinglephase_with_state(void) if (module == NULL) { return NULL; } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); +#endif module_state *state = get_module_state(module); assert(state != NULL); diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index f5e3b42600675e4..39d309729d88b81 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -2544,6 +2544,7 @@ The 'threading' module provides a more convenient interface."); static PyModuleDef_Slot thread_module_slots[] = { {Py_mod_exec, thread_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index e3789867dc085fd..ecb7ca8de62ef1b 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3205,6 +3205,9 @@ PyInit__tkinter(void) m = PyModule_Create(&_tkintermodule); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif Tkinter_TclError = PyErr_NewException("_tkinter.TclError", NULL, NULL); if (PyModule_AddObjectRef(m, "TclError", Tkinter_TclError)) { diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 6dba3cac01c1c89..55028dc3a65ff46 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -219,6 +219,9 @@ PyInit__tracemalloc(void) m = PyModule_Create(&module_def); if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (_PyTraceMalloc_Init() < 0) { Py_DECREF(m); diff --git a/Modules/_typingmodule.c b/Modules/_typingmodule.c index 180f3d7eb01794c..09fbb3c5e8b91d7 100644 --- a/Modules/_typingmodule.c +++ b/Modules/_typingmodule.c @@ -72,6 +72,7 @@ _typing_exec(PyObject *m) static struct PyModuleDef_Slot _typingmodule_slots[] = { {Py_mod_exec, _typing_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index 052cb9fef3b21c0..c5e78b1510b5e3a 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -3,10 +3,10 @@ * DCE compatible Universally Unique Identifier library. */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -111,6 +111,7 @@ static PyMethodDef uuid_methods[] = { static PyModuleDef_Slot uuid_slots[] = { {Py_mod_exec, uuid_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_weakref.c b/Modules/_weakref.c index 1ea3ed5e40b761f..a5c15c0f10b930a 100644 --- a/Modules/_weakref.c +++ b/Modules/_weakref.c @@ -171,6 +171,7 @@ weakref_exec(PyObject *module) static struct PyModuleDef_Slot weakref_slots[] = { {Py_mod_exec, weakref_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 23e3c0d87f03190..cd5dd503abe61f4 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -3189,6 +3189,7 @@ static int winapi_exec(PyObject *m) static PyModuleDef_Slot winapi_slots[] = { {Py_mod_exec, winapi_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_xxtestfuzz/_xxtestfuzz.c b/Modules/_xxtestfuzz/_xxtestfuzz.c index a2dbabce71ed67c..2952d7043e01fec 100644 --- a/Modules/_xxtestfuzz/_xxtestfuzz.c +++ b/Modules/_xxtestfuzz/_xxtestfuzz.c @@ -28,13 +28,18 @@ static PyMethodDef module_methods[] = { {NULL}, }; +static PyModuleDef_Slot module_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static struct PyModuleDef _fuzzmodule = { PyModuleDef_HEAD_INIT, "_fuzz", NULL, 0, module_methods, - NULL, + module_slots, NULL, NULL, NULL @@ -43,5 +48,5 @@ static struct PyModuleDef _fuzzmodule = { PyMODINIT_FUNC PyInit__xxtestfuzz(void) { - return PyModule_Create(&_fuzzmodule); + return PyModuleDef_Init(&_fuzzmodule); } diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index fcd4af64df0be94..38c3f0c45d803fd 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -2760,6 +2760,7 @@ zoneinfomodule_exec(PyObject *m) static PyModuleDef_Slot zoneinfomodule_slots[] = { {Py_mod_exec, zoneinfomodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 317f4974814945e..a3b833d47cd9eae 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -3220,6 +3220,7 @@ array_modexec(PyObject *m) static PyModuleDef_Slot arrayslots[] = { {Py_mod_exec, array_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 8e908da2534c554..297a8d74ba3bf4c 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -322,6 +322,7 @@ Two public functions, register and unregister, are defined.\n\ static PyModuleDef_Slot atexitmodule_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/binascii.c b/Modules/binascii.c index 86493241a1fb7e2..250f03a9531eedc 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1278,6 +1278,7 @@ binascii_exec(PyObject *module) static PyModuleDef_Slot binascii_slots[] = { {Py_mod_exec, binascii_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 766f82983025e46..2b446ba5226ac03 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -503,6 +503,7 @@ static struct PyMethodDef _cjk_methods[] = { static PyModuleDef_Slot _cjk_slots[] = { {Py_mod_exec, _cjk_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index e5433d7dd853066..1c671adb4ff188a 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -2094,6 +2094,7 @@ static struct PyMethodDef _multibytecodec_methods[] = { static PyModuleDef_Slot _multibytecodec_slots[] = { {Py_mod_exec, _multibytecodec_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 57bc55632be4859..d901b350bc53430 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1363,6 +1363,7 @@ cmath_exec(PyObject *mod) static PyModuleDef_Slot cmath_slots[] = { {Py_mod_exec, cmath_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 97e5f0180d76fb1..3f96f2f846d6129 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -1,9 +1,9 @@ /* Errno module */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -951,6 +951,7 @@ errno_exec(PyObject *module) static PyModuleDef_Slot errno_slots[] = { {Py_mod_exec, errno_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index c70d43a36b5cc79..cfa3cbdc34bc86a 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -1292,6 +1292,7 @@ static PyModuleDef_Slot faulthandler_slots[] = { {Py_mod_exec, PyExec_faulthandler}, // XXX gh-103092: fix isolation. //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index e24e5f98f4bc4d7..b6eeec2c66f6e59 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -745,6 +745,7 @@ fcntl_exec(PyObject *module) static PyModuleDef_Slot fcntl_slots[] = { {Py_mod_exec, fcntl_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 8a1b483eddae354..57e4aae9ed557eb 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -535,6 +535,7 @@ gcmodule_exec(PyObject *module) static PyModuleDef_Slot gcmodule_slots[] = { {Py_mod_exec, gcmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index a1fa6cf20f71fd8..f7d3e12f347ec2f 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -342,6 +342,7 @@ grpmodule_exec(PyObject *module) static PyModuleDef_Slot grpmodule_slots[] = { {Py_mod_exec, grpmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 21ce3ecfad03547..8641c2f87e6db20 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4781,6 +4781,7 @@ itertoolsmodule_exec(PyObject *mod) static struct PyModuleDef_Slot itertoolsmodule_slots[] = { {Py_mod_exec, itertoolsmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 8ba0431f4a47b79..a3cbfc383761a02 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -4177,6 +4177,7 @@ static PyMethodDef math_methods[] = { static PyModuleDef_Slot math_slots[] = { {Py_mod_exec, math_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/md5module.c b/Modules/md5module.c index 9cbf11feaa9c326..ef9163e8be5b6c7 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -375,6 +375,7 @@ md5_exec(PyObject *m) static PyModuleDef_Slot _md5_slots[] = { {Py_mod_exec, md5_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 0cce7c27f9b16a5..dfc16ff43703494 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1801,6 +1801,7 @@ mmap_exec(PyObject *module) static PyModuleDef_Slot mmap_slots[] = { {Py_mod_exec, mmap_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/overlapped.c b/Modules/overlapped.c index b9881d91ded2447..77ee70ae133c858 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -2070,6 +2070,7 @@ overlapped_exec(PyObject *module) static PyModuleDef_Slot overlapped_slots[] = { {Py_mod_exec, overlapped_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e1a14e772c4bd03..9f4be98b35186ee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -18071,6 +18071,7 @@ posixmodule_exec(PyObject *m) static PyModuleDef_Slot posixmodile_slots[] = { {Py_mod_exec, posixmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pwdmodule.c b/Modules/pwdmodule.c index f58735aff997992..2240e2078b2d982 100644 --- a/Modules/pwdmodule.c +++ b/Modules/pwdmodule.c @@ -344,6 +344,7 @@ pwdmodule_exec(PyObject *module) static PyModuleDef_Slot pwdmodule_slots[] = { {Py_mod_exec, pwdmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f04f96bc2f7601c..f67d480f19de001 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -2117,6 +2117,7 @@ pyexpat_free(void *module) static PyModuleDef_Slot pyexpat_slots[] = { {Py_mod_exec, pyexpat_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/readline.c b/Modules/readline.c index c5c34535de01546..f59f8a9834caafc 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1552,6 +1552,9 @@ PyInit_readline(void) if (m == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); +#endif if (PyModule_AddIntConstant(m, "_READLINE_VERSION", RL_READLINE_VERSION) < 0) { diff --git a/Modules/resource.c b/Modules/resource.c index 8ee07bd0c8054c1..3fe18e7c98e3d81 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -513,6 +513,7 @@ resource_exec(PyObject *module) static struct PyModuleDef_Slot resource_slots[] = { {Py_mod_exec, resource_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 6ea141ab1f91897..3eaee22c652c284 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2802,6 +2802,7 @@ _select_exec(PyObject *m) static PyModuleDef_Slot _select_slots[] = { {Py_mod_exec, _select_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index 345a6c215eb1671..34a427a39b5cf84 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -371,6 +371,7 @@ _sha1_exec(PyObject *module) static PyModuleDef_Slot _sha1_slots[] = { {Py_mod_exec, _sha1_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha2module.c b/Modules/sha2module.c index 60be4228a00a035..7d6a1e40243f9d4 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -866,6 +866,7 @@ static int sha2_exec(PyObject *module) static PyModuleDef_Slot _sha2_slots[] = { {Py_mod_exec, sha2_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/sha3module.c b/Modules/sha3module.c index c30e924a7072f76..084332c1efa0e0f 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -602,6 +602,7 @@ _sha3_exec(PyObject *m) static PyModuleDef_Slot _sha3_slots[] = { {Py_mod_exec, _sha3_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 08fedeacd96d283..7de5ebe0899b358 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1698,6 +1698,7 @@ _signal_module_free(void *module) static PyModuleDef_Slot signal_slots[] = { {Py_mod_exec, signal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 7720d59e46590ea..daec560ddfcac70 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -8896,6 +8896,7 @@ socket_exec(PyObject *m) static struct PyModuleDef_Slot socket_slots[] = { {Py_mod_exec, socket_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index ddc9ac3324356dc..b4dbb54c3b47b00 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -110,6 +110,7 @@ symtable_init_constants(PyObject *m) static PyModuleDef_Slot symtable_slots[] = { {Py_mod_exec, symtable_init_constants}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index cb3f2b03990cb84..14e7ca591a076b2 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -439,6 +439,7 @@ syslog_exec(PyObject *module) static PyModuleDef_Slot syslog_slots[] = { {Py_mod_exec, syslog_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/termios.c b/Modules/termios.c index a29474d650127f1..f2c5a4bafa70124 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -1364,6 +1364,7 @@ termios_exec(PyObject *mod) static PyModuleDef_Slot termios_slots[] = { {Py_mod_exec, termios_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3211c7530da0b2f..0511339978897a0 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -2128,6 +2128,7 @@ time_module_free(void *module) static struct PyModuleDef_Slot time_slots[] = { {Py_mod_exec, time_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 6ae35b9372b830c..333ffe68a454e49 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -1668,6 +1668,7 @@ unicodedata_exec(PyObject *module) static PyModuleDef_Slot unicodedata_slots[] = { {Py_mod_exec, unicodedata_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 3357b8076b67b11..d86741e1dfc18cf 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -62,10 +62,10 @@ pass */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -395,6 +395,7 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/xxlimited_35.c b/Modules/xxlimited_35.c index 52690d9d10a81f8..1063e54217b7465 100644 --- a/Modules/xxlimited_35.c +++ b/Modules/xxlimited_35.c @@ -297,6 +297,10 @@ xx_modexec(PyObject *m) static PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_modexec}, +#ifdef Py_GIL_DISABLED + // These definitions are in the limited API, but not until 3.13. + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, +#endif {0, NULL} }; diff --git a/Modules/xxmodule.c b/Modules/xxmodule.c index 1e4e0ea3743ce3e..a46bf8f0e64ee21 100644 --- a/Modules/xxmodule.c +++ b/Modules/xxmodule.c @@ -384,6 +384,7 @@ xx_exec(PyObject *m) static struct PyModuleDef_Slot xx_slots[] = { {Py_mod_exec, xx_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/xxsubtype.c b/Modules/xxsubtype.c index 560f43e5b3a643a..9c548f44558d417 100644 --- a/Modules/xxsubtype.c +++ b/Modules/xxsubtype.c @@ -288,6 +288,7 @@ xxsubtype_exec(PyObject* m) static struct PyModuleDef_Slot xxsubtype_slots[] = { {Py_mod_exec, xxsubtype_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index fe9a6d8d4150ab9..b115f67f228ba7d 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -2106,6 +2106,7 @@ zlib_exec(PyObject *mod) static PyModuleDef_Slot zlib_slots[] = { {Py_mod_exec, zlib_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index 2f6adb9a2e12be5..d877edaf5453e10 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -249,6 +249,9 @@ _PyModule_CreateInitialized(PyModuleDef* module, int module_api_version) } } m->md_def = module; +#ifdef Py_GIL_DISABLE + m->md_gil = Py_MOD_GIL_USED; +#endif return (PyObject*)m; } @@ -261,6 +264,8 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio PyObject *m = NULL; int has_multiple_interpreters_slot = 0; void *multiple_interpreters = (void *)0; + int has_gil_slot = 0; + void *gil_slot = Py_MOD_GIL_USED; int has_execution_slots = 0; const char *name; int ret; @@ -315,6 +320,17 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio multiple_interpreters = cur_slot->value; has_multiple_interpreters_slot = 1; break; + case Py_mod_gil: + if (has_gil_slot) { + PyErr_Format( + PyExc_SystemError, + "module %s has more than one 'gil' slot", + name); + goto error; + } + gil_slot = cur_slot->value; + has_gil_slot = 1; + break; default: assert(cur_slot->slot < 0 || cur_slot->slot > _Py_mod_LAST_SLOT); PyErr_Format( @@ -374,6 +390,11 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio if (PyModule_Check(m)) { ((PyModuleObject*)m)->md_state = NULL; ((PyModuleObject*)m)->md_def = def; +#ifdef Py_GIL_DISABLED + ((PyModuleObject*)m)->md_gil = gil_slot; +#else + (void)gil_slot; +#endif } else { if (def->m_size > 0 || def->m_traverse || def->m_clear || def->m_free) { PyErr_Format( @@ -415,6 +436,19 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio return NULL; } +#ifdef Py_GIL_DISABLED +int +PyModule_ExperimentalSetGIL(PyObject *module, void *gil) +{ + if (!PyModule_Check(module)) { + PyErr_BadInternalCall(); + return -1; + } + ((PyModuleObject *)module)->md_gil = gil; + return 0; +} +#endif + int PyModule_ExecDef(PyObject *module, PyModuleDef *def) { @@ -470,6 +504,7 @@ PyModule_ExecDef(PyObject *module, PyModuleDef *def) } break; case Py_mod_multiple_interpreters: + case Py_mod_gil: /* handled in PyModule_FromDefAndSpec2 */ break; default: diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 67b1282de790e09..057b417074ebeaf 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -15541,6 +15541,7 @@ static PyMethodDef _string_methods[] = { static PyModuleDef_Slot module_slots[] = { {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/_testconsole.c b/PC/_testconsole.c index f1ace003df483b0..0dcea866f65d359 100644 --- a/PC/_testconsole.c +++ b/PC/_testconsole.c @@ -1,10 +1,10 @@ /* Testing module for multi-phase initialization of extension modules (PEP 489) */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include "Python.h" @@ -31,6 +31,7 @@ static int execfunc(PyObject *m) PyModuleDef_Slot testconsole_slots[] = { {Py_mod_exec, execfunc}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL}, }; diff --git a/PC/msvcrtmodule.c b/PC/msvcrtmodule.c index 5ff703217b421f8..b170e06b47dd599 100644 --- a/PC/msvcrtmodule.c +++ b/PC/msvcrtmodule.c @@ -656,6 +656,7 @@ exec_module(PyObject* m) static PyModuleDef_Slot msvcrt_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/winreg.c b/PC/winreg.c index 8096d17e43b7bce..efdf8addc061863 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -2179,6 +2179,7 @@ exec_module(PyObject *m) static PyModuleDef_Slot winreg_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/PC/winsound.c b/PC/winsound.c index a6b2dac6ac14669..094c77ae34d6788 100644 --- a/PC/winsound.c +++ b/PC/winsound.c @@ -35,10 +35,10 @@ winsound.PlaySound(None, 0) */ -// Need limited C API version 3.12 for Py_MOD_PER_INTERPRETER_GIL_SUPPORTED +// Need limited C API version 3.13 for Py_mod_gil #include "pyconfig.h" // Py_GIL_DISABLED #ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030c0000 +# define Py_LIMITED_API 0x030d0000 #endif #include @@ -246,6 +246,7 @@ exec_module(PyObject *module) static PyModuleDef_Slot sound_slots[] = { {Py_mod_exec, exec_module}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index c4df2c52c032bce..1f0be456655b25b 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1443,6 +1443,7 @@ def visitModule(self, mod): static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index cc7734e0dbbf9f6..1953142f6def44b 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -17588,6 +17588,7 @@ astmodule_exec(PyObject *m) static PyModuleDef_Slot astmodule_slots[] = { {Py_mod_exec, astmodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index a7891709b3b44a2..41e8107e205b469 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -322,6 +322,7 @@ static PyMethodDef tokenize_methods[] = { static PyModuleDef_Slot tokenizemodule_slots[] = { {Py_mod_exec, tokenizemodule_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index 2ba704dcaa79b20..793cbc657f31845 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1498,6 +1498,7 @@ warnings_module_exec(PyObject *module) static PyModuleDef_Slot warnings_slots[] = { {Py_mod_exec, warnings_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 722353ebcbfc3d8..88d858dc944863f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3124,6 +3124,9 @@ _PyBuiltin_Init(PyInterpreterState *interp) mod = _PyModule_CreateInitialized(&builtinsmodule, PYTHON_API_VERSION); if (mod == NULL) return NULL; +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); +#endif dict = PyModule_GetDict(mod); #ifdef Py_TRACE_REFS diff --git a/Python/import.c b/Python/import.c index f120a3841b495da..4f91f03f091edd6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -4212,6 +4212,7 @@ imp_module_exec(PyObject *module) static PyModuleDef_Slot imp_slots[] = { {Py_mod_exec, imp_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/marshal.c b/Python/marshal.c index 4bd8bb1d3a9308f..ca22d6d679a2300 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -1952,6 +1952,7 @@ marshal_module_exec(PyObject *mod) static PyModuleDef_Slot marshalmodule_slots[] = { {Py_mod_exec, marshal_module_exec}, {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index f469f165e7fb84d..645b76fccf602c5 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3756,6 +3756,9 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) if (sysmod == NULL) { return _PyStatus_ERR("failed to create a module object"); } +#ifdef Py_GIL_DISABLED + PyModule_ExperimentalSetGIL(sysmod, Py_MOD_GIL_NOT_USED); +#endif PyObject *sysdict = PyModule_GetDict(sysmod); if (sysdict == NULL) { From 608192ee2f6c60aaae1c6f2f64ea45ad2667ff2d Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 3 May 2024 11:33:05 -0400 Subject: [PATCH 086/220] gh-118527: Use deferred reference counting for C functions on modules (#118529) This addresses a scaling bottleneck in the free-threaded build when calling functions like `math.floor()` concurrently from multiple threads. --- Objects/moduleobject.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index d877edaf5453e10..a570b13e1208637 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -183,6 +183,7 @@ _add_methods_to_object(PyObject *module, PyObject *name, PyMethodDef *functions) if (func == NULL) { return -1; } + _PyObject_SetDeferredRefcount(func); if (PyObject_SetAttrString(module, fdef->ml_name, func) != 0) { Py_DECREF(func); return -1; From 9d67b72a4952766fdba803eb6eadd41dfee29dff Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Fri, 3 May 2024 11:48:27 -0400 Subject: [PATCH 087/220] docs: clarify csv.DictReader's treatment of the first data row (#118549) --- Doc/library/csv.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 4ee7820585d3a21..d17468023c6de1f 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -156,8 +156,10 @@ The :mod:`csv` module defines the following classes: The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the - dictionary preserves their original ordering. + fieldnames and will be omitted from the results. If + *fieldnames* is provided, they will be used and the first row will be + included in the results. Regardless of how the fieldnames are determined, + the dictionary preserves their original ordering. If a row has more fields than fieldnames, the remaining data is put in a list and stored with the fieldname specified by *restkey* (which defaults From 13245027526bf1b21fae6e7ca62ceec2e39fcfb7 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 3 May 2024 18:00:43 +0200 Subject: [PATCH 088/220] gh-115119: Update Windows installer to mpdecimal 4.0.0 (#118506) --- .../2024-05-02-09-28-04.gh-issue-115119.cUKMXo.rst | 1 + Misc/externals.spdx.json | 8 ++++---- PCbuild/get_externals.bat | 2 +- PCbuild/python.props | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2024-05-02-09-28-04.gh-issue-115119.cUKMXo.rst diff --git a/Misc/NEWS.d/next/Windows/2024-05-02-09-28-04.gh-issue-115119.cUKMXo.rst b/Misc/NEWS.d/next/Windows/2024-05-02-09-28-04.gh-issue-115119.cUKMXo.rst new file mode 100644 index 000000000000000..74e9b956a0feaac --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2024-05-02-09-28-04.gh-issue-115119.cUKMXo.rst @@ -0,0 +1 @@ +Update Windows installer to use libmpdecimal 4.0.0. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index bef0ce7d7e61752..58f8e0afd71f1b3 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -48,14 +48,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "93118043651ffa33dcaaab445bae4f8929fca25d2d749079b78e97f220c3d8b1" + "checksumValue": "338fac3fb8cdd60f406b6326431338756f58a8af94229ffd9bf1e7c2b1ad71ca" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-2.5.1.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/mpdecimal-4.0.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:2.5.1:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:bytereef:mpdecimal:4.0.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -63,7 +63,7 @@ "name": "mpdecimal", "originator": "Organization: bytereef.org", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.5.1" + "versionInfo": "4.0.0" }, { "SPDXID": "SPDXRef-PACKAGE-openssl", diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 8f07c414272c474..761d3de93b777d2 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,7 +54,7 @@ set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.13 -set libraries=%libraries% mpdecimal-2.5.1 +set libraries=%libraries% mpdecimal-4.0.0 set libraries=%libraries% sqlite-3.45.3.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.13.1 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.13.1 diff --git a/PCbuild/python.props b/PCbuild/python.props index 9f8b42d71cd2f3d..86fe8531d7df550 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -74,7 +74,7 @@ $(ExternalsDir)libffi-3.4.4\ $(libffiDir)$(ArchName)\ $(libffiOutDir)include - $(ExternalsDir)\mpdecimal-2.5.1\ + $(ExternalsDir)\mpdecimal-4.0.0\ $(ExternalsDir)openssl-3.0.13\ $(ExternalsDir)openssl-bin-3.0.13\$(ArchName)\ $(opensslOutDir)include From dd8f05fee26cb98770744291a007ceb6b49a5089 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Fri, 3 May 2024 18:49:40 +0200 Subject: [PATCH 089/220] gh-115119: Update macOS installer to build with libmpdec 4.0.0 (GH-118382) Co-authored-by: Ned Deily --- Mac/BuildScript/README.rst | 2 ++ Mac/BuildScript/build-installer.py | 10 ++++++++++ .../2024-05-03-12-13-27.gh-issue-115119.ltDtoR.rst | 1 + 3 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/macOS/2024-05-03-12-13-27.gh-issue-115119.ltDtoR.rst diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index 4f74e7dc00520ad..a9fae36ba28ae99 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -82,6 +82,7 @@ download them. * SQLite * XZ * libffi + * mpdecimal - uses system-supplied versions of third-party libraries @@ -111,6 +112,7 @@ download them. * SQLite * XZ * libffi + * mpdecimal - uses system-supplied versions of third-party libraries diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 286df4862793fba..8386e407f49aa3c 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -378,6 +378,15 @@ def library_recipes(): '--disable-dependency-tracking', ] ), + dict( + name="libmpdec 4.0.0", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", + checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + configure_pre=[ + "--disable-cxx", + "MACHINE=universal", + ] + ), ]) if not PYTHON_3: @@ -1150,6 +1159,7 @@ def buildPython(): print(" NOTE: --with-mimalloc=no pending resolution of weak linking issues") runCommand("%s -C --enable-framework --enable-universalsdk=/ " "--with-mimalloc=no " + "--with-system-libmpdec " "--with-universal-archs=%s " "%s " "%s " diff --git a/Misc/NEWS.d/next/macOS/2024-05-03-12-13-27.gh-issue-115119.ltDtoR.rst b/Misc/NEWS.d/next/macOS/2024-05-03-12-13-27.gh-issue-115119.ltDtoR.rst new file mode 100644 index 000000000000000..693dcc726282823 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-05-03-12-13-27.gh-issue-115119.ltDtoR.rst @@ -0,0 +1 @@ +Update macOS installer to use libmpdecimal 4.0.0. From cb57a52a85a7845b1c017085f05a7f6d71855edc Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 3 May 2024 10:49:51 -0700 Subject: [PATCH 090/220] GH-118487: Add Black to `.pre-commit-config.yaml` for JIT files (GH-118537) --- .pre-commit-config.yaml | 8 ++++++++ Tools/jit/_targets.py | 11 +++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 663a11897d98e2e..fde9d9149bf62b1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,6 +11,14 @@ repos: args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.4.2 + hooks: + - id: black + name: Run Black on Tools/jit/ + files: ^Tools/jit/ + language_version: python3.12 + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 23bb18947f80ea2..023ef498a21d7c4 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -154,13 +154,20 @@ async def _compile( args_ll = args + [ # -fomit-frame-pointer is necessary because the GHC calling # convention uses RBP to pass arguments: - "-S", "-emit-llvm", "-fomit-frame-pointer", "-o", f"{ll}", f"{c}" + "-S", + "-emit-llvm", + "-fomit-frame-pointer", + "-o", + f"{ll}", + f"{c}", ] await _llvm.run("clang", args_ll, echo=self.verbose) ir = ll.read_text() # This handles declarations, definitions, and calls to named symbols # starting with "_JIT_": - ir = re.sub(r"(((noalias|nonnull|noundef) )*ptr @_JIT_\w+\()", r"ghccc \1", ir) + ir = re.sub( + r"(((noalias|nonnull|noundef) )*ptr @_JIT_\w+\()", r"ghccc \1", ir + ) # This handles calls to anonymous callees, since anything with # "musttail" needs to use the same calling convention: ir = ir.replace("musttail call", "musttail call ghccc") From 998c3856c1e922ece806c162858dc587a1e92e02 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 3 May 2024 11:45:46 -0700 Subject: [PATCH 091/220] gh-83856: Honor atexit for all multiprocessing start methods (GH-114279) Use atexit for all multiprocessing start methods to cleanup. See the GH-114279 PR discussion and related issue for details as to why. --- Lib/multiprocessing/forkserver.py | 4 ++++ Lib/multiprocessing/popen_fork.py | 4 ++++ Lib/multiprocessing/process.py | 7 ++---- Lib/test/_test_multiprocessing.py | 23 +++++++++++++++++++ ...4-01-19-05-40-46.gh-issue-83856.jN5M80.rst | 1 + 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-01-19-05-40-46.gh-issue-83856.jN5M80.rst diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 4642707dae2f4e2..53b8c492675878a 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -271,6 +272,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +281,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47627ca..a57ef6bdad5ccc6 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 271ba3fd325138d..b45f7df476f7d8c 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -310,11 +310,8 @@ def _bootstrap(self, parent_sentinel=None): # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 5fc4181a1eeadb6..46afdfca331a23a 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6161,6 +6161,29 @@ def submain(): pass self.assertFalse(err, msg=err.decode('utf-8')) +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded diff --git a/Misc/NEWS.d/next/Library/2024-01-19-05-40-46.gh-issue-83856.jN5M80.rst b/Misc/NEWS.d/next/Library/2024-01-19-05-40-46.gh-issue-83856.jN5M80.rst new file mode 100644 index 000000000000000..b2889f216a0beb5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-01-19-05-40-46.gh-issue-83856.jN5M80.rst @@ -0,0 +1 @@ +Honor :mod:`atexit` for all :mod:`multiprocessing` start methods From 9c14ed06188aa4d462cd0fc4218c6023f9bf03cb Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Fri, 3 May 2024 11:49:24 -0700 Subject: [PATCH 092/220] gh-107674: Improve performance of `sys.settrace` (GH-117133) * Check tracing in RESUME_CHECK * Only change to RESUME_CHECK if not tracing --- ...-02-04-07-45-29.gh-issue-107674.q8mCmi.rst | 1 + Python/bytecodes.c | 33 ++++++++++--------- Python/ceval.c | 28 +++++++++------- Python/ceval_macros.h | 16 +++++---- Python/executor_cases.c.h | 2 +- Python/generated_cases.c.h | 31 ++++++++--------- Python/instrumentation.c | 4 +-- 7 files changed, 63 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst new file mode 100644 index 000000000000000..f9b96788bfad940 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-02-04-07-45-29.gh-issue-107674.q8mCmi.rst @@ -0,0 +1 @@ +Improved the performance of :func:`sys.settrace` significantly diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 9769cfe68aaeba2..e8383eda6a9d493 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -148,20 +148,18 @@ dummy_func( tier1 inst(RESUME, (--)) { assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - PyCodeObject *code = _PyFrame_GetCode(frame); - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(code, tstate->interp); - ERROR_IF(err, error); - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + ERROR_IF(err, error); + next_instr = this_instr; + DISPATCH(); } assert(this_instr->op.code == RESUME || this_instr->op.code == RESUME_CHECK || @@ -173,6 +171,9 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ } } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } } inst(RESUME_CHECK, (--)) { @@ -189,7 +190,7 @@ dummy_func( inst(INSTRUMENTED_RESUME, (--)) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { ERROR_NO_POP(); } @@ -4284,7 +4285,7 @@ dummy_func( #endif uintptr_t eval_breaker = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker); DEOPT_IF(eval_breaker & _PY_EVAL_EVENTS_MASK); - assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); } // END BYTECODES // diff --git a/Python/ceval.c b/Python/ceval.c index 59498bc826e9412..118746909902f3b 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -808,17 +808,23 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int { _Py_CODEUNIT *prev = frame->instr_ptr; _Py_CODEUNIT *here = frame->instr_ptr = next_instr; - _PyFrame_SetStackPointer(frame, stack_pointer); - int original_opcode = _Py_call_instrumentation_line( - tstate, frame, here, prev); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (original_opcode < 0) { - next_instr = here+1; - goto error; - } - next_instr = frame->instr_ptr; - if (next_instr != here) { - DISPATCH(); + int original_opcode = 0; + if (tstate->tracing) { + PyCodeObject *code = _PyFrame_GetCode(frame); + original_opcode = code->_co_monitoring->lines[(int)(here - _PyCode_CODE(code))].original_opcode; + } else { + _PyFrame_SetStackPointer(frame, stack_pointer); + original_opcode = _Py_call_instrumentation_line( + tstate, frame, here, prev); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (original_opcode < 0) { + next_instr = here+1; + goto error; + } + next_instr = frame->instr_ptr; + if (next_instr != here) { + DISPATCH(); + } } if (_PyOpcode_Caches[original_opcode]) { _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1); diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index c88a07c1f5e9514..50941e4ec473e89 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -358,12 +358,16 @@ do { \ // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ - _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ - stack_pointer = _PyFrame_GetStackPointer(frame); \ - if (next_instr == NULL) { \ - next_instr = (dest)+1; \ - goto error; \ + if (tstate->tracing) {\ + next_instr = dest; \ + } else { \ + _PyFrame_SetStackPointer(frame, stack_pointer); \ + next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \ + stack_pointer = _PyFrame_GetStackPointer(frame); \ + if (next_instr == NULL) { \ + next_instr = (dest)+1; \ + goto error; \ + } \ } \ } while (0); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 03db9b623cbd869..c3ee6e9039c9000 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4346,7 +4346,7 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - assert(eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); + assert(tstate->tracing || eval_breaker == FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version)); break; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 2a0f268ce6ed547..800d19229e3d6a5 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -3292,7 +3292,7 @@ INSTRUCTION_STATS(INSTRUMENTED_RESUME); uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); - if (code_version != global_version) { + if (code_version != global_version && tstate->tracing == 0) { if (_Py_Instrument(_PyFrame_GetCode(frame), tstate->interp)) { goto error; } @@ -4948,20 +4948,18 @@ _Py_CODEUNIT *this_instr = next_instr - 1; (void)this_instr; assert(frame == tstate->current_frame); - uintptr_t global_version = - _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & - ~_PY_EVAL_EVENTS_MASK; - PyCodeObject *code = _PyFrame_GetCode(frame); - uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); - assert((code_version & 255) == 0); - if (code_version != global_version) { - int err = _Py_Instrument(code, tstate->interp); - if (err) goto error; - next_instr = this_instr; - } - else { - if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { - CHECK_EVAL_BREAKER(); + if (tstate->tracing == 0) { + uintptr_t global_version = + _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & + ~_PY_EVAL_EVENTS_MASK; + PyCodeObject* code = _PyFrame_GetCode(frame); + uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version); + assert((code_version & 255) == 0); + if (code_version != global_version) { + int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp); + if (err) goto error; + next_instr = this_instr; + DISPATCH(); } assert(this_instr->op.code == RESUME || this_instr->op.code == RESUME_CHECK || @@ -4973,6 +4971,9 @@ #endif /* ENABLE_SPECIALIZATION */ } } + if ((oparg & RESUME_OPARG_LOCATION_MASK) < RESUME_AFTER_YIELD_FROM) { + CHECK_EVAL_BREAKER(); + } DISPATCH(); } diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 5614f4867a390d4..8085d7335fe21af 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1217,15 +1217,13 @@ int _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *prev) { PyCodeObject *code = _PyFrame_GetCode(frame); + assert(tstate->tracing == 0); assert(is_version_up_to_date(code, tstate->interp)); assert(instrumentation_cross_checks(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = &monitoring->lines[i]; - if (tstate->tracing) { - goto done; - } PyInterpreterState *interp = tstate->interp; int8_t line_delta = line_data->line_delta; int line = 0; From 52485967813acdb35c274e1b2eaedd34e9ac01fc Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Fri, 3 May 2024 12:07:10 -0700 Subject: [PATCH 093/220] GH-118251: Skip fewer test in emulated JIT CI (GH-118536) --- .github/workflows/jit.yml | 13 ++-- Tools/jit/ignore-tests-emulated-linux.txt | 79 +++++++++++++++++++++++ 2 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 Tools/jit/ignore-tests-emulated-linux.txt diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b969e6d13e61a3e..014cced8e81b522 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -75,14 +75,10 @@ jobs: architecture: aarch64 runner: ubuntu-latest compiler: gcc - # These fail because of emulation, not because of the JIT: - exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection - target: aarch64-unknown-linux-gnu/clang architecture: aarch64 runner: ubuntu-latest compiler: clang - # These fail because of emulation, not because of the JIT: - exclude: test_pathlib test_posixpath test_unix_events test_init test_process_pool test_shutdown test_multiprocessing_fork test_cmd_line test_faulthandler test_os test_perf_profiler test_posix test_signal test_socket test_subprocess test_threading test_venv test_external_inspection env: CC: ${{ matrix.compiler }} steps: @@ -97,7 +93,7 @@ jobs: choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No PGO or tests (yet): - name: Emulated Windows @@ -115,7 +111,7 @@ jobs: SDKROOT="$(xcrun --show-sdk-path)" \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} make all --jobs 4 - ./python.exe -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Native Linux @@ -125,11 +121,12 @@ jobs: export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }} make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553) - name: Emulated Linux if: runner.os == 'Linux' && matrix.architecture != 'x86_64' + # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" @@ -146,4 +143,4 @@ jobs: HOSTRUNNER=qemu-${{ matrix.architecture }} \ ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 - ./python -m test --exclude ${{ matrix.exclude }} --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt new file mode 100644 index 000000000000000..84e8c0ee8afedb5 --- /dev/null +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -0,0 +1,79 @@ +test_multiprocessing_fork +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run +test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess +test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling +test.test_cmd_line.CmdLineTest.test_no_std_streams +test.test_cmd_line.CmdLineTest.test_no_stdin +test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers +test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace +test.test_external_inspection.TestGetStackTrace.test_self_trace +test.test_faulthandler.FaultHandlerTests.test_enable_fd +test.test_faulthandler.FaultHandlerTests.test_enable_file +test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer +test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists +test.test_os.TimerfdTests.test_timerfd_initval +test.test_os.TimerfdTests.test_timerfd_interval +test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME +test.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.PosixPathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root +test.test_pathlib.test_pathlib.PathTest.test_is_mount_root +test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root +test.test_posix.TestPosixSpawn.test_close_file +test.test_posix.TestPosixSpawnP.test_close_file +test.test_posixpath.PosixPathTest.test_ismount +test.test_signal.StressTest.test_stress_modifying_handlers +test.test_socket.BasicCANTest.testFilter +test.test_socket.BasicCANTest.testLoopback +test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm +test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 +test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 +test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCase.test_empty_env +test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCase.test_one_environment_variable +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg +test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable +test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env +test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename +test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable +test.test_venv.BasicTest.test_zippath_from_non_installed_posix \ No newline at end of file From 37d095002216a4e45b4e82539ca0421ded8aaae3 Mon Sep 17 00:00:00 2001 From: mpage Date: Fri, 3 May 2024 13:21:04 -0700 Subject: [PATCH 094/220] gh-117657: Disable the function/code cache in free-threaded builds (#118301) This is only used by the specializing interpreter and the tier 2 optimizer, both of which are disabled in free-threaded builds. --- Include/internal/pycore_function.h | 7 +++++++ Objects/codeobject.c | 6 ++++++ Objects/funcobject.c | 10 ++++++++++ Tools/tsan/suppressions_free_threading.txt | 1 - 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 24fbb3ddbee602c..6d44e933e8a8cb3 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include "pycore_lock.h" + #ifndef Py_BUILD_CORE # error "this header requires Py_BUILD_CORE define" #endif @@ -24,6 +26,11 @@ struct _func_version_cache_item { }; struct _py_func_state { +#ifdef Py_GIL_DISABLED + // Protects next_version + PyMutex mutex; +#endif + uint32_t next_version; // Borrowed references to function and code objects whose // func_version % FUNC_VERSION_CACHE_SIZE diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 810f847485acdab..7d02b038cf52caf 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -416,10 +416,16 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_ncellvars = ncellvars; co->co_nfreevars = nfreevars; PyInterpreterState *interp = _PyInterpreterState_GET(); +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&interp->func_state.mutex); +#endif co->co_version = interp->func_state.next_version; if (interp->func_state.next_version != 0) { interp->func_state.next_version++; } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&interp->func_state.mutex); +#endif co->_co_monitoring = NULL; co->_co_instrumentation_version = 0; /* not set */ diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 276b3db29703711..8a30213888ef870 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -287,6 +287,7 @@ functions is running. void _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) { +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); if (func->func_version != 0) { struct _func_version_cache_item *slot = @@ -297,7 +298,9 @@ _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) // Leave slot->code alone, there may be use for it. } } +#endif func->func_version = version; +#ifndef Py_GIL_DISABLED if (version != 0) { struct _func_version_cache_item *slot = interp->func_state.func_version_cache @@ -305,11 +308,13 @@ _PyFunction_SetVersion(PyFunctionObject *func, uint32_t version) slot->func = func; slot->code = func->func_code; } +#endif } void _PyFunction_ClearCodeByVersion(uint32_t version) { +#ifndef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); struct _func_version_cache_item *slot = interp->func_state.func_version_cache @@ -322,11 +327,15 @@ _PyFunction_ClearCodeByVersion(uint32_t version) slot->func = NULL; } } +#endif } PyFunctionObject * _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) { +#ifdef Py_GIL_DISABLED + return NULL; +#else PyInterpreterState *interp = _PyInterpreterState_GET(); struct _func_version_cache_item *slot = interp->func_state.func_version_cache @@ -346,6 +355,7 @@ _PyFunction_LookupByVersion(uint32_t version, PyObject **p_code) return slot->func; } return NULL; +#endif } uint32_t diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 4f6648a75731840..74dbf4bb1cb6882 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -15,7 +15,6 @@ race:_add_to_weak_set race:_in_weak_set race:_mi_heap_delayed_free_partial race:_PyEval_EvalFrameDefault -race:_PyFunction_SetVersion race:_PyImport_AcquireLock race:_PyImport_ReleaseLock race:_PyInterpreterState_SetNotRunningMain From a40f557d7b7a355a55bb90c068e3e9202fd9c8f2 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 3 May 2024 21:29:25 +0100 Subject: [PATCH 095/220] GH-116380: Move pathlib globbing implementation into `pathlib._glob` (#118562) Moving this code under the `pathlib` package makes it quite a lot easier to backport in the `pathlib-abc` PyPI package. It was a bit foolish of me to add it to `glob` in the first place. Also add `translate()` to `__all__` in `glob`. This function is new in 3.13, so there's no NEWS needed. --- Lib/glob.py | 306 +--------------------------------------- Lib/pathlib/__init__.py | 5 +- Lib/pathlib/_abc.py | 7 +- Lib/pathlib/_glob.py | 305 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 314 insertions(+), 309 deletions(-) create mode 100644 Lib/pathlib/_glob.py diff --git a/Lib/glob.py b/Lib/glob.py index 72cf22299763f04..9fe0e9f98ac5a7d 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -2,15 +2,14 @@ import contextlib import os -import re import fnmatch -import functools import itertools -import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] +from pathlib._glob import translate, magic_check, magic_check_bytes + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -226,9 +225,6 @@ def _join(dirname, basename): return dirname or basename return os.path.join(dirname, basename) -magic_check = re.compile('([*?[])') -magic_check_bytes = re.compile(b'([*?[])') - def has_magic(s): if isinstance(s, bytes): match = magic_check_bytes.search(s) @@ -258,300 +254,4 @@ def escape(pathname): return drive + pathname -_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) -_no_recurse_symlinks = object() - - -def translate(pat, *, recursive=False, include_hidden=False, seps=None): - """Translate a pathname with shell wildcards to a regular expression. - - If `recursive` is true, the pattern segment '**' will match any number of - path segments. - - If `include_hidden` is true, wildcards can match path segments beginning - with a dot ('.'). - - If a sequence of separator characters is given to `seps`, they will be - used to split the pattern into segments and match path separators. If not - given, os.path.sep and os.path.altsep (where available) are used. - """ - if not seps: - if os.path.altsep: - seps = (os.path.sep, os.path.altsep) - else: - seps = os.path.sep - escaped_seps = ''.join(map(re.escape, seps)) - any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps - not_sep = f'[^{escaped_seps}]' - if include_hidden: - one_last_segment = f'{not_sep}+' - one_segment = f'{one_last_segment}{any_sep}' - any_segments = f'(?:.+{any_sep})?' - any_last_segments = '.*' - else: - one_last_segment = f'[^{escaped_seps}.]{not_sep}*' - one_segment = f'{one_last_segment}{any_sep}' - any_segments = f'(?:{one_segment})*' - any_last_segments = f'{any_segments}(?:{one_last_segment})?' - - results = [] - parts = re.split(any_sep, pat) - last_part_idx = len(parts) - 1 - for idx, part in enumerate(parts): - if part == '*': - results.append(one_segment if idx < last_part_idx else one_last_segment) - elif recursive and part == '**': - if idx < last_part_idx: - if parts[idx + 1] != '**': - results.append(any_segments) - else: - results.append(any_last_segments) - else: - if part: - if not include_hidden and part[0] in '*?': - results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) - if idx < last_part_idx: - results.append(any_sep) - res = ''.join(results) - return fr'(?s:{res})\Z' - - -@functools.lru_cache(maxsize=512) -def _compile_pattern(pat, sep, case_sensitive, recursive=True): - """Compile given glob pattern to a re.Pattern object (observing case - sensitivity).""" - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) - return re.compile(regex, flags=flags).match - - -class _Globber: - """Class providing shell-style pattern matching and globbing. - """ - - def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): - self.sep = sep - self.case_sensitive = case_sensitive - self.case_pedantic = case_pedantic - self.recursive = recursive - - # Low-level methods - - lstat = staticmethod(os.lstat) - scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') - concat_path = operator.add - - if os.name == 'nt': - @staticmethod - def add_slash(pathname): - tail = os.path.splitroot(pathname)[2] - if not tail or tail[-1] in '\\/': - return pathname - return f'{pathname}\\' - else: - @staticmethod - def add_slash(pathname): - if not pathname or pathname[-1] == '/': - return pathname - return f'{pathname}/' - - # High-level methods - - def compile(self, pat): - return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) - - def selector(self, parts): - """Returns a function that selects from a given path, walking and - filtering according to the glob-style pattern parts in *parts*. - """ - if not parts: - return self.select_exists - part = parts.pop() - if self.recursive and part == '**': - selector = self.recursive_selector - elif part in _special_parts: - selector = self.special_selector - elif not self.case_pedantic and magic_check.search(part) is None: - selector = self.literal_selector - else: - selector = self.wildcard_selector - return selector(part, parts) - - def special_selector(self, part, parts): - """Returns a function that selects special children of the given path. - """ - select_next = self.selector(parts) - - def select_special(path, exists=False): - path = self.concat_path(self.add_slash(path), part) - return select_next(path, exists) - return select_special - - def literal_selector(self, part, parts): - """Returns a function that selects a literal descendant of a path. - """ - - # Optimization: consume and join any subsequent literal parts here, - # rather than leaving them for the next selector. This reduces the - # number of string concatenation operations and calls to add_slash(). - while parts and magic_check.search(parts[-1]) is None: - part += self.sep + parts.pop() - - select_next = self.selector(parts) - - def select_literal(path, exists=False): - path = self.concat_path(self.add_slash(path), part) - return select_next(path, exists=False) - return select_literal - - def wildcard_selector(self, part, parts): - """Returns a function that selects direct children of a given path, - filtering by pattern. - """ - - match = None if part == '*' else self.compile(part) - dir_only = bool(parts) - if dir_only: - select_next = self.selector(parts) - - def select_wildcard(path, exists=False): - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - if match is None or match(entry.name): - if dir_only: - try: - if not entry.is_dir(): - continue - except OSError: - continue - entry_path = self.parse_entry(entry) - if dir_only: - yield from select_next(entry_path, exists=True) - else: - yield entry_path - return select_wildcard - - def recursive_selector(self, part, parts): - """Returns a function that selects a given path and all its children, - recursively, filtering by pattern. - """ - # Optimization: consume following '**' parts, which have no effect. - while parts and parts[-1] == '**': - parts.pop() - - # Optimization: consume and join any following non-special parts here, - # rather than leaving them for the next selector. They're used to - # build a regular expression, which we use to filter the results of - # the recursive walk. As a result, non-special pattern segments - # following a '**' wildcard don't require additional filesystem access - # to expand. - follow_symlinks = self.recursive is not _no_recurse_symlinks - if follow_symlinks: - while parts and parts[-1] not in _special_parts: - part += self.sep + parts.pop() - - match = None if part == '**' else self.compile(part) - dir_only = bool(parts) - select_next = self.selector(parts) - - def select_recursive(path, exists=False): - path = self.add_slash(path) - match_pos = len(str(path)) - if match is None or match(str(path), match_pos): - yield from select_next(path, exists) - stack = [path] - while stack: - yield from select_recursive_step(stack, match_pos) - - def select_recursive_step(stack, match_pos): - path = stack.pop() - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - is_dir = False - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - is_dir = True - except OSError: - pass - - if is_dir or not dir_only: - entry_path = self.parse_entry(entry) - if match is None or match(str(entry_path), match_pos): - if dir_only: - yield from select_next(entry_path, exists=True) - else: - # Optimization: directly yield the path if this is - # last pattern part. - yield entry_path - if is_dir: - stack.append(entry_path) - - return select_recursive - - def select_exists(self, path, exists=False): - """Yields the given path, if it exists. - """ - if exists: - # Optimization: this path is already known to exist, e.g. because - # it was returned from os.scandir(), so we skip calling lstat(). - yield path - else: - try: - self.lstat(path) - yield path - except OSError: - pass - - @classmethod - def walk(cls, root, top_down, on_error, follow_symlinks): - """Walk the directory tree from the given root, similar to os.walk(). - """ - paths = [root] - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - try: - with cls.scandir(path) as scandir_it: - dirnames = [] - filenames = [] - if not top_down: - paths.append((path, dirnames, filenames)) - for entry in scandir_it: - name = entry.name - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - if not top_down: - paths.append(cls.parse_entry(entry)) - dirnames.append(name) - else: - filenames.append(name) - except OSError: - filenames.append(name) - except OSError as error: - if on_error is not None: - on_error(error) - else: - if top_down: - yield path, dirnames, filenames - if dirnames: - prefix = cls.add_slash(path) - paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 8eecf2cefb790fa..381b08cbe8e01f2 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,7 +5,6 @@ operating systems. """ -import glob import io import ntpath import operator @@ -25,7 +24,7 @@ except ImportError: grp = None -from . import _abc +from . import _abc, _glob __all__ = [ @@ -113,7 +112,7 @@ class PurePath(_abc.PurePathBase): '_hash', ) parser = os.path - _globber = glob._Globber + _globber = _glob.Globber def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index c7e8e2f68ed0587..591df443be093af 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,11 +12,12 @@ """ import functools -import glob import operator from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from . import _glob + # # Internals # @@ -43,7 +44,7 @@ def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' -class Globber(glob._Globber): +class Globber(_glob.Globber): lstat = operator.methodcaller('lstat') add_slash = operator.methodcaller('joinpath', '') @@ -692,7 +693,7 @@ def _glob_selector(self, parts, case_sensitive, recurse_symlinks): # know the case sensitivity of the underlying filesystem, so we # must use scandir() for everything, including non-wildcard parts. case_pedantic = True - recursive = True if recurse_symlinks else glob._no_recurse_symlinks + recursive = True if recurse_symlinks else _glob.no_recurse_symlinks globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) return globber.selector(parts) diff --git a/Lib/pathlib/_glob.py b/Lib/pathlib/_glob.py new file mode 100644 index 000000000000000..73ccc0677920ce9 --- /dev/null +++ b/Lib/pathlib/_glob.py @@ -0,0 +1,305 @@ +import os +import re +import fnmatch +import functools +import operator + + +special_parts = ('', '.', '..') +magic_check = re.compile('([*?[])') +magic_check_bytes = re.compile(b'([*?[])') +no_recurse_symlinks = object() + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class Globber: + """Class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Low-level methods + + lstat = staticmethod(os.lstat) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' + + # High-level methods + + def compile(self, pat): + return compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.parse_entry(entry) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.parse_entry(entry) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + else: + try: + self.lstat(path) + yield path + except OSError: + pass + + @classmethod + def walk(cls, root, top_down, on_error, follow_symlinks): + """Walk the directory tree from the given root, similar to os.walk(). + """ + paths = [root] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + try: + with cls.scandir(path) as scandir_it: + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + for entry in scandir_it: + name = entry.name + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(cls.parse_entry(entry)) + dirnames.append(name) + else: + filenames.append(name) + except OSError: + filenames.append(name) + except OSError as error: + if on_error is not None: + on_error(error) + else: + if top_down: + yield path, dirnames, filenames + if dirnames: + prefix = cls.add_slash(path) + paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] From 6d9e8e989e9e0931f84155451f921621b906e213 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 May 2024 17:07:47 -0500 Subject: [PATCH 096/220] Minor improvements to the itertools recipes (#118563) --- Doc/library/itertools.rst | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 9a5cb8be37d3496..820906085151a32 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -826,10 +826,7 @@ and :term:`generators ` which incur interpreter overhead. return map(function, count(start)) def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ + "Repeat calls to func with specified arguments." if times is None: return starmap(func, repeat(args)) return starmap(func, repeat(args, times)) @@ -851,10 +848,8 @@ and :term:`generators ` which incur interpreter overhead. "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: - # feed the entire iterator into a zero-length deque collections.deque(iterator, maxlen=0) else: - # advance to the empty slice starting at position n next(islice(iterator, n, n), None) def nth(iterable, n, default=None): @@ -873,7 +868,7 @@ and :term:`generators ` which incur interpreter overhead. def all_equal(iterable, key=None): "Returns True if all the elements are equal to each other." - # all_equal('4٤໔４৪', key=int) → True + # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 def unique_justseen(iterable, key=None): @@ -903,9 +898,9 @@ and :term:`generators ` which incur interpreter overhead. def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n-1), maxlen=n) - for x in it: + iterator = iter(iterable) + window = collections.deque(islice(iterator, n - 1), maxlen=n) + for x in iterator: window.append(x) yield tuple(window) @@ -955,8 +950,8 @@ and :term:`generators ` which incur interpreter overhead. seq_index = getattr(iterable, 'index', None) if seq_index is None: # Path for general iterables - it = islice(iterable, start, stop) - for i, element in enumerate(it, start): + iterator = islice(iterable, start, stop) + for i, element in enumerate(iterator, start): if element is value or element == value: yield i else: @@ -971,10 +966,7 @@ and :term:`generators ` which incur interpreter overhead. pass def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - """ + "Convert a call-until-exception interface to an iterator interface." # iter_except(d.popitem, KeyError) → non-blocking dictionary iterator try: if first is not None: @@ -1074,14 +1066,10 @@ The following recipes have a more mathematical flavor: # sieve(30) → 2 3 5 7 11 13 17 19 23 29 if n > 2: yield 2 - start = 3 data = bytearray((0, 1)) * (n // 2) - limit = math.isqrt(n) + 1 - for p in iter_index(data, 1, start, limit): - yield from iter_index(data, 1, start, p*p) + for p in iter_index(data, 1, start=3, stop=math.isqrt(n) + 1): data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - start = p*p - yield from iter_index(data, 1, start) + yield from iter_index(data, 1, start=3) def factor(n): "Prime factors of n." @@ -1101,8 +1089,8 @@ The following recipes have a more mathematical flavor: "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html # totient(12) → 4 because len([1, 5, 7, 11]) == 4 - for p in unique_justseen(factor(n)): - n -= n // p + for prime in set(factor(n)): + n -= n // prime return n From 0e78a545e6e78b16eb7709ec3db2243364603134 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 3 May 2024 18:13:40 -0400 Subject: [PATCH 097/220] gh-118534: Fix load of `gil->locked` (#118553) --- Python/ceval_gil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index e4f7217255efd83..b456a9d80455e4f 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -224,7 +224,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) return; } #endif - if (!_Py_atomic_load_ptr_relaxed(&gil->locked)) { + if (!_Py_atomic_load_int_relaxed(&gil->locked)) { Py_FatalError("drop_gil: GIL is not locked"); } From 37c31bea72fb6971f1008faa2e253d12167c221f Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Fri, 3 May 2024 18:16:45 -0400 Subject: [PATCH 098/220] gh-118527: Intern filename, name, and qualname in code objects. (#118558) This interns the strings for `co_filename`, `co_name`, and `co_qualname` on codeobjects in the free-threaded build. This partially addresses a reference counting bottleneck when creating closures concurrently. The closures take the name and qualified name from the code object. --- Objects/codeobject.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 7d02b038cf52caf..15d3960fc2df512 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -390,6 +390,11 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); +#ifdef Py_GIL_DISABLED + PyUnicode_InternInPlace(&co->co_filename); + PyUnicode_InternInPlace(&co->co_name); + PyUnicode_InternInPlace(&co->co_qualname); +#endif co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; From 139dc487b5ac37b0c2c4b93f2bfba194507d8310 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 3 May 2024 16:10:02 -0700 Subject: [PATCH 099/220] GH-118251: Fix incomplete ternary expression in JIT workflow (GH-118564) --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 014cced8e81b522..f68567c8a3153e8 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -93,7 +93,7 @@ jobs: choco upgrade llvm -y choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} - ./PCbuild/rt.bat ${{ matrix.debug && '-d' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 # No PGO or tests (yet): - name: Emulated Windows From 1b7e5e6e60e0d22b2a928cbbb36ebb989183450f Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Fri, 3 May 2024 16:41:07 -0700 Subject: [PATCH 100/220] GH-113464: Generate a more efficient JIT (GH-118512) --- Python/jit.c | 587 +++++++++++++++++++---------------------- Tools/jit/_stencils.py | 116 +++++++- Tools/jit/_writer.py | 107 +++----- 3 files changed, 418 insertions(+), 392 deletions(-) diff --git a/Python/jit.c b/Python/jit.c index df14e48c564447a..7c316a410dda6a1 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -16,8 +16,6 @@ #include "pycore_sliceobject.h" #include "pycore_jit.h" -#include "jit_stencils.h" - // Memory management stuff: //////////////////////////////////////////////////// #ifndef MS_WINDOWS @@ -146,256 +144,275 @@ set_bits(uint32_t *loc, uint8_t loc_start, uint64_t value, uint8_t value_start, #define IS_AARCH64_LDR_OR_STR(I) (((I) & 0x3B000000) == 0x39000000) #define IS_AARCH64_MOV(I) (((I) & 0x9F800000) == 0x92800000) -// Fill all of stencil's holes in the memory pointed to by base, using the -// values in patches. -static void -patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) +// LLD is a great reference for performing relocations... just keep in +// mind that Tools/jit/build.py does filtering and preprocessing for us! +// Here's a good place to start for each platform: +// - aarch64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h +// - aarch64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - aarch64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp +// - i686-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-apple-darwin: +// - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp +// - x86_64-pc-windows-msvc: +// - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp +// - x86_64-unknown-linux-gnu: +// - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp + +// Many of these patches are "relaxing", meaning that they can rewrite the +// code they're patching to be more efficient (like turning a 64-bit memory +// load into a 32-bit immediate load). These patches have an "x" in their name. +// Relative patches have an "r" in their name. + +// 32-bit absolute address. +void +patch_32(unsigned char *location, uint64_t value) { - for (size_t i = 0; i < stencil->holes_size; i++) { - const Hole *hole = &stencil->holes[i]; - unsigned char *location = base + hole->offset; - uint64_t value = patches[hole->value] + (uintptr_t)hole->symbol + hole->addend; - uint8_t *loc8 = (uint8_t *)location; - uint32_t *loc32 = (uint32_t *)location; - uint64_t *loc64 = (uint64_t *)location; - // LLD is a great reference for performing relocations... just keep in - // mind that Tools/jit/build.py does filtering and preprocessing for us! - // Here's a good place to start for each platform: - // - aarch64-apple-darwin: - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64.cpp - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.cpp - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/ARM64Common.h - // - aarch64-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - aarch64-unknown-linux-gnu: - // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/AArch64.cpp - // - i686-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - x86_64-apple-darwin: - // - https://github.com/llvm/llvm-project/blob/main/lld/MachO/Arch/X86_64.cpp - // - x86_64-pc-windows-msvc: - // - https://github.com/llvm/llvm-project/blob/main/lld/COFF/Chunks.cpp - // - x86_64-unknown-linux-gnu: - // - https://github.com/llvm/llvm-project/blob/main/lld/ELF/Arch/X86_64.cpp - switch (hole->kind) { - case HoleKind_IMAGE_REL_I386_DIR32: - // 32-bit absolute address. - // Check that we're not out of range of 32 unsigned bits: - assert(value < (1ULL << 32)); - *loc32 = (uint32_t)value; - continue; - case HoleKind_ARM64_RELOC_UNSIGNED: - case HoleKind_R_AARCH64_ABS64: - case HoleKind_X86_64_RELOC_UNSIGNED: - case HoleKind_R_X86_64_64: - // 64-bit absolute address. - *loc64 = value; - continue; - case HoleKind_IMAGE_REL_AMD64_REL32: - case HoleKind_IMAGE_REL_I386_REL32: - case HoleKind_R_X86_64_GOTPCRELX: - case HoleKind_R_X86_64_REX_GOTPCRELX: - case HoleKind_X86_64_RELOC_GOT: - case HoleKind_X86_64_RELOC_GOT_LOAD: { - // 32-bit relative address. - // Try to relax the GOT load into an immediate value: - uint64_t relaxed = *(uint64_t *)(value + 4) - 4; - if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && - (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) - { - if (loc8[-2] == 0x8B) { - // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] - loc8[-2] = 0x8D; - value = relaxed; - } - else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { - // call qword ptr [rip + AAA] -> nop; call XXX - loc8[-2] = 0x90; - loc8[-1] = 0xE8; - value = relaxed; - } - else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { - // jmp qword ptr [rip + AAA] -> nop; jmp XXX - loc8[-2] = 0x90; - loc8[-1] = 0xE9; - value = relaxed; - } - } - } - // Fall through... - case HoleKind_R_X86_64_GOTPCREL: - case HoleKind_R_X86_64_PC32: - case HoleKind_X86_64_RELOC_SIGNED: - case HoleKind_X86_64_RELOC_BRANCH: - // 32-bit relative address. - value -= (uintptr_t)location; - // Check that we're not out of range of 32 signed bits: - assert((int64_t)value >= -(1LL << 31)); - assert((int64_t)value < (1LL << 31)); - *loc32 = (uint32_t)value; - continue; - case HoleKind_ARM64_RELOC_BRANCH26: - case HoleKind_IMAGE_REL_ARM64_BRANCH26: - case HoleKind_R_AARCH64_CALL26: - case HoleKind_R_AARCH64_JUMP26: - // 28-bit relative branch. - assert(IS_AARCH64_BRANCH(*loc32)); - value -= (uintptr_t)location; - // Check that we're not out of range of 28 signed bits: - assert((int64_t)value >= -(1 << 27)); - assert((int64_t)value < (1 << 27)); - // Since instructions are 4-byte aligned, only use 26 bits: - assert(get_bits(value, 0, 2) == 0); - set_bits(loc32, 0, value, 2, 26); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G0_NC: - // 16-bit low part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 0 of 3"): - assert(get_bits(*loc32, 21, 2) == 0); - set_bits(loc32, 5, value, 0, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G1_NC: - // 16-bit middle-low part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 1 of 3"): - assert(get_bits(*loc32, 21, 2) == 1); - set_bits(loc32, 5, value, 16, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G2_NC: - // 16-bit middle-high part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 2 of 3"): - assert(get_bits(*loc32, 21, 2) == 2); - set_bits(loc32, 5, value, 32, 16); - continue; - case HoleKind_R_AARCH64_MOVW_UABS_G3: - // 16-bit high part of an absolute address. - assert(IS_AARCH64_MOV(*loc32)); - // Check the implicit shift (this is "part 3 of 3"): - assert(get_bits(*loc32, 21, 2) == 3); - set_bits(loc32, 5, value, 48, 16); - continue; - case HoleKind_ARM64_RELOC_GOT_LOAD_PAGE21: - case HoleKind_IMAGE_REL_ARM64_PAGEBASE_REL21: - case HoleKind_R_AARCH64_ADR_GOT_PAGE: - case HoleKind_R_AARCH64_ADR_PREL_PG_HI21: - // 21-bit count of pages between this page and an absolute address's - // page... I know, I know, it's weird. Pairs nicely with - // ARM64_RELOC_GOT_LOAD_PAGEOFF12 (below). - assert(IS_AARCH64_ADRP(*loc32)); - // Try to relax the pair of GOT loads into an immediate value: - const Hole *next_hole = &stencil->holes[i + 1]; - if (i + 1 < stencil->holes_size && - (next_hole->kind == HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12 || - next_hole->kind == HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L || - next_hole->kind == HoleKind_R_AARCH64_LD64_GOT_LO12_NC) && - next_hole->offset == hole->offset + 4 && - next_hole->symbol == hole->symbol && - next_hole->addend == hole->addend && - next_hole->value == hole->value) - { - unsigned char reg = get_bits(loc32[0], 0, 5); - assert(IS_AARCH64_LDR_OR_STR(loc32[1])); - // There should be only one register involved: - assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. - assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. - uint64_t relaxed = *(uint64_t *)value; - if (relaxed < (1UL << 16)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xD503201F; - i++; - continue; - } - if (relaxed < (1ULL << 32)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; - i++; - continue; - } - relaxed = value - (uintptr_t)location; - if ((relaxed & 0x3) == 0 && - (int64_t)relaxed >= -(1L << 19) && - (int64_t)relaxed < (1L << 19)) - { - // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop - loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; - loc32[1] = 0xD503201F; - i++; - continue; - } - } - // Fall through... - case HoleKind_ARM64_RELOC_PAGE21: - // Number of pages between this page and the value's page: - value = (value >> 12) - ((uintptr_t)location >> 12); - // Check that we're not out of range of 21 signed bits: - assert((int64_t)value >= -(1 << 20)); - assert((int64_t)value < (1 << 20)); - // value[0:2] goes in loc[29:31]: - set_bits(loc32, 29, value, 0, 2); - // value[2:21] goes in loc[5:26]: - set_bits(loc32, 5, value, 2, 19); - continue; - case HoleKind_ARM64_RELOC_GOT_LOAD_PAGEOFF12: - case HoleKind_ARM64_RELOC_PAGEOFF12: - case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12A: - case HoleKind_IMAGE_REL_ARM64_PAGEOFFSET_12L: - case HoleKind_R_AARCH64_ADD_ABS_LO12_NC: - case HoleKind_R_AARCH64_LD64_GOT_LO12_NC: - // 12-bit low part of an absolute address. Pairs nicely with - // ARM64_RELOC_GOT_LOAD_PAGE21 (above). - assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); - // There might be an implicit shift encoded in the instruction: - uint8_t shift = 0; - if (IS_AARCH64_LDR_OR_STR(*loc32)) { - shift = (uint8_t)get_bits(*loc32, 30, 2); - // If both of these are set, the shift is supposed to be 4. - // That's pretty weird, and it's never actually been observed... - assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); - } - value = get_bits(value, 0, 12); - assert(get_bits(value, 0, shift) == 0); - set_bits(loc32, 10, value, shift, 12); - continue; - } - Py_UNREACHABLE(); + uint32_t *loc32 = (uint32_t *)location; + // Check that we're not out of range of 32 unsigned bits: + assert(value < (1ULL << 32)); + *loc32 = (uint32_t)value; +} + +// 32-bit relative address. +void +patch_32r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value -= (uintptr_t)location; + // Check that we're not out of range of 32 signed bits: + assert((int64_t)value >= -(1LL << 31)); + assert((int64_t)value < (1LL << 31)); + *loc32 = (uint32_t)value; +} + +// 64-bit absolute address. +void +patch_64(unsigned char *location, uint64_t value) +{ + uint64_t *loc64 = (uint64_t *)location; + *loc64 = value; +} + +// 12-bit low part of an absolute address. Pairs nicely with patch_aarch64_21r +// (below). +void +patch_aarch64_12(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_LDR_OR_STR(*loc32) || IS_AARCH64_ADD_OR_SUB(*loc32)); + // There might be an implicit shift encoded in the instruction: + uint8_t shift = 0; + if (IS_AARCH64_LDR_OR_STR(*loc32)) { + shift = (uint8_t)get_bits(*loc32, 30, 2); + // If both of these are set, the shift is supposed to be 4. + // That's pretty weird, and it's never actually been observed... + assert(get_bits(*loc32, 23, 1) == 0 || get_bits(*loc32, 26, 1) == 0); } + value = get_bits(value, 0, 12); + assert(get_bits(value, 0, shift) == 0); + set_bits(loc32, 10, value, shift, 12); } -static void -copy_and_patch(unsigned char *base, const Stencil *stencil, uintptr_t patches[]) +// Relaxable 12-bit low part of an absolute address. Pairs nicely with +// patch_aarch64_21rx (below). +void +patch_aarch64_12x(unsigned char *location, uint64_t value) { - memcpy(base, stencil->body, stencil->body_size); - patch(base, stencil, patches); + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_21rx. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_12(location, value); } -static void -emit(const StencilGroup *group, uintptr_t patches[]) +// 16-bit low part of an absolute address. +void +patch_aarch64_16a(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 0 of 3"): + assert(get_bits(*loc32, 21, 2) == 0); + set_bits(loc32, 5, value, 0, 16); +} + +// 16-bit middle-low part of an absolute address. +void +patch_aarch64_16b(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 1 of 3"): + assert(get_bits(*loc32, 21, 2) == 1); + set_bits(loc32, 5, value, 16, 16); +} + +// 16-bit middle-high part of an absolute address. +void +patch_aarch64_16c(unsigned char *location, uint64_t value) { - copy_and_patch((unsigned char *)patches[HoleValue_DATA], &group->data, patches); - copy_and_patch((unsigned char *)patches[HoleValue_CODE], &group->code, patches); + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 2 of 3"): + assert(get_bits(*loc32, 21, 2) == 2); + set_bits(loc32, 5, value, 32, 16); } +// 16-bit high part of an absolute address. +void +patch_aarch64_16d(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_MOV(*loc32)); + // Check the implicit shift (this is "part 3 of 3"): + assert(get_bits(*loc32, 21, 2) == 3); + set_bits(loc32, 5, value, 48, 16); +} + +// 21-bit count of pages between this page and an absolute address's page... I +// know, I know, it's weird. Pairs nicely with patch_aarch64_12 (above). +void +patch_aarch64_21r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + value = (value >> 12) - ((uintptr_t)location >> 12); + // Check that we're not out of range of 21 signed bits: + assert((int64_t)value >= -(1 << 20)); + assert((int64_t)value < (1 << 20)); + // value[0:2] goes in loc[29:31]: + set_bits(loc32, 29, value, 0, 2); + // value[2:21] goes in loc[5:26]: + set_bits(loc32, 5, value, 2, 19); +} + +// Relaxable 21-bit count of pages between this page and an absolute address's +// page. Pairs nicely with patch_aarch64_12x (above). +void +patch_aarch64_21rx(unsigned char *location, uint64_t value) +{ + // This can *only* be relaxed if it occurs immediately before a matching + // patch_aarch64_12x. If that happens, the JIT build step will replace both + // calls with a single call to patch_aarch64_33rx. Otherwise, we end up + // here, and the instruction is patched normally: + patch_aarch64_21r(location, value); +} + +// 28-bit relative branch. +void +patch_aarch64_26r(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + assert(IS_AARCH64_BRANCH(*loc32)); + value -= (uintptr_t)location; + // Check that we're not out of range of 28 signed bits: + assert((int64_t)value >= -(1 << 27)); + assert((int64_t)value < (1 << 27)); + // Since instructions are 4-byte aligned, only use 26 bits: + assert(get_bits(value, 0, 2) == 0); + set_bits(loc32, 0, value, 2, 26); +} + +// A pair of patch_aarch64_21rx and patch_aarch64_12x. +void +patch_aarch64_33rx(unsigned char *location, uint64_t value) +{ + uint32_t *loc32 = (uint32_t *)location; + // Try to relax the pair of GOT loads into an immediate value: + assert(IS_AARCH64_ADRP(*loc32)); + unsigned char reg = get_bits(loc32[0], 0, 5); + assert(IS_AARCH64_LDR_OR_STR(loc32[1])); + // There should be only one register involved: + assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. + assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. + uint64_t relaxed = *(uint64_t *)value; + if (relaxed < (1UL << 16)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + if (relaxed < (1ULL << 32)) { + // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY + loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; + loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; + return; + } + relaxed = value - (uintptr_t)location; + if ((relaxed & 0x3) == 0 && + (int64_t)relaxed >= -(1L << 19) && + (int64_t)relaxed < (1L << 19)) + { + // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop + loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; + loc32[1] = 0xD503201F; + return; + } + // Couldn't do it. Just patch the two instructions normally: + patch_aarch64_21rx(location, value); + patch_aarch64_12x(location + 4, value); +} + +// Relaxable 32-bit relative address. +void +patch_x86_64_32rx(unsigned char *location, uint64_t value) +{ + uint8_t *loc8 = (uint8_t *)location; + // Try to relax the GOT load into an immediate value: + uint64_t relaxed = *(uint64_t *)(value + 4) - 4; + if ((int64_t)relaxed - (int64_t)location >= -(1LL << 31) && + (int64_t)relaxed - (int64_t)location + 1 < (1LL << 31)) + { + if (loc8[-2] == 0x8B) { + // mov reg, dword ptr [rip + AAA] -> lea reg, [rip + XXX] + loc8[-2] = 0x8D; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x15) { + // call qword ptr [rip + AAA] -> nop; call XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE8; + value = relaxed; + } + else if (loc8[-2] == 0xFF && loc8[-1] == 0x25) { + // jmp qword ptr [rip + AAA] -> nop; jmp XXX + loc8[-2] = 0x90; + loc8[-1] = 0xE9; + value = relaxed; + } + } + patch_32r(location, value); +} + +#include "jit_stencils.h" + // Compiles executor in-place. Don't forget to call _PyJIT_Free later! int -_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size_t length) +_PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], size_t length) { + const StencilGroup *group; // Loop once to find the total compiled size: - size_t instruction_starts[UOP_MAX_TRACE_LENGTH]; - size_t code_size = trampoline.code.body_size; - size_t data_size = trampoline.data.body_size; + uintptr_t instruction_starts[UOP_MAX_TRACE_LENGTH]; + size_t code_size = 0; + size_t data_size = 0; + group = &trampoline; + code_size += group->code_size; + data_size += group->data_size; for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; - const StencilGroup *group = &stencil_groups[instruction->opcode]; + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; instruction_starts[i] = code_size; - code_size += group->code.body_size; - data_size += group->data.body_size; + code_size += group->code_size; + data_size += group->data_size; } - code_size += stencil_groups[_FATAL_ERROR].code.body_size; - data_size += stencil_groups[_FATAL_ERROR].data.body_size; + group = &stencil_groups[_FATAL_ERROR]; + code_size += group->code_size; + data_size += group->data_size; // Round up to the nearest page: size_t page_size = get_page_size(); assert((page_size & (page_size - 1)) == 0); @@ -405,87 +422,35 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size if (memory == NULL) { return -1; } + // Update the offsets of each instruction: + for (size_t i = 0; i < length; i++) { + instruction_starts[i] += (uintptr_t)memory; + } // Loop again to emit the code: unsigned char *code = memory; unsigned char *data = memory + code_size; - { - // Compile the trampoline, which handles converting between the native - // calling convention and the calling convention used by jitted code - // (which may be different for efficiency reasons). On platforms where - // we don't change calling conventions, the trampoline is empty and - // nothing is emitted here: - const StencilGroup *group = &trampoline; - // Think of patches as a dictionary mapping HoleValue to uintptr_t: - uintptr_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uintptr_t)code; - patches[HoleValue_CONTINUE] = (uintptr_t)code + group->code.body_size; - patches[HoleValue_DATA] = (uintptr_t)data; - patches[HoleValue_EXECUTOR] = (uintptr_t)executor; - patches[HoleValue_TOP] = (uintptr_t)memory + trampoline.code.body_size; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; - } + // Compile the trampoline, which handles converting between the native + // calling convention and the calling convention used by jitted code + // (which may be different for efficiency reasons). On platforms where + // we don't change calling conventions, the trampoline is empty and + // nothing is emitted here: + group = &trampoline; + group->emit(code, data, executor, NULL, instruction_starts); + code += group->code_size; + data += group->data_size; assert(trace[0].opcode == _START_EXECUTOR || trace[0].opcode == _COLD_EXIT); for (size_t i = 0; i < length; i++) { - _PyUOpInstruction *instruction = (_PyUOpInstruction *)&trace[i]; - const StencilGroup *group = &stencil_groups[instruction->opcode]; - uintptr_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uintptr_t)code; - patches[HoleValue_CONTINUE] = (uintptr_t)code + group->code.body_size; - patches[HoleValue_DATA] = (uintptr_t)data; - patches[HoleValue_EXECUTOR] = (uintptr_t)executor; - patches[HoleValue_OPARG] = instruction->oparg; - #if SIZEOF_VOID_P == 8 - patches[HoleValue_OPERAND] = instruction->operand; - #else - assert(SIZEOF_VOID_P == 4); - patches[HoleValue_OPERAND_HI] = instruction->operand >> 32; - patches[HoleValue_OPERAND_LO] = instruction->operand & UINT32_MAX; - #endif - switch (instruction->format) { - case UOP_FORMAT_TARGET: - patches[HoleValue_TARGET] = instruction->target; - break; - case UOP_FORMAT_EXIT: - assert(instruction->exit_index < executor->exit_count); - patches[HoleValue_EXIT_INDEX] = instruction->exit_index; - if (instruction->error_target < length) { - patches[HoleValue_ERROR_TARGET] = (uintptr_t)memory + instruction_starts[instruction->error_target]; - } - break; - case UOP_FORMAT_JUMP: - assert(instruction->jump_target < length); - patches[HoleValue_JUMP_TARGET] = (uintptr_t)memory + instruction_starts[instruction->jump_target]; - if (instruction->error_target < length) { - patches[HoleValue_ERROR_TARGET] = (uintptr_t)memory + instruction_starts[instruction->error_target]; - } - break; - default: - assert(0); - Py_FatalError("Illegal instruction format"); - } - patches[HoleValue_TOP] = (uintptr_t)memory + instruction_starts[1]; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; - } - { - // Protect against accidental buffer overrun into data: - const StencilGroup *group = &stencil_groups[_FATAL_ERROR]; - uintptr_t patches[] = GET_PATCHES(); - patches[HoleValue_CODE] = (uintptr_t)code; - patches[HoleValue_CONTINUE] = (uintptr_t)code; - patches[HoleValue_DATA] = (uintptr_t)data; - patches[HoleValue_EXECUTOR] = (uintptr_t)executor; - patches[HoleValue_TOP] = (uintptr_t)code; - patches[HoleValue_ZERO] = 0; - emit(group, patches); - code += group->code.body_size; - data += group->data.body_size; + const _PyUOpInstruction *instruction = &trace[i]; + group = &stencil_groups[instruction->opcode]; + group->emit(code, data, executor, instruction, instruction_starts); + code += group->code_size; + data += group->data_size; } + // Protect against accidental buffer overrun into data: + group = &stencil_groups[_FATAL_ERROR]; + group->emit(code, data, executor, NULL, instruction_starts); + code += group->code_size; + data += group->data_size; assert(code == memory + code_size); assert(data == memory + code_size + data_size); if (mark_executable(memory, total_size)) { @@ -493,7 +458,7 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction *trace, size return -1; } executor->jit_code = memory; - executor->jit_side_entry = memory + trampoline.code.body_size; + executor->jit_side_entry = memory + trampoline.code_size; executor->jit_size = total_size; return 0; } diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 9feceb45388d05d..6e046df3026ae91 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -3,6 +3,7 @@ import dataclasses import enum import sys +import typing import _schema @@ -47,6 +48,73 @@ class HoleValue(enum.Enum): ZERO = enum.auto() +# Map relocation types to our JIT's patch functions. "r" suffixes indicate that +# the patch function is relative. "x" suffixes indicate that they are "relaxing" +# (see comments in jit.c for more info): +_PATCH_FUNCS = { + # aarch64-apple-darwin: + "ARM64_RELOC_BRANCH26": "patch_aarch64_26r", + "ARM64_RELOC_GOT_LOAD_PAGE21": "patch_aarch64_21rx", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12": "patch_aarch64_12x", + "ARM64_RELOC_PAGE21": "patch_aarch64_21r", + "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", + "ARM64_RELOC_UNSIGNED": "patch_64", + # x86_64-pc-windows-msvc: + "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", + # aarch64-pc-windows-msvc: + "IMAGE_REL_ARM64_BRANCH26": "patch_aarch64_26r", + "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21rx", + "IMAGE_REL_ARM64_PAGEOFFSET_12A": "patch_aarch64_12", + "IMAGE_REL_ARM64_PAGEOFFSET_12L": "patch_aarch64_12x", + # i686-pc-windows-msvc: + "IMAGE_REL_I386_DIR32": "patch_32", + "IMAGE_REL_I386_REL32": "patch_x86_64_32rx", + # aarch64-unknown-linux-gnu: + "R_AARCH64_ABS64": "patch_64", + "R_AARCH64_ADD_ABS_LO12_NC": "patch_aarch64_12", + "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21rx", + "R_AARCH64_ADR_PREL_PG_HI21": "patch_aarch64_21r", + "R_AARCH64_CALL26": "patch_aarch64_26r", + "R_AARCH64_JUMP26": "patch_aarch64_26r", + "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12x", + "R_AARCH64_MOVW_UABS_G0_NC": "patch_aarch64_16a", + "R_AARCH64_MOVW_UABS_G1_NC": "patch_aarch64_16b", + "R_AARCH64_MOVW_UABS_G2_NC": "patch_aarch64_16c", + "R_AARCH64_MOVW_UABS_G3": "patch_aarch64_16d", + # x86_64-unknown-linux-gnu: + "R_X86_64_64": "patch_64", + "R_X86_64_GOTPCREL": "patch_32r", + "R_X86_64_GOTPCRELX": "patch_x86_64_32rx", + "R_X86_64_PC32": "patch_32r", + "R_X86_64_REX_GOTPCRELX": "patch_x86_64_32rx", + # x86_64-apple-darwin: + "X86_64_RELOC_BRANCH": "patch_32r", + "X86_64_RELOC_GOT": "patch_x86_64_32rx", + "X86_64_RELOC_GOT_LOAD": "patch_x86_64_32rx", + "X86_64_RELOC_SIGNED": "patch_32r", + "X86_64_RELOC_UNSIGNED": "patch_64", +} +# Translate HoleValues to C expressions: +_HOLE_EXPRS = { + HoleValue.CODE: "(uintptr_t)code", + HoleValue.CONTINUE: "(uintptr_t)code + sizeof(code_body)", + HoleValue.DATA: "(uintptr_t)data", + HoleValue.EXECUTOR: "(uintptr_t)executor", + # These should all have been turned into DATA values by process_relocations: + # HoleValue.GOT: "", + HoleValue.OPARG: "instruction->oparg", + HoleValue.OPERAND: "instruction->operand", + HoleValue.OPERAND_HI: "(instruction->operand >> 32)", + HoleValue.OPERAND_LO: "(instruction->operand & UINT32_MAX)", + HoleValue.TARGET: "instruction->target", + HoleValue.JUMP_TARGET: "instruction_starts[instruction->jump_target]", + HoleValue.ERROR_TARGET: "instruction_starts[instruction->error_target]", + HoleValue.EXIT_INDEX: "instruction->exit_index", + HoleValue.TOP: "instruction_starts[1]", + HoleValue.ZERO: "", +} + + @dataclasses.dataclass class Hole: """ @@ -63,19 +131,43 @@ class Hole: symbol: str | None # ...plus this addend: addend: int + func: str = dataclasses.field(init=False) # Convenience method: replace = dataclasses.replace - def as_c(self) -> str: - """Dump this hole as an initialization of a C Hole struct.""" - parts = [ - f"{self.offset:#x}", - f"HoleKind_{self.kind}", - f"HoleValue_{self.value.name}", - f"&{self.symbol}" if self.symbol else "NULL", - f"{_signed(self.addend):#x}", - ] - return f"{{{', '.join(parts)}}}" + def __post_init__(self) -> None: + self.func = _PATCH_FUNCS[self.kind] + + def fold(self, other: typing.Self) -> typing.Self | None: + """Combine two holes into a single hole, if possible.""" + if ( + self.offset + 4 == other.offset + and self.value == other.value + and self.symbol == other.symbol + and self.addend == other.addend + and self.func == "patch_aarch64_21rx" + and other.func == "patch_aarch64_12x" + ): + # These can *only* be properly relaxed when they appear together and + # patch the same value: + folded = self.replace() + folded.func = "patch_aarch64_33rx" + return folded + return None + + def as_c(self, where: str) -> str: + """Dump this hole as a call to a patch_* function.""" + location = f"{where} + {self.offset:#x}" + value = _HOLE_EXPRS[self.value] + if self.symbol: + if value: + value += " + " + value += f"(uintptr_t)&{self.symbol}" + if _signed(self.addend): + if value: + value += " + " + value += f"{_signed(self.addend):#x}" + return f"{self.func}({location}, {value});" @dataclasses.dataclass @@ -265,6 +357,10 @@ def _emit_global_offset_table(self) -> None: ) self.data.body.extend([0] * 8) + def as_c(self, opname: str) -> str: + """Dump this hole as a StencilGroup initializer.""" + return f"{{emit_{opname}, {len(self.code.body)}, {len(self.data.body)}}}" + def symbol_to_value(symbol: str) -> tuple[HoleValue, str | None]: """ diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index ccd67850c37787c..9d11094f85c7fff 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -1,100 +1,65 @@ """Utilities for writing StencilGroups out to a C header file.""" +import itertools import typing -import _schema import _stencils -def _dump_header() -> typing.Iterator[str]: - yield "typedef enum {" - for kind in typing.get_args(_schema.HoleKind): - yield f" HoleKind_{kind}," - yield "} HoleKind;" - yield "" - yield "typedef enum {" - for value in _stencils.HoleValue: - yield f" HoleValue_{value.name}," - yield "} HoleValue;" - yield "" - yield "typedef struct {" - yield " const size_t offset;" - yield " const HoleKind kind;" - yield " const HoleValue value;" - yield " const void *symbol;" - yield " const uint64_t addend;" - yield "} Hole;" - yield "" +def _dump_footer(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: yield "typedef struct {" - yield " const size_t body_size;" - yield " const unsigned char * const body;" - yield " const size_t holes_size;" - yield " const Hole * const holes;" - yield "} Stencil;" - yield "" - yield "typedef struct {" - yield " const Stencil code;" - yield " const Stencil data;" + yield " void (*emit)(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[]);" + yield " size_t code_size;" + yield " size_t data_size;" yield "} StencilGroup;" yield "" - - -def _dump_footer(opnames: typing.Iterable[str]) -> typing.Iterator[str]: - yield "#define INIT_STENCIL(STENCIL) { \\" - yield " .body_size = Py_ARRAY_LENGTH(STENCIL##_body) - 1, \\" - yield " .body = STENCIL##_body, \\" - yield " .holes_size = Py_ARRAY_LENGTH(STENCIL##_holes) - 1, \\" - yield " .holes = STENCIL##_holes, \\" - yield "}" - yield "" - yield "#define INIT_STENCIL_GROUP(OP) { \\" - yield " .code = INIT_STENCIL(OP##_code), \\" - yield " .data = INIT_STENCIL(OP##_data), \\" - yield "}" + yield f"static const StencilGroup trampoline = {groups['trampoline'].as_c('trampoline')};" yield "" - yield "static const StencilGroup stencil_groups[512] = {" - for opname in opnames: + yield "static const StencilGroup stencil_groups[MAX_UOP_ID + 1] = {" + for opname, group in sorted(groups.items()): if opname == "trampoline": continue - yield f" [{opname}] = INIT_STENCIL_GROUP({opname})," + yield f" [{opname}] = {group.as_c(opname)}," yield "};" - yield "" - yield "static const StencilGroup trampoline = INIT_STENCIL_GROUP(trampoline);" - yield "" - yield "#define GET_PATCHES() { \\" - for value in _stencils.HoleValue: - yield f" [HoleValue_{value.name}] = (uintptr_t)0xBADBADBADBADBADB, \\" - yield "}" def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator[str]: - yield f"// {opname}" + yield "void" + yield f"emit_{opname}(" + yield " unsigned char *code, unsigned char *data, _PyExecutorObject *executor," + yield " const _PyUOpInstruction *instruction, uintptr_t instruction_starts[])" + yield "{" for part, stencil in [("code", group.code), ("data", group.data)]: for line in stencil.disassembly: - yield f"// {line}" + yield f" // {line}" if stencil.body: - size = len(stencil.body) + 1 - yield f"static const unsigned char {opname}_{part}_body[{size}] = {{" + yield f" const unsigned char {part}_body[{len(stencil.body)}] = {{" for i in range(0, len(stencil.body), 8): row = " ".join(f"{byte:#04x}," for byte in stencil.body[i : i + 8]) - yield f" {row}" - yield "};" - else: - yield f"static const unsigned char {opname}_{part}_body[1];" - if stencil.holes: - size = len(stencil.holes) + 1 - yield f"static const Hole {opname}_{part}_holes[{size}] = {{" - for hole in stencil.holes: - yield f" {hole.as_c()}," - yield "};" - else: - yield f"static const Hole {opname}_{part}_holes[1];" + yield f" {row}" + yield " };" + # Data is written first (so relaxations in the code work properly): + for part, stencil in [("data", group.data), ("code", group.code)]: + if stencil.body: + yield f" memcpy({part}, {part}_body, sizeof({part}_body));" + skip = False + stencil.holes.sort(key=lambda hole: hole.offset) + for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): + if skip: + skip = False + continue + if pair and (folded := hole.fold(pair)): + skip = True + hole = folded + yield f" {hole.as_c(part)}" + yield "}" yield "" def dump(groups: dict[str, _stencils.StencilGroup]) -> typing.Iterator[str]: """Yield a JIT compiler line-by-line as a C header file.""" - yield from _dump_header() - for opname, group in groups.items(): + for opname, group in sorted(groups.items()): yield from _dump_stencil(opname, group) yield from _dump_footer(groups) From 42dc5b4ace39a3983cd9853719527f4724693adc Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 3 May 2024 23:13:36 -0500 Subject: [PATCH 101/220] gh-115532 Add kde_random() to the statistic module (#118210) --- Doc/library/statistics.rst | 84 +++++++++-------------------- Doc/whatsnew/3.13.rst | 3 +- Lib/statistics.py | 103 ++++++++++++++++++++++++++++++++++-- Lib/test/test_statistics.py | 80 ++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 63 deletions(-) diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index cc72396964342e5..d5a316e45ee3e20 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -77,6 +77,7 @@ or sample. :func:`geometric_mean` Geometric mean of data. :func:`harmonic_mean` Harmonic mean of data. :func:`kde` Estimate the probability density distribution of the data. +:func:`kde_random` Random sampling from the PDF generated by kde(). :func:`median` Median (middle value) of data. :func:`median_low` Low median of data. :func:`median_high` High median of data. @@ -311,6 +312,30 @@ However, for reading convenience, most of the examples show sorted sequences. .. versionadded:: 3.13 +.. function:: kde_random(data, h, kernel='normal', *, seed=None) + + Return a function that makes a random selection from the estimated + probability density function produced by ``kde(data, h, kernel)``. + + Providing a *seed* allows reproducible selections. In the future, the + values may change slightly as more accurate kernel inverse CDF estimates + are implemented. The seed may be an integer, float, str, or bytes. + + A :exc:`StatisticsError` will be raised if the *data* sequence is empty. + + Continuing the example for :func:`kde`, we can use + :func:`kde_random` to generate new random selections from an + estimated probability density function: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + .. versionadded:: 3.13 + + .. function:: median(data) Return the median (middle value) of numeric data, using the common "mean of @@ -1148,65 +1173,6 @@ The final prediction goes to the largest posterior. This is known as the 'female' -Sampling from kernel density estimation -*************************************** - -The :func:`kde()` function creates a continuous probability density -function from discrete samples. Some applications need a way to make -random selections from that distribution. - -The technique is to pick a sample from a bandwidth scaled kernel -function and recenter the result around a randomly chosen point from -the input data. This can be done with any kernel that has a known or -accurately approximated inverse cumulative distribution function. - -.. testcode:: - - from random import choice, random, seed - from math import sqrt, log, pi, tan, asin, cos, acos - from statistics import NormalDist - - kernel_invcdfs = { - 'normal': NormalDist().inv_cdf, - 'logistic': lambda p: log(p / (1 - p)), - 'sigmoid': lambda p: log(tan(p * pi/2)), - 'rectangular': lambda p: 2*p - 1, - 'triangular': lambda p: sqrt(2*p) - 1 if p < 0.5 else 1 - sqrt(2 - 2*p), - 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), - 'cosine': lambda p: 2*asin(2*p - 1)/pi, - } - - def kde_random(data, h, kernel='normal'): - 'Return a function that samples from kde() smoothed data.' - kernel_invcdf = kernel_invcdfs[kernel] - def rand(): - return h * kernel_invcdf(random()) + choice(data) - return rand - -For example: - -.. doctest:: - - >>> discrete_samples = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] - >>> rand = kde_random(discrete_samples, h=1.5) - >>> seed(8675309) - >>> selections = [rand() for i in range(10)] - >>> [round(x, 1) for x in selections] - [4.7, 7.4, 1.2, 7.8, 6.9, -1.3, 5.8, 0.2, -1.4, 5.7] - -.. testcode:: - :hide: - - from statistics import kde - from math import isclose - - # Verify that cdf / invcdf will round trip - xarr = [i/100 for i in range(-100, 101)] - for kernel, invcdf in kernel_invcdfs.items(): - cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) - for x in xarr: - assert isclose(invcdf(cdf(x)), x, abs_tol=1E-9) - .. # This modelines must appear within the last ten lines of the file. kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d996cf6c4d9b52b..269a7cc985ad198 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -745,7 +745,8 @@ statistics * Add :func:`statistics.kde` for kernel density estimation. This makes it possible to estimate a continuous probability density function - from a fixed number of discrete samples. + from a fixed number of discrete samples. Also added :func:`statistics.kde_random` + for sampling from the estimated probability density function. (Contributed by Raymond Hettinger in :gh:`115863`.) .. _whatsnew313-subprocess: diff --git a/Lib/statistics.py b/Lib/statistics.py index fc00891b083dc37..f3ce2d8b6b442a5 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -113,6 +113,7 @@ 'geometric_mean', 'harmonic_mean', 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -138,12 +139,13 @@ from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod -from math import isfinite, isinf, pi, cos, sin, cosh, atan +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos from functools import reduce from operator import itemgetter from collections import Counter, namedtuple, defaultdict _SQRT2 = sqrt(2.0) +_random = random # === Exceptions === @@ -978,11 +980,9 @@ def pdf(x): return sum(K((x - x_i) / h) for x_i in data) / (n * h) def cdf(x): - n = len(data) return sum(W((x - x_i) / h) for x_i in data) / n - else: sample = sorted(data) @@ -1078,6 +1078,7 @@ def quantiles(data, *, n=4, method='exclusive'): if ld == 1: return data * (n - 1) raise StatisticsError('must have at least one data point') + if method == 'inclusive': m = ld - 1 result = [] @@ -1086,6 +1087,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n result.append(interpolated) return result + if method == 'exclusive': m = ld + 1 result = [] @@ -1096,6 +1098,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n result.append(interpolated) return result + raise ValueError(f'Unknown method: {method!r}') @@ -1709,3 +1712,97 @@ def __getstate__(self): def __setstate__(self, state): self._mu, self._sigma = state + + +## kde_random() ############################################################## + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p >= 0.004 < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +_quartic_invcdf = _newton_raphson( + f_inv_estimate = _quartic_invcdf_estimate, + f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, + f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) + +def _triweight_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + return x * sign + +_triweight_invcdf = _newton_raphson( + f_inv_estimate = _triweight_invcdf_estimate, + f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, + f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) + +_kernel_invcdfs = { + 'normal': NormalDist().inv_cdf, + 'logistic': lambda p: log(p / (1 - p)), + 'sigmoid': lambda p: log(tan(p * pi/2)), + 'rectangular': lambda p: 2*p - 1, + 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), + 'quartic': _quartic_invcdf, + 'triweight': _triweight_invcdf, + 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), + 'cosine': lambda p: 2 * asin(2*p - 1) / pi, +} +_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] +_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] +_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] +_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + try: + kernel_invcdf = _kernel_invcdfs[kernel] + except KeyError: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * kernel_invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 204787a88a9c5fb..fe6c59c30dae283 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2426,6 +2426,86 @@ def integrate(func, low, high, steps=10_000): self.assertEqual(f_hat(-1.0), 1/2) self.assertEqual(f_hat(1.0), 1/2) + def test_kde_kernel_invcdfs(self): + kernel_invcdfs = statistics._kernel_invcdfs + kde = statistics.kde + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + for kernel, invcdf in kernel_invcdfs.items(): + with self.subTest(kernel=kernel): + cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + prng = kde_random(sample, h, kernel) + self.assertEqual(prng.__name__, 'rand') + self.assertIn(kernel, prng.__doc__) + self.assertIn(repr(h), prng.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + n = 1_000_000 + h = 1.75 + dx = 0.1 + + def p_expected(x): + return F_hat(x + dx) - F_hat(x - dx) + + def p_observed(x): + # P(x-dx <= X < x+dx) / (2*dx) + i = bisect.bisect_left(big_sample, x - dx) + j = bisect.bisect_right(big_sample, x + dx) + return (j - i) / len(big_sample) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + F_hat = statistics.kde(data, h, kernel, cumulative=True) + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + + for x in range(-40, 190): + x /= 10 + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.001)) + class TestQuantiles(unittest.TestCase): From 0b7814e0b638631fa2f5c81bcbab7b94064948d7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 4 May 2024 09:39:58 +0200 Subject: [PATCH 102/220] gh-110850: Use _PyDeadline_Get() in EnterNonRecursiveMutex() (#118556) Use _PyDeadline_Init() and _PyDeadline_Get() in EnterNonRecursiveMutex() of thread_nt.h. _PyDeadline_Get() uses the monotonic clock which is now the same as the perf counter clock on all platforms. So this change does not cause any behavior change. It just reuses existing helper functions. --- Python/thread_nt.h | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 9dca833ff203ca9..425658131c2fce9 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -77,17 +77,18 @@ EnterNonRecursiveMutex(PNRMUTEX mutex, DWORD milliseconds) } } else if (milliseconds != 0) { /* wait at least until the deadline */ - PyTime_t nanoseconds = (PyTime_t)milliseconds * (1000 * 1000); - PyTime_t deadline = _PyTime_Add(_PyTime_PerfCounterUnchecked(), nanoseconds); + PyTime_t timeout = (PyTime_t)milliseconds * (1000 * 1000); + PyTime_t deadline = _PyDeadline_Init(timeout); while (mutex->locked) { - PyTime_t microseconds = _PyTime_AsMicroseconds(nanoseconds, - _PyTime_ROUND_TIMEOUT); + PyTime_t microseconds = _PyTime_AsMicroseconds(timeout, + _PyTime_ROUND_TIMEOUT); if (PyCOND_TIMEDWAIT(&mutex->cv, &mutex->cs, microseconds) < 0) { result = WAIT_FAILED; break; } - nanoseconds = deadline - _PyTime_PerfCounterUnchecked(); - if (nanoseconds <= 0) { + + timeout = _PyDeadline_Get(deadline); + if (timeout <= 0) { break; } } From da2cfc4cb6b756b819b45bf34dd735c27b74d803 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 4 May 2024 08:50:24 +0100 Subject: [PATCH 103/220] GH-113464: Remove the extra jump via `_SIDE_EXIT` in `_EXIT_TRACE` (GH-118545) --- Include/internal/pycore_uop_ids.h | 37 +++++++++++++------------- Include/internal/pycore_uop_metadata.h | 6 +---- Python/bytecodes.c | 6 +---- Python/executor_cases.c.h | 10 +------ Python/optimizer.c | 21 +++++++-------- Python/optimizer_cases.c.h | 4 --- 6 files changed, 31 insertions(+), 53 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index aa3940456b62f2e..988464bcc210c89 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -240,33 +240,32 @@ extern "C" { #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _SIDE_EXIT 429 -#define _START_EXECUTOR 430 -#define _STORE_ATTR 431 -#define _STORE_ATTR_INSTANCE_VALUE 432 -#define _STORE_ATTR_SLOT 433 +#define _START_EXECUTOR 429 +#define _STORE_ATTR 430 +#define _STORE_ATTR_INSTANCE_VALUE 431 +#define _STORE_ATTR_SLOT 432 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 434 -#define _STORE_FAST_0 435 -#define _STORE_FAST_1 436 -#define _STORE_FAST_2 437 -#define _STORE_FAST_3 438 -#define _STORE_FAST_4 439 -#define _STORE_FAST_5 440 -#define _STORE_FAST_6 441 -#define _STORE_FAST_7 442 +#define _STORE_FAST 433 +#define _STORE_FAST_0 434 +#define _STORE_FAST_1 435 +#define _STORE_FAST_2 436 +#define _STORE_FAST_3 437 +#define _STORE_FAST_4 438 +#define _STORE_FAST_5 439 +#define _STORE_FAST_6 440 +#define _STORE_FAST_7 441 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 443 +#define _STORE_SUBSCR 442 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 444 -#define _TO_BOOL 445 +#define _TIER2_RESUME_CHECK 443 +#define _TO_BOOL 444 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -276,13 +275,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 446 +#define _UNPACK_SEQUENCE 445 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 446 +#define MAX_UOP_ID 445 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index d0860024e0db4d8..475924dcd9c6b83 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -237,7 +237,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_SET_IP] = 0, [_CHECK_STACK_SPACE_OPERAND] = HAS_DEOPT_FLAG, [_SAVE_RETURN_OFFSET] = HAS_ARG_FLAG, - [_EXIT_TRACE] = HAS_EXIT_FLAG, + [_EXIT_TRACE] = 0, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, @@ -252,7 +252,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_FATAL_ERROR] = 0, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, [_DEOPT] = 0, - [_SIDE_EXIT] = 0, [_ERROR_POP_N] = HAS_ARG_FLAG, [_TIER2_RESUME_CHECK] = HAS_DEOPT_FLAG, }; @@ -459,7 +458,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_SET_FUNCTION_ATTRIBUTE] = "_SET_FUNCTION_ATTRIBUTE", [_SET_IP] = "_SET_IP", [_SET_UPDATE] = "_SET_UPDATE", - [_SIDE_EXIT] = "_SIDE_EXIT", [_START_EXECUTOR] = "_START_EXECUTOR", [_STORE_ATTR] = "_STORE_ATTR", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", @@ -970,8 +968,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _DEOPT: return 0; - case _SIDE_EXIT: - return 0; case _ERROR_POP_N: return oparg; case _TIER2_RESUME_CHECK: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index e8383eda6a9d493..ddada96bea71b8c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -4133,7 +4133,7 @@ dummy_func( } tier2 op(_EXIT_TRACE, (--)) { - EXIT_IF(1); + EXIT_TO_TRACE(); } tier2 op(_CHECK_VALIDITY, (--)) { @@ -4266,10 +4266,6 @@ dummy_func( EXIT_TO_TIER1(); } - tier2 op(_SIDE_EXIT, (--)) { - EXIT_TO_TRACE(); - } - tier2 op(_ERROR_POP_N, (target/2, unused[oparg] --)) { frame->instr_ptr = ((_Py_CODEUNIT *)_PyFrame_GetCode(frame)->co_code_adaptive) + target; SYNC_SP(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c3ee6e9039c9000..d0b794c61ef4a82 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4127,10 +4127,7 @@ } case _EXIT_TRACE: { - if (1) { - UOP_STAT_INC(uopcode, miss); - JUMP_TO_JUMP_TARGET(); - } + EXIT_TO_TRACE(); break; } @@ -4319,11 +4316,6 @@ break; } - case _SIDE_EXIT: { - EXIT_TO_TRACE(); - break; - } - case _ERROR_POP_N: { oparg = CURRENT_OPARG(); uint32_t target = (uint32_t)CURRENT_OPERAND(); diff --git a/Python/optimizer.c b/Python/optimizer.c index 56768ae8f542f61..c0e1be96353d3fb 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -976,7 +976,7 @@ count_exits(_PyUOpInstruction *buffer, int length) int exit_count = 0; for (int i = 0; i < length; i++) { int opcode = buffer[i].opcode; - if (opcode == _SIDE_EXIT || opcode == _DYNAMIC_EXIT) { + if (opcode == _EXIT_TRACE || opcode == _DYNAMIC_EXIT) { exit_count++; } } @@ -1021,7 +1021,7 @@ prepare_for_execution(_PyUOpInstruction *buffer, int length) int32_t target = (int32_t)uop_get_target(inst); if (_PyUop_Flags[opcode] & (HAS_EXIT_FLAG | HAS_DEOPT_FLAG)) { uint16_t exit_op = (_PyUop_Flags[opcode] & HAS_EXIT_FLAG) ? - _SIDE_EXIT : _DEOPT; + _EXIT_TRACE : _DEOPT; int32_t jump_target = target; if (is_for_iter_test[opcode]) { /* Target the POP_TOP immediately after the END_FOR, @@ -1112,7 +1112,7 @@ sanity_check(_PyExecutorObject *executor) CHECK(target_unused(opcode)); break; case UOP_FORMAT_EXIT: - CHECK(opcode == _SIDE_EXIT); + CHECK(opcode == _EXIT_TRACE); CHECK(inst->exit_index < executor->exit_count); break; case UOP_FORMAT_JUMP: @@ -1138,9 +1138,9 @@ sanity_check(_PyExecutorObject *executor) uint16_t opcode = inst->opcode; CHECK( opcode == _DEOPT || - opcode == _SIDE_EXIT || + opcode == _EXIT_TRACE || opcode == _ERROR_POP_N); - if (opcode == _SIDE_EXIT) { + if (opcode == _EXIT_TRACE) { CHECK(inst->format == UOP_FORMAT_EXIT); } } @@ -1178,7 +1178,7 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil dest--; *dest = buffer[i]; assert(opcode != _POP_JUMP_IF_FALSE && opcode != _POP_JUMP_IF_TRUE); - if (opcode == _SIDE_EXIT) { + if (opcode == _EXIT_TRACE) { executor->exits[next_exit].target = buffer[i].target; dest->exit_index = next_exit; dest->format = UOP_FORMAT_EXIT; @@ -1398,14 +1398,13 @@ counter_optimize( return 0; } _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[5] = { - { .opcode = _START_EXECUTOR, .jump_target = 4, .format=UOP_FORMAT_JUMP }, + _PyUOpInstruction buffer[4] = { + { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, { .opcode = _LOAD_CONST_INLINE_BORROW, .operand = (uintptr_t)self }, { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .jump_target = 4, .format=UOP_FORMAT_JUMP }, - { .opcode = _SIDE_EXIT, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } + { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, 5, &EMPTY_FILTER); + _PyExecutorObject *executor = make_executor_from_uops(buffer, 4, &EMPTY_FILTER); if (executor == NULL) { return -1; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index e680d76141776f7..b602d663b08ba6c 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2140,10 +2140,6 @@ break; } - case _SIDE_EXIT: { - break; - } - case _ERROR_POP_N: { stack_pointer += -oparg; break; From 85af78996117dbe8ad45716633a3d6c39ff7bab2 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Sat, 4 May 2024 09:23:50 +0100 Subject: [PATCH 104/220] gh-111997: C-API for signalling monitoring events (#116413) --- Doc/c-api/index.rst | 1 + Doc/c-api/monitoring.rst | 164 ++++++ Doc/conf.py | 1 + Doc/library/sys.monitoring.rst | 5 +- Include/Python.h | 1 + Include/cpython/monitoring.h | 250 +++++++++ Include/internal/pycore_instruments.h | 32 -- Include/monitoring.h | 18 + Lib/test/test_monitoring.py | 183 ++++++- Makefile.pre.in | 2 + ...-03-13-17-48-24.gh-issue-111997.8ZbHlA.rst | 1 + Modules/Setup.stdlib.in | 2 +- Modules/_testcapi/monitoring.c | 507 ++++++++++++++++++ Modules/_testcapi/parts.h | 1 + Modules/_testcapimodule.c | 3 + PCbuild/_testcapi.vcxproj | 1 + PCbuild/_testcapi.vcxproj.filters | 3 + PCbuild/_testinternalcapi.vcxproj.filters | 2 +- Python/instrumentation.c | 301 +++++++++++ Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 20 files changed, 1442 insertions(+), 37 deletions(-) create mode 100644 Doc/c-api/monitoring.rst create mode 100644 Include/cpython/monitoring.h create mode 100644 Include/monitoring.h create mode 100644 Misc/NEWS.d/next/C API/2024-03-13-17-48-24.gh-issue-111997.8ZbHlA.rst create mode 100644 Modules/_testcapi/monitoring.c diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index 9a8f1507b3f4cc5..ba56b03c6ac8e7c 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -25,3 +25,4 @@ document the API functions in detail. memory.rst objimpl.rst apiabiversion.rst + monitoring.rst diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst new file mode 100644 index 000000000000000..763ec8ef761e4eb --- /dev/null +++ b/Doc/c-api/monitoring.rst @@ -0,0 +1,164 @@ +.. highlight:: c + +.. _monitoring: + +Monitorong C API +================ + +Added in version 3.13. + +An extension may need to interact with the event monitoring system. Subscribing +to events and registering callbacks can be done via the Python API exposed in +:mod:`sys.monitoring`. + +Generating Execution Events +=========================== + +The functions below make it possible for an extension to fire monitoring +events as it emulates the execution of Python code. Each of these functions +accepts a ``PyMonitoringState`` struct which contains concise information +about the activation state of events, as well as the event arguments, which +include a ``PyObject*`` representing the code object, the instruction offset +and sometimes additional, event-specific arguments (see :mod:`sys.monitoring` +for details about the signatures of the different event callbacks). +The ``codelike`` argument should be an instance of :class:`types.CodeType` +or of a type that emulates it. + +The VM disables tracing when firing an event, so there is no need for user +code to do that. + +Monitoring functions should not be called with an exception set, +except those listed below as working with the current exception. + +.. c:type:: PyMonitoringState + + Representation of the state of an event type. It is allocated by the user + while its contents are maintained by the monitoring API functions described below. + + +All of the functions below return 0 on success and -1 (with an exception set) on error. + +See :mod:`sys.monitoring` for descriptions of the events. + +.. c:function:: int PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_START`` event. + + +.. c:function:: int PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_RESUME`` event. + + +.. c:function:: int PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* retval) + + Fire a ``PY_YIELD`` event. + + +.. c:function:: int PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject* callable, PyObject *arg0) + + Fire a ``CALL`` event. + + +.. c:function:: int PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, int lineno) + + Fire a ``LINE`` event. + + +.. c:function:: int PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``JUMP`` event. + + +.. c:function:: int PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *target_offset) + + Fire a ``BRANCH`` event. + + +.. c:function:: int PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, PyObject *retval) + + Fire a ``C_RETURN`` event. + + +.. c:function:: int PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_THROW`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``C_RAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``RERAISE`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire an ``EXCEPTION_HANDLED`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``PY_UNWIND`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +.. c:function:: int PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) + + Fire a ``STOP_ITERATION`` event with the current exception (as returned by + :c:func:`PyErr_GetRaisedException`). + + +Managing the Monitoring State +----------------------------- + +Monitoring states can be managed with the help of monitoring scopes. A scope +would typically correspond to a python function. + +.. :c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) + + Enter a monitored scope. ``event_types`` is an array of the event IDs for + events that may be fired from the scope. For example, the ID of a ``PY_START`` + event is the value ``PY_MONITORING_EVENT_PY_START``, which is numerically equal + to the base-2 logarithm of ``sys.monitoring.events.PY_START``. + ``state_array`` is an array with a monitoring state entry for each event in + ``event_types``, it is allocated by the user but populated by + ``PyMonitoring_EnterScope`` with information about the activation state of + the event. The size of ``event_types`` (and hence also of ``state_array``) + is given in ``length``. + + The ``version`` argument is a pointer to a value which should be allocated + by the user together with ``state_array`` and initialized to 0, + and then set only by ``PyMonitoring_EnterScope`` itelf. It allows this + function to determine whether event states have changed since the previous call, + and to return quickly if they have not. + + The scopes referred to here are lexical scopes: a function, class or method. + ``PyMonitoring_EnterScope`` should be called whenever the lexical scope is + entered. Scopes can be reentered, reusing the same *state_array* and *version*, + in situations like when emulating a recursive Python function. When a code-like's + execution is paused, such as when emulating a generator, the scope needs to + be exited and re-entered. + + +.. :c:function:: int PyMonitoring_ExitScope(void) + + Exit the last scope that was entered with ``PyMonitoring_EnterScope``. diff --git a/Doc/conf.py b/Doc/conf.py index 73abe8276f29fee..86371d17ae742a3 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -131,6 +131,7 @@ ('c:func', 'vsnprintf'), # Standard C types ('c:type', 'FILE'), + ('c:type', 'int32_t'), ('c:type', 'int64_t'), ('c:type', 'intmax_t'), ('c:type', 'off_t'), diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 4980227c60b21e3..0e0095e108e9c06 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -255,7 +255,10 @@ No events are active by default. Per code object events '''''''''''''''''''''' -Events can also be controlled on a per code object basis. +Events can also be controlled on a per code object basis. The functions +defined below which accept a :class:`types.CodeType` should be prepared +to accept a look-alike object from functions which are not defined +in Python (see :ref:`monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int diff --git a/Include/Python.h b/Include/Python.h index bb771fb3aec980d..e05901b9e52b5a8 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -84,6 +84,7 @@ #include "setobject.h" #include "methodobject.h" #include "moduleobject.h" +#include "monitoring.h" #include "cpython/funcobject.h" #include "cpython/classobject.h" #include "fileobject.h" diff --git a/Include/cpython/monitoring.h b/Include/cpython/monitoring.h new file mode 100644 index 000000000000000..efb9ec0e587552c --- /dev/null +++ b/Include/cpython/monitoring.h @@ -0,0 +1,250 @@ +#ifndef Py_CPYTHON_MONITORING_H +# error "this header file must not be included directly" +#endif + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ + ((ev) < _PY_MONITORING_LOCAL_EVENTS) + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 +#define PY_MONITORING_EVENT_RERAISE 14 + + +/* Ancillary events */ + +#define PY_MONITORING_EVENT_C_RETURN 15 +#define PY_MONITORING_EVENT_C_RAISE 16 + + +typedef struct _PyMonitoringState { + uint8_t active; + uint8_t opaque; +} PyMonitoringState; + + +PyAPI_FUNC(int) +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length); + +PyAPI_FUNC(int) +PyMonitoring_ExitScope(void); + + +PyAPI_FUNC(int) +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0); + +PyAPI_FUNC(int) +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno); + +PyAPI_FUNC(int) +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + +PyAPI_FUNC(int) +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset); + + +#define _PYMONITORING_IF_ACTIVE(STATE, X) \ + if ((STATE)->active) { \ + return (X); \ + } \ + else { \ + return 0; \ + } + +static inline int +PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyStartEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyResumeEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0)); +} + +static inline int +PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireLineEvent(state, codelike, offset, lineno)); +} + +static inline int +PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset)); +} + +static inline int +PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCReturnEvent(state, codelike, offset, retval)); +} + +static inline int +PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyThrowEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireReraiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireExceptionHandledEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireCRaiseEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FirePyUnwindEvent(state, codelike, offset)); +} + +static inline int +PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + _PYMONITORING_IF_ACTIVE( + state, + _PyMonitoring_FireStopIterationEvent(state, codelike, offset)); +} + +#undef _PYMONITORING_IF_ACTIVE diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 7f84d4a763bbcf6..c98e82c8be5546a 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -13,38 +13,6 @@ extern "C" { #define PY_MONITORING_TOOL_IDS 8 -/* Local events. - * These require bytecode instrumentation */ - -#define PY_MONITORING_EVENT_PY_START 0 -#define PY_MONITORING_EVENT_PY_RESUME 1 -#define PY_MONITORING_EVENT_PY_RETURN 2 -#define PY_MONITORING_EVENT_PY_YIELD 3 -#define PY_MONITORING_EVENT_CALL 4 -#define PY_MONITORING_EVENT_LINE 5 -#define PY_MONITORING_EVENT_INSTRUCTION 6 -#define PY_MONITORING_EVENT_JUMP 7 -#define PY_MONITORING_EVENT_BRANCH 8 -#define PY_MONITORING_EVENT_STOP_ITERATION 9 - -#define PY_MONITORING_IS_INSTRUMENTED_EVENT(ev) \ - ((ev) < _PY_MONITORING_LOCAL_EVENTS) - -/* Other events, mainly exceptions */ - -#define PY_MONITORING_EVENT_RAISE 10 -#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 -#define PY_MONITORING_EVENT_PY_UNWIND 12 -#define PY_MONITORING_EVENT_PY_THROW 13 -#define PY_MONITORING_EVENT_RERAISE 14 - - -/* Ancillary events */ - -#define PY_MONITORING_EVENT_C_RETURN 15 -#define PY_MONITORING_EVENT_C_RAISE 16 - - typedef uint32_t _PyMonitoringEventSet; /* Tool IDs */ diff --git a/Include/monitoring.h b/Include/monitoring.h new file mode 100644 index 000000000000000..985f7f230e44e3d --- /dev/null +++ b/Include/monitoring.h @@ -0,0 +1,18 @@ +#ifndef Py_MONITORING_H +#define Py_MONITORING_H +#ifdef __cplusplus +extern "C" { +#endif + +// There is currently no limited API for monitoring + +#ifndef Py_LIMITED_API +# define Py_CPYTHON_MONITORING_H +# include "cpython/monitoring.h" +# undef Py_CPYTHON_MONITORING_H +#endif + +#ifdef __cplusplus +} +#endif +#endif /* !Py_MONITORING_H */ diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index a9140d4d3dd7434..eeb3f88a081750d 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,16 +3,20 @@ import collections import dis import functools +import math import operator import sys import textwrap import types import unittest import asyncio -from test import support + +import test.support from test.support import requires_specialization, script_helper from test.support.import_helper import import_module +_testcapi = test.support.import_helper.import_module("_testcapi") + PAIR = (0,1) def f1(): @@ -1887,5 +1891,180 @@ def test_monitoring_live_at_shutdown(self): # gh-115832: An object destructor running during the final GC of # interpreter shutdown triggered an infinite loop in the # instrumentation code. - script = support.findfile("_test_monitoring_shutdown.py") + script = test.support.findfile("_test_monitoring_shutdown.py") script_helper.run_test_script(script) + + +class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): + + class Scope: + def __init__(self, *args): + self.args = args + + def __enter__(self): + _testcapi.monitoring_enter_scope(*self.args) + + def __exit__(self, *args): + _testcapi.monitoring_exit_scope() + + def setUp(self): + super(TestCApiEventGeneration, self).setUp() + + capi = _testcapi + + self.codelike = capi.CodeLike(2) + + self.cases = [ + # (Event, function, *args) + ( 1, E.PY_START, capi.fire_event_py_start), + ( 1, E.PY_RESUME, capi.fire_event_py_resume), + ( 1, E.PY_YIELD, capi.fire_event_py_yield, 10), + ( 1, E.PY_RETURN, capi.fire_event_py_return, 20), + ( 2, E.CALL, capi.fire_event_call, callable, 40), + ( 1, E.JUMP, capi.fire_event_jump, 60), + ( 1, E.BRANCH, capi.fire_event_branch, 70), + ( 1, E.PY_THROW, capi.fire_event_py_throw, ValueError(1)), + ( 1, E.RAISE, capi.fire_event_raise, ValueError(2)), + ( 1, E.EXCEPTION_HANDLED, capi.fire_event_exception_handled, ValueError(5)), + ( 1, E.PY_UNWIND, capi.fire_event_py_unwind, ValueError(6)), + ( 1, E.STOP_ITERATION, capi.fire_event_stop_iteration, ValueError(7)), + ] + + + def check_event_count(self, event, func, args, expected): + class Counter: + def __init__(self): + self.count = 0 + def __call__(self, *args): + self.count += 1 + + try: + counter = Counter() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + try: + func(*args) + except ValueError as e: + self.assertIsInstance(expected, ValueError) + self.assertEqual(str(e), str(expected)) + return + else: + self.assertEqual(counter.count, expected) + + prev = sys.monitoring.register_callback(TEST_TOOL, event, None) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, 0) + self.assertEqual(prev, counter) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_fire_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args) + self.check_event_count(event, function, args_, expected) + + def test_missing_exception(self): + for _, event, function, *args in self.cases: + if not (args and isinstance(args[-1], BaseException)): + continue + offset = 0 + self.codelike = _testcapi.CodeLike(1) + with self.subTest(function.__name__): + args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) + evt = int(math.log2(event)) + expected = ValueError(f"Firing event {evt} with no exception set") + self.check_event_count(event, function, args_, expected) + + + CANNOT_DISABLE = { E.PY_THROW, E.RAISE, E.RERAISE, + E.EXCEPTION_HANDLED, E.PY_UNWIND } + + def check_disable(self, event, func, args, expected): + try: + counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, event, counter) + if event == E.C_RETURN or event == E.C_RAISE: + sys.monitoring.set_events(TEST_TOOL, E.CALL) + else: + sys.monitoring.set_events(TEST_TOOL, event) + event_value = int(math.log2(event)) + with self.Scope(self.codelike, event_value): + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.disable = True + if event in self.CANNOT_DISABLE: + # use try-except rather then assertRaises to avoid + # events from framework code + try: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + except ValueError: + pass + else: + self.Error("Expected a ValueError") + else: + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected) + counter.count = 0 + func(*args) + self.assertEqual(counter.count, expected - 1) + finally: + sys.monitoring.set_events(TEST_TOOL, 0) + + def test_disable_event(self): + for expected, event, function, *args in self.cases: + offset = 0 + self.codelike = _testcapi.CodeLike(2) + with self.subTest(function.__name__): + args_ = (self.codelike, 0) + tuple(args) + self.check_disable(event, function, args_, expected) + + def test_enter_scope_two_events(self): + try: + yield_counter = CounterWithDisable() + unwind_counter = CounterWithDisable() + sys.monitoring.register_callback(TEST_TOOL, E.PY_YIELD, yield_counter) + sys.monitoring.register_callback(TEST_TOOL, E.PY_UNWIND, unwind_counter) + sys.monitoring.set_events(TEST_TOOL, E.PY_YIELD | E.PY_UNWIND) + + yield_value = int(math.log2(E.PY_YIELD)) + unwind_value = int(math.log2(E.PY_UNWIND)) + cl = _testcapi.CodeLike(2) + common_args = (cl, 0) + with self.Scope(cl, yield_value, unwind_value): + yield_counter.count = 0 + unwind_counter.count = 0 + + _testcapi.fire_event_py_unwind(*common_args, ValueError(42)) + assert(yield_counter.count == 0) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 1) + assert(unwind_counter.count == 1) + + yield_counter.disable = True + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + _testcapi.fire_event_py_yield(*common_args, ValueError(42)) + assert(yield_counter.count == 2) + assert(unwind_counter.count == 1) + + finally: + sys.monitoring.set_events(TEST_TOOL, 0) diff --git a/Makefile.pre.in b/Makefile.pre.in index e69d1fe6e2dd14e..bd17debf309f0ee 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1022,6 +1022,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/methodobject.h \ $(srcdir)/Include/modsupport.h \ $(srcdir)/Include/moduleobject.h \ + $(srcdir)/Include/monitoring.h \ $(srcdir)/Include/object.h \ $(srcdir)/Include/objimpl.h \ $(srcdir)/Include/opcode.h \ @@ -1091,6 +1092,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/cpython/longobject.h \ $(srcdir)/Include/cpython/memoryobject.h \ $(srcdir)/Include/cpython/methodobject.h \ + $(srcdir)/Include/cpython/monitoring.h \ $(srcdir)/Include/cpython/object.h \ $(srcdir)/Include/cpython/objimpl.h \ $(srcdir)/Include/cpython/odictobject.h \ diff --git a/Misc/NEWS.d/next/C API/2024-03-13-17-48-24.gh-issue-111997.8ZbHlA.rst b/Misc/NEWS.d/next/C API/2024-03-13-17-48-24.gh-issue-111997.8ZbHlA.rst new file mode 100644 index 000000000000000..e74c0397b85aa14 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2024-03-13-17-48-24.gh-issue-111997.8ZbHlA.rst @@ -0,0 +1 @@ +Add a C-API for firing monitoring events. diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 61037f592f82f18..78b979698fcd75d 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -163,7 +163,7 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c @MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_testcapi/monitoring.c b/Modules/_testcapi/monitoring.c new file mode 100644 index 000000000000000..aa90cfc06c1536f --- /dev/null +++ b/Modules/_testcapi/monitoring.c @@ -0,0 +1,507 @@ +#include "parts.h" +#include "util.h" + +#include "monitoring.h" + +#define Py_BUILD_CORE +#include "internal/pycore_instruments.h" + +typedef struct { + PyObject_HEAD + PyMonitoringState *monitoring_states; + uint64_t version; + int num_events; + /* Other fields */ +} PyCodeLikeObject; + + +static PyObject * +CodeLike_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + int num_events; + if (!PyArg_ParseTuple(args, "i", &num_events)) { + return NULL; + } + PyMonitoringState *states = (PyMonitoringState *)PyMem_Calloc( + num_events, sizeof(PyMonitoringState)); + if (states == NULL) { + return NULL; + } + PyCodeLikeObject *self = (PyCodeLikeObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->version = 0; + self->monitoring_states = states; + self->num_events = num_events; + } + else { + PyMem_Free(states); + } + return (PyObject *) self; +} + +static void +CodeLike_dealloc(PyCodeLikeObject *self) +{ + if (self->monitoring_states) { + PyMem_Free(self->monitoring_states); + } + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +CodeLike_str(PyCodeLikeObject *self) +{ + PyObject *res = NULL; + PyObject *sep = NULL; + PyObject *parts = NULL; + if (self->monitoring_states) { + parts = PyList_New(0); + if (parts == NULL) { + goto end; + } + + PyObject *heading = PyUnicode_FromString("PyCodeLikeObject"); + if (heading == NULL) { + goto end; + } + int err = PyList_Append(parts, heading); + Py_DECREF(heading); + if (err < 0) { + goto end; + } + + for (int i = 0; i < self->num_events; i++) { + PyObject *part = PyUnicode_FromFormat(" %d", self->monitoring_states[i].active); + if (part == NULL) { + goto end; + } + int err = PyList_Append(parts, part); + Py_XDECREF(part); + if (err < 0) { + goto end; + } + } + sep = PyUnicode_FromString(": "); + if (sep == NULL) { + goto end; + } + res = PyUnicode_Join(sep, parts); + } +end: + Py_XDECREF(sep); + Py_XDECREF(parts); + return res; +} + +static PyTypeObject PyCodeLike_Type = { + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "monitoring.CodeLike", + .tp_doc = PyDoc_STR("CodeLike objects"), + .tp_basicsize = sizeof(PyCodeLikeObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = CodeLike_new, + .tp_dealloc = (destructor) CodeLike_dealloc, + .tp_str = (reprfunc) CodeLike_str, +}; + +#define RAISE_UNLESS_CODELIKE(v) if (!Py_IS_TYPE((v), &PyCodeLike_Type)) { \ + PyErr_Format(PyExc_TypeError, "expected a code-like, got %s", Py_TYPE(v)->tp_name); \ + return NULL; \ + } + +/*******************************************************************/ + +static PyMonitoringState * +setup_fire(PyObject *codelike, int offset, PyObject *exc) +{ + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = ((PyCodeLikeObject *)codelike); + assert(offset >= 0 && offset < cl->num_events); + PyMonitoringState *state = &cl->monitoring_states[offset]; + + if (exc != NULL) { + PyErr_SetRaisedException(Py_NewRef(exc)); + } + return state; +} + +static int +teardown_fire(int res, PyMonitoringState *state, PyObject *exception) +{ + if (res == -1) { + return -1; + } + if (exception) { + assert(PyErr_Occurred()); + assert(((PyObject*)Py_TYPE(exception)) == PyErr_Occurred()); + } + + else { + assert(!PyErr_Occurred()); + } + PyErr_Clear(); + return state->active; +} + +static PyObject * +fire_event_py_start(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyStartEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_resume(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + if (!PyArg_ParseTuple(args, "Oi", &codelike, &offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyResumeEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_return(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCReturnEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_yield(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *retval; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &retval)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyYieldEvent(state, codelike, offset, retval); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_call(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *callable, *arg0; + if (!PyArg_ParseTuple(args, "OiOO", &codelike, &offset, &callable, &arg0)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCallEvent(state, codelike, offset, callable, arg0); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_line(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset, lineno; + if (!PyArg_ParseTuple(args, "Oii", &codelike, &offset, &lineno)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireLineEvent(state, codelike, offset, lineno); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_jump(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireJumpEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_branch(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *target_offset; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &target_offset)) { + return NULL; + } + PyObject *exception = NULL; + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireBranchEvent(state, codelike, offset, target_offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_throw(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyThrowEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_c_raise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireCRaiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_reraise(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireReraiseEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_exception_handled(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireExceptionHandledEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_py_unwind(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FirePyUnwindEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +static PyObject * +fire_event_stop_iteration(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int offset; + PyObject *exception; + if (!PyArg_ParseTuple(args, "OiO", &codelike, &offset, &exception)) { + return NULL; + } + NULLABLE(exception); + PyMonitoringState *state = setup_fire(codelike, offset, exception); + if (state == NULL) { + return NULL; + } + int res = PyMonitoring_FireStopIterationEvent(state, codelike, offset); + RETURN_INT(teardown_fire(res, state, exception)); +} + +/*******************************************************************/ + +static PyObject * +enter_scope(PyObject *self, PyObject *args) +{ + PyObject *codelike; + int event1, event2=0; + Py_ssize_t num_events = PyTuple_Size(args) - 1; + if (num_events == 1) { + if (!PyArg_ParseTuple(args, "Oi", &codelike, &event1)) { + return NULL; + } + } + else { + assert(num_events == 2); + if (!PyArg_ParseTuple(args, "Oii", &codelike, &event1, &event2)) { + return NULL; + } + } + RAISE_UNLESS_CODELIKE(codelike); + PyCodeLikeObject *cl = (PyCodeLikeObject *) codelike; + + uint8_t events[] = { event1, event2 }; + + PyMonitoring_EnterScope(cl->monitoring_states, + &cl->version, + events, + num_events); + + Py_RETURN_NONE; +} + +static PyObject * +exit_scope(PyObject *self, PyObject *args) +{ + PyMonitoring_ExitScope(); + Py_RETURN_NONE; +} + +static PyMethodDef TestMethods[] = { + {"fire_event_py_start", fire_event_py_start, METH_VARARGS}, + {"fire_event_py_resume", fire_event_py_resume, METH_VARARGS}, + {"fire_event_py_return", fire_event_py_return, METH_VARARGS}, + {"fire_event_c_return", fire_event_c_return, METH_VARARGS}, + {"fire_event_py_yield", fire_event_py_yield, METH_VARARGS}, + {"fire_event_call", fire_event_call, METH_VARARGS}, + {"fire_event_line", fire_event_line, METH_VARARGS}, + {"fire_event_jump", fire_event_jump, METH_VARARGS}, + {"fire_event_branch", fire_event_branch, METH_VARARGS}, + {"fire_event_py_throw", fire_event_py_throw, METH_VARARGS}, + {"fire_event_raise", fire_event_raise, METH_VARARGS}, + {"fire_event_c_raise", fire_event_c_raise, METH_VARARGS}, + {"fire_event_reraise", fire_event_reraise, METH_VARARGS}, + {"fire_event_exception_handled", fire_event_exception_handled, METH_VARARGS}, + {"fire_event_py_unwind", fire_event_py_unwind, METH_VARARGS}, + {"fire_event_stop_iteration", fire_event_stop_iteration, METH_VARARGS}, + {"monitoring_enter_scope", enter_scope, METH_VARARGS}, + {"monitoring_exit_scope", exit_scope, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestCapi_Init_Monitoring(PyObject *m) +{ + if (PyType_Ready(&PyCodeLike_Type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "CodeLike", (PyObject *) &PyCodeLike_Type) < 0) { + Py_DECREF(m); + return -1; + } + if (PyModule_AddFunctions(m, TestMethods) < 0) { + return -1; + } + return 0; +} diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index 0e24e44083ea054..41d190961c69ee4 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -58,6 +58,7 @@ int _PyTestCapi_Init_Immortal(PyObject *module); int _PyTestCapi_Init_GC(PyObject *module); int _PyTestCapi_Init_Hash(PyObject *module); int _PyTestCapi_Init_Time(PyObject *module); +int _PyTestCapi_Init_Monitoring(PyObject *module); int _PyTestCapi_Init_Object(PyObject *module); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index beae13cd74c7319..e7e342e529eff47 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4135,6 +4135,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Time(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Monitoring(m) < 0) { + return NULL; + } if (_PyTestCapi_Init_Object(m) < 0) { return NULL; } diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index cc25b6ebd7c6735..44dbf2348137e17 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -125,6 +125,7 @@ + diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 28c82254d85d4c1..cae44bc955f7f12 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -108,6 +108,9 @@ Source Files + + Source Files + diff --git a/PCbuild/_testinternalcapi.vcxproj.filters b/PCbuild/_testinternalcapi.vcxproj.filters index abfeeb39630daf6..27429ea5833077f 100644 --- a/PCbuild/_testinternalcapi.vcxproj.filters +++ b/PCbuild/_testinternalcapi.vcxproj.filters @@ -27,4 +27,4 @@ Resource Files - \ No newline at end of file + diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 8085d7335fe21af..72c9d2af5b3202b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2424,3 +2424,304 @@ PyObject *_Py_CreateMonitoringObject(void) Py_DECREF(mod); return NULL; } + + +static int +capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject **args, Py_ssize_t nargs, int event) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; + + uint8_t tools = state->active; + assert(args[1] == NULL); + args[1] = codelike; + if (offset < 0) { + PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); + return -1; + } + PyObject *offset_obj = PyLong_FromLong(offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; + Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + PyObject **callargs = &args[1]; + int err = 0; + + while (tools) { + int tool = most_significant_bit(tools); + assert(tool >= 0 && tool < 8); + assert(tools & (1 << tool)); + tools ^= (1 << tool); + int res = call_one_instrument(interp, tstate, callargs, nargsf, tool, event); + if (res == 0) { + /* Nothing to do */ + } + else if (res < 0) { + /* error */ + err = -1; + break; + } + else { + /* DISABLE */ + if (!PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { + PyErr_Format(PyExc_ValueError, + "Cannot disable %s events. Callback removed.", + event_names[event]); + /* Clear tool to prevent infinite loop */ + Py_CLEAR(interp->monitoring_callables[tool][event]); + err = -1; + break; + } + else { + state->active &= ~(1 << tool); + } + } + } + return err; +} + +int +PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, + const uint8_t *event_types, Py_ssize_t length) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (global_version(interp) == *version) { + return 0; + } + + _Py_GlobalMonitors *m = &interp->monitors; + for (Py_ssize_t i = 0; i < length; i++) { + int event = event_types[i]; + state_array[i].active = m->tools[event]; + } + *version = global_version(interp); + return 0; +} + +int +PyMonitoring_ExitScope(void) +{ + return 0; +} + +int +_PyMonitoring_FirePyStartEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_START); +} + +int +_PyMonitoring_FirePyResumeEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + assert(state->active); + PyObject *args[3] = { NULL, NULL, NULL }; + return capi_call_instrumentation(state, codelike, offset, args, 2, + PY_MONITORING_EVENT_PY_RESUME); +} + + + +int +_PyMonitoring_FirePyReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_RETURN); +} + +int +_PyMonitoring_FirePyYieldEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_PY_YIELD); +} + +int +_PyMonitoring_FireCallEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject* callable, PyObject *arg0) +{ + assert(state->active); + PyObject *args[5] = { NULL, NULL, NULL, callable, arg0 }; + return capi_call_instrumentation(state, codelike, offset, args, 4, + PY_MONITORING_EVENT_CALL); +} + +int +_PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + int lineno) +{ + assert(state->active); + PyObject *lno = PyLong_FromLong(lineno); + if (lno == NULL) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, lno }; + int res= capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_LINE); + Py_DECREF(lno); + return res; +} + +int +_PyMonitoring_FireJumpEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_JUMP); +} + +int +_PyMonitoring_FireBranchEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *target_offset) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, target_offset }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_BRANCH); +} + +int +_PyMonitoring_FireCReturnEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset, + PyObject *retval) +{ + assert(state->active); + PyObject *args[4] = { NULL, NULL, NULL, retval }; + return capi_call_instrumentation(state, codelike, offset, args, 3, + PY_MONITORING_EVENT_C_RETURN); +} + +static inline int +exception_event_setup(PyObject **exc, int event) { + *exc = PyErr_GetRaisedException(); + if (*exc == NULL) { + PyErr_Format(PyExc_ValueError, + "Firing event %d with no exception set", + event); + return -1; + } + return 0; +} + + +static inline int +exception_event_teardown(int err, PyObject *exc) { + if (err == 0) { + PyErr_SetRaisedException(exc); + } + else { + assert(PyErr_Occurred()); + Py_DECREF(exc); + } + return err; +} + +int +_PyMonitoring_FirePyThrowEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_THROW; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireCRaiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_C_RAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireReraiseEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_RERAISE; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireExceptionHandledEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_EXCEPTION_HANDLED; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FirePyUnwindEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_PY_UNWIND; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} + +int +_PyMonitoring_FireStopIterationEvent(PyMonitoringState *state, PyObject *codelike, int32_t offset) +{ + int event = PY_MONITORING_EVENT_STOP_ITERATION; + assert(state->active); + PyObject *exc; + if (exception_event_setup(&exc, event) < 0) { + return -1; + } + PyObject *args[4] = { NULL, NULL, NULL, exc }; + int err = capi_call_instrumentation(state, codelike, offset, args, 3, event); + return exception_event_teardown(err, exc); +} diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index b58e9d9fae380fe..285129fd361665f 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -397,6 +397,7 @@ Modules/xxmodule.c - Str_Type - Modules/xxmodule.c - Xxo_Type - Modules/xxsubtype.c - spamdict_type - Modules/xxsubtype.c - spamlist_type - +Modules/_testcapi/monitoring.c - PyCodeLike_Type - ##----------------------- ## non-static types - initialized once From 00da0afa0d98ce1fae67f7258c7f3db2b81a07e7 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 03:26:40 -0700 Subject: [PATCH 105/220] gh-113081: Print colorized exception just like built-in traceback in pdb (#113082) --- Lib/pdb.py | 3 ++- Lib/test/test_pdb.py | 4 +++- .../Library/2023-12-14-02-51-38.gh-issue-113081.S-9Qyn.rst | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-14-02-51-38.gh-issue-113081.S-9Qyn.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index bb669a0d2c1ce56..e6450861328b280 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -84,6 +84,7 @@ import tokenize import traceback import linecache +import _colorize from contextlib import contextmanager from rlcompleter import Completer @@ -2347,7 +2348,7 @@ def main(): print("The program exited via sys.exit(). Exit status:", end=' ') print(e) except BaseException as e: - traceback.print_exc() + traceback.print_exception(e, colorize=_colorize.can_colorize()) print("Uncaught exception. Entering post mortem debugging") print("Running 'cont' or 'step' will restart the program") pdb.interaction(None, e) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index a3d2dda43b086d9..c9ea52717490b1d 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -15,7 +15,7 @@ from contextlib import ExitStack, redirect_stdout from io import StringIO from test import support -from test.support import os_helper +from test.support import force_not_colorized, os_helper from test.support.import_helper import import_module from test.support.pty_helper import run_pty, FakeInput from unittest.mock import patch @@ -2919,6 +2919,7 @@ def start_pdb(): self.assertNotIn(b'Error', stdout, "Got an error running test script under PDB") + @force_not_colorized def test_issue16180(self): # A syntax error in the debuggee. script = "def f: pass\n" @@ -2932,6 +2933,7 @@ def test_issue16180(self): 'Fail to handle a syntax error in the debuggee.' .format(expected, stderr)) + @force_not_colorized def test_issue84583(self): # A syntax error from ast.literal_eval should not make pdb exit. script = "import ast; ast.literal_eval('')\n" diff --git a/Misc/NEWS.d/next/Library/2023-12-14-02-51-38.gh-issue-113081.S-9Qyn.rst b/Misc/NEWS.d/next/Library/2023-12-14-02-51-38.gh-issue-113081.S-9Qyn.rst new file mode 100644 index 000000000000000..e6b2d01837c7df9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-14-02-51-38.gh-issue-113081.S-9Qyn.rst @@ -0,0 +1 @@ +Print colorized exception just like built-in traceback in :mod:`pdb` From 1ab6356ebec25f216a0eddbd81225abcb93f2d55 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Sat, 4 May 2024 12:11:11 +0100 Subject: [PATCH 106/220] GH-118095: Use broader specializations of CALL in tier 1, for better tier 2 support of calls. (GH-118322) * Add CALL_PY_GENERAL, CALL_BOUND_METHOD_GENERAL and call CALL_NON_PY_GENERAL specializations. * Remove CALL_PY_WITH_DEFAULTS specialization * Use CALL_NON_PY_GENERAL in more cases when otherwise failing to specialize --- Include/internal/pycore_frame.h | 5 + Include/internal/pycore_opcode_metadata.h | 31 ++- Include/internal/pycore_uop_ids.h | 261 +++++++++++----------- Include/internal/pycore_uop_metadata.h | 26 ++- Include/opcode_ids.h | 116 +++++----- Lib/_opcode_metadata.py | 120 +++++----- Lib/test/test_call.py | 13 +- Lib/test/test_dis.py | 2 +- Lib/test/test_dynamic.py | 3 +- Lib/test/test_glob.py | 3 + Python/bytecodes.c | 141 +++++++++--- Python/ceval.c | 8 +- Python/executor_cases.c.h | 149 +++++++++++- Python/generated_cases.c.h | 235 ++++++++++++++++--- Python/opcode_targets.h | 6 +- Python/optimizer.c | 1 + Python/optimizer_bytecodes.c | 11 +- Python/optimizer_cases.c.h | 56 ++++- Python/specialize.c | 124 ++-------- 19 files changed, 863 insertions(+), 448 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 37ae5ae850389b3..e13fdd9bb2e01cc 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -318,6 +318,11 @@ PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) return (PyGenObject *)(((char *)frame) - offset_in_gen); } +PyAPI_FUNC(_PyInterpreterFrame *) +_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, + PyObject *locals, PyObject* const* args, + size_t argcount, PyObject *kwnames); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index d10224c70f82f51..2a237bc6dd8ee53 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -95,6 +95,8 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; + case CALL_BOUND_METHOD_GENERAL: + return 2 + oparg; case CALL_BUILTIN_CLASS: return 2 + oparg; case CALL_BUILTIN_FAST: @@ -125,9 +127,11 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 2 + oparg; case CALL_METHOD_DESCRIPTOR_O: return 2 + oparg; + case CALL_NON_PY_GENERAL: + return 2 + oparg; case CALL_PY_EXACT_ARGS: return 2 + oparg; - case CALL_PY_WITH_DEFAULTS: + case CALL_PY_GENERAL: return 2 + oparg; case CALL_STR_1: return 3; @@ -524,6 +528,8 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_BOUND_METHOD_EXACT_ARGS: return 0; + case CALL_BOUND_METHOD_GENERAL: + return 0; case CALL_BUILTIN_CLASS: return 1; case CALL_BUILTIN_FAST: @@ -554,10 +560,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 1; case CALL_METHOD_DESCRIPTOR_O: return 1; + case CALL_NON_PY_GENERAL: + return 1; case CALL_PY_EXACT_ARGS: return 0; - case CALL_PY_WITH_DEFAULTS: - return 1; + case CALL_PY_GENERAL: + return 0; case CALL_STR_1: return 1; case CALL_TUPLE_1: @@ -985,6 +993,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_ALLOC_AND_ENTER_INIT] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, + [CALL_BOUND_METHOD_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_CLASS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1000,8 +1009,9 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[268] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_METHOD_DESCRIPTOR_O] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [CALL_NON_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_PY_EXACT_ARGS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, - [CALL_PY_WITH_DEFAULTS] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, + [CALL_PY_GENERAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [CALL_STR_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TUPLE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [CALL_TYPE_1] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1211,6 +1221,7 @@ _PyOpcode_macro_expansion[256] = { [BUILD_STRING] = { .nuops = 1, .uops = { { _BUILD_STRING, 0, 0 } } }, [BUILD_TUPLE] = { .nuops = 1, .uops = { { _BUILD_TUPLE, 0, 0 } } }, [CALL_BOUND_METHOD_EXACT_ARGS] = { .nuops = 8, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _INIT_CALL_BOUND_METHOD_EXACT_ARGS, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_BOUND_METHOD_GENERAL] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_METHOD_VERSION, 2, 1 }, { _EXPAND_METHOD, 0, 0 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_BUILTIN_CLASS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_CLASS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_BUILTIN_FAST] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_BUILTIN_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, @@ -1223,7 +1234,9 @@ _PyOpcode_macro_expansion[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_NOARGS] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_NOARGS, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_METHOD_DESCRIPTOR_O] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_O, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, + [CALL_NON_PY_GENERAL] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE, 0, 0 }, { _CALL_NON_PY_GENERAL, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_PY_EXACT_ARGS] = { .nuops = 6, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_EXACT_ARGS, 2, 1 }, { _CHECK_STACK_SPACE, 0, 0 }, { _INIT_CALL_PY_EXACT_ARGS, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, + [CALL_PY_GENERAL] = { .nuops = 5, .uops = { { _CHECK_PEP_523, 0, 0 }, { _CHECK_FUNCTION_VERSION, 2, 1 }, { _PY_FRAME_GENERAL, 0, 0 }, { _SAVE_RETURN_OFFSET, 7, 3 }, { _PUSH_FRAME, 0, 0 } } }, [CALL_STR_1] = { .nuops = 2, .uops = { { _CALL_STR_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TUPLE_1] = { .nuops = 2, .uops = { { _CALL_TUPLE_1, 0, 0 }, { _CHECK_PERIODIC, 0, 0 } } }, [CALL_TYPE_1] = { .nuops = 1, .uops = { { _CALL_TYPE_1, 0, 0 } } }, @@ -1383,6 +1396,7 @@ const char *_PyOpcode_OpName[268] = { [CALL] = "CALL", [CALL_ALLOC_AND_ENTER_INIT] = "CALL_ALLOC_AND_ENTER_INIT", [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BOUND_METHOD_GENERAL] = "CALL_BOUND_METHOD_GENERAL", [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [CALL_BUILTIN_FAST] = "CALL_BUILTIN_FAST", [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", @@ -1398,8 +1412,9 @@ const char *_PyOpcode_OpName[268] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [CALL_METHOD_DESCRIPTOR_NOARGS] = "CALL_METHOD_DESCRIPTOR_NOARGS", [CALL_METHOD_DESCRIPTOR_O] = "CALL_METHOD_DESCRIPTOR_O", + [CALL_NON_PY_GENERAL] = "CALL_NON_PY_GENERAL", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", - [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", + [CALL_PY_GENERAL] = "CALL_PY_GENERAL", [CALL_STR_1] = "CALL_STR_1", [CALL_TUPLE_1] = "CALL_TUPLE_1", [CALL_TYPE_1] = "CALL_TYPE_1", @@ -1636,6 +1651,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL] = CALL, [CALL_ALLOC_AND_ENTER_INIT] = CALL, [CALL_BOUND_METHOD_EXACT_ARGS] = CALL, + [CALL_BOUND_METHOD_GENERAL] = CALL, [CALL_BUILTIN_CLASS] = CALL, [CALL_BUILTIN_FAST] = CALL, [CALL_BUILTIN_FAST_WITH_KEYWORDS] = CALL, @@ -1651,8 +1667,9 @@ const uint8_t _PyOpcode_Deopt[256] = { [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = CALL, [CALL_METHOD_DESCRIPTOR_NOARGS] = CALL, [CALL_METHOD_DESCRIPTOR_O] = CALL, + [CALL_NON_PY_GENERAL] = CALL, [CALL_PY_EXACT_ARGS] = CALL, - [CALL_PY_WITH_DEFAULTS] = CALL, + [CALL_PY_GENERAL] = CALL, [CALL_STR_1] = CALL, [CALL_TUPLE_1] = CALL, [CALL_TYPE_1] = CALL, @@ -1852,8 +1869,6 @@ const uint8_t _PyOpcode_Deopt[256] = { case 146: \ case 147: \ case 148: \ - case 221: \ - case 222: \ case 223: \ case 224: \ case 225: \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 988464bcc210c89..1e6ef8e54a221ac 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -51,32 +51,35 @@ extern "C" { #define _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 317 #define _CALL_METHOD_DESCRIPTOR_NOARGS 318 #define _CALL_METHOD_DESCRIPTOR_O 319 -#define _CALL_PY_WITH_DEFAULTS CALL_PY_WITH_DEFAULTS -#define _CALL_STR_1 320 -#define _CALL_TUPLE_1 321 +#define _CALL_NON_PY_GENERAL 320 +#define _CALL_STR_1 321 +#define _CALL_TUPLE_1 322 #define _CALL_TYPE_1 CALL_TYPE_1 -#define _CHECK_ATTR_CLASS 322 -#define _CHECK_ATTR_METHOD_LAZY_DICT 323 -#define _CHECK_ATTR_MODULE 324 -#define _CHECK_ATTR_WITH_HINT 325 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 326 +#define _CHECK_ATTR_CLASS 323 +#define _CHECK_ATTR_METHOD_LAZY_DICT 324 +#define _CHECK_ATTR_MODULE 325 +#define _CHECK_ATTR_WITH_HINT 326 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 327 #define _CHECK_EG_MATCH CHECK_EG_MATCH #define _CHECK_EXC_MATCH CHECK_EXC_MATCH -#define _CHECK_FUNCTION 327 -#define _CHECK_FUNCTION_EXACT_ARGS 328 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 329 -#define _CHECK_PEP_523 330 -#define _CHECK_PERIODIC 331 -#define _CHECK_STACK_SPACE 332 -#define _CHECK_STACK_SPACE_OPERAND 333 -#define _CHECK_VALIDITY 334 -#define _CHECK_VALIDITY_AND_SET_IP 335 -#define _COLD_EXIT 336 -#define _COMPARE_OP 337 -#define _COMPARE_OP_FLOAT 338 -#define _COMPARE_OP_INT 339 -#define _COMPARE_OP_STR 340 -#define _CONTAINS_OP 341 +#define _CHECK_FUNCTION 328 +#define _CHECK_FUNCTION_EXACT_ARGS 329 +#define _CHECK_FUNCTION_VERSION 330 +#define _CHECK_IS_NOT_PY_CALLABLE 331 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 332 +#define _CHECK_METHOD_VERSION 333 +#define _CHECK_PEP_523 334 +#define _CHECK_PERIODIC 335 +#define _CHECK_STACK_SPACE 336 +#define _CHECK_STACK_SPACE_OPERAND 337 +#define _CHECK_VALIDITY 338 +#define _CHECK_VALIDITY_AND_SET_IP 339 +#define _COLD_EXIT 340 +#define _COMPARE_OP 341 +#define _COMPARE_OP_FLOAT 342 +#define _COMPARE_OP_INT 343 +#define _COMPARE_OP_STR 344 +#define _CONTAINS_OP 345 #define _CONTAINS_OP_DICT CONTAINS_OP_DICT #define _CONTAINS_OP_SET CONTAINS_OP_SET #define _CONVERT_VALUE CONVERT_VALUE @@ -88,52 +91,53 @@ extern "C" { #define _DELETE_GLOBAL DELETE_GLOBAL #define _DELETE_NAME DELETE_NAME #define _DELETE_SUBSCR DELETE_SUBSCR -#define _DEOPT 342 +#define _DEOPT 346 #define _DICT_MERGE DICT_MERGE #define _DICT_UPDATE DICT_UPDATE -#define _DYNAMIC_EXIT 343 +#define _DYNAMIC_EXIT 347 #define _END_SEND END_SEND -#define _ERROR_POP_N 344 +#define _ERROR_POP_N 348 #define _EXIT_INIT_CHECK EXIT_INIT_CHECK -#define _FATAL_ERROR 345 +#define _EXPAND_METHOD 349 +#define _FATAL_ERROR 350 #define _FORMAT_SIMPLE FORMAT_SIMPLE #define _FORMAT_WITH_SPEC FORMAT_WITH_SPEC -#define _FOR_ITER 346 -#define _FOR_ITER_GEN_FRAME 347 -#define _FOR_ITER_TIER_TWO 348 +#define _FOR_ITER 351 +#define _FOR_ITER_GEN_FRAME 352 +#define _FOR_ITER_TIER_TWO 353 #define _GET_AITER GET_AITER #define _GET_ANEXT GET_ANEXT #define _GET_AWAITABLE GET_AWAITABLE #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 349 -#define _GUARD_BOTH_INT 350 -#define _GUARD_BOTH_UNICODE 351 -#define _GUARD_BUILTINS_VERSION 352 -#define _GUARD_DORV_NO_DICT 353 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 354 -#define _GUARD_GLOBALS_VERSION 355 -#define _GUARD_IS_FALSE_POP 356 -#define _GUARD_IS_NONE_POP 357 -#define _GUARD_IS_NOT_NONE_POP 358 -#define _GUARD_IS_TRUE_POP 359 -#define _GUARD_KEYS_VERSION 360 -#define _GUARD_NOS_FLOAT 361 -#define _GUARD_NOS_INT 362 -#define _GUARD_NOT_EXHAUSTED_LIST 363 -#define _GUARD_NOT_EXHAUSTED_RANGE 364 -#define _GUARD_NOT_EXHAUSTED_TUPLE 365 -#define _GUARD_TOS_FLOAT 366 -#define _GUARD_TOS_INT 367 -#define _GUARD_TYPE_VERSION 368 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 369 -#define _INIT_CALL_PY_EXACT_ARGS 370 -#define _INIT_CALL_PY_EXACT_ARGS_0 371 -#define _INIT_CALL_PY_EXACT_ARGS_1 372 -#define _INIT_CALL_PY_EXACT_ARGS_2 373 -#define _INIT_CALL_PY_EXACT_ARGS_3 374 -#define _INIT_CALL_PY_EXACT_ARGS_4 375 +#define _GUARD_BOTH_FLOAT 354 +#define _GUARD_BOTH_INT 355 +#define _GUARD_BOTH_UNICODE 356 +#define _GUARD_BUILTINS_VERSION 357 +#define _GUARD_DORV_NO_DICT 358 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 359 +#define _GUARD_GLOBALS_VERSION 360 +#define _GUARD_IS_FALSE_POP 361 +#define _GUARD_IS_NONE_POP 362 +#define _GUARD_IS_NOT_NONE_POP 363 +#define _GUARD_IS_TRUE_POP 364 +#define _GUARD_KEYS_VERSION 365 +#define _GUARD_NOS_FLOAT 366 +#define _GUARD_NOS_INT 367 +#define _GUARD_NOT_EXHAUSTED_LIST 368 +#define _GUARD_NOT_EXHAUSTED_RANGE 369 +#define _GUARD_NOT_EXHAUSTED_TUPLE 370 +#define _GUARD_TOS_FLOAT 371 +#define _GUARD_TOS_INT 372 +#define _GUARD_TYPE_VERSION 373 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 374 +#define _INIT_CALL_PY_EXACT_ARGS 375 +#define _INIT_CALL_PY_EXACT_ARGS_0 376 +#define _INIT_CALL_PY_EXACT_ARGS_1 377 +#define _INIT_CALL_PY_EXACT_ARGS_2 378 +#define _INIT_CALL_PY_EXACT_ARGS_3 379 +#define _INIT_CALL_PY_EXACT_ARGS_4 380 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -150,65 +154,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 376 -#define _IS_NONE 377 +#define _INTERNAL_INCREMENT_OPT_COUNTER 381 +#define _IS_NONE 382 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 378 -#define _ITER_CHECK_RANGE 379 -#define _ITER_CHECK_TUPLE 380 -#define _ITER_JUMP_LIST 381 -#define _ITER_JUMP_RANGE 382 -#define _ITER_JUMP_TUPLE 383 -#define _ITER_NEXT_LIST 384 -#define _ITER_NEXT_RANGE 385 -#define _ITER_NEXT_TUPLE 386 -#define _JUMP_TO_TOP 387 +#define _ITER_CHECK_LIST 383 +#define _ITER_CHECK_RANGE 384 +#define _ITER_CHECK_TUPLE 385 +#define _ITER_JUMP_LIST 386 +#define _ITER_JUMP_RANGE 387 +#define _ITER_JUMP_TUPLE 388 +#define _ITER_NEXT_LIST 389 +#define _ITER_NEXT_RANGE 390 +#define _ITER_NEXT_TUPLE 391 +#define _JUMP_TO_TOP 392 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 388 -#define _LOAD_ATTR_CLASS 389 -#define _LOAD_ATTR_CLASS_0 390 -#define _LOAD_ATTR_CLASS_1 391 +#define _LOAD_ATTR 393 +#define _LOAD_ATTR_CLASS 394 +#define _LOAD_ATTR_CLASS_0 395 +#define _LOAD_ATTR_CLASS_1 396 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 392 -#define _LOAD_ATTR_INSTANCE_VALUE_0 393 -#define _LOAD_ATTR_INSTANCE_VALUE_1 394 -#define _LOAD_ATTR_METHOD_LAZY_DICT 395 -#define _LOAD_ATTR_METHOD_NO_DICT 396 -#define _LOAD_ATTR_METHOD_WITH_VALUES 397 -#define _LOAD_ATTR_MODULE 398 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 399 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 400 +#define _LOAD_ATTR_INSTANCE_VALUE 397 +#define _LOAD_ATTR_INSTANCE_VALUE_0 398 +#define _LOAD_ATTR_INSTANCE_VALUE_1 399 +#define _LOAD_ATTR_METHOD_LAZY_DICT 400 +#define _LOAD_ATTR_METHOD_NO_DICT 401 +#define _LOAD_ATTR_METHOD_WITH_VALUES 402 +#define _LOAD_ATTR_MODULE 403 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 404 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 405 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 401 -#define _LOAD_ATTR_SLOT_0 402 -#define _LOAD_ATTR_SLOT_1 403 -#define _LOAD_ATTR_WITH_HINT 404 +#define _LOAD_ATTR_SLOT 406 +#define _LOAD_ATTR_SLOT_0 407 +#define _LOAD_ATTR_SLOT_1 408 +#define _LOAD_ATTR_WITH_HINT 409 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 405 -#define _LOAD_CONST_INLINE_BORROW 406 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 407 -#define _LOAD_CONST_INLINE_WITH_NULL 408 +#define _LOAD_CONST_INLINE 410 +#define _LOAD_CONST_INLINE_BORROW 411 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 412 +#define _LOAD_CONST_INLINE_WITH_NULL 413 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 409 -#define _LOAD_FAST_0 410 -#define _LOAD_FAST_1 411 -#define _LOAD_FAST_2 412 -#define _LOAD_FAST_3 413 -#define _LOAD_FAST_4 414 -#define _LOAD_FAST_5 415 -#define _LOAD_FAST_6 416 -#define _LOAD_FAST_7 417 +#define _LOAD_FAST 414 +#define _LOAD_FAST_0 415 +#define _LOAD_FAST_1 416 +#define _LOAD_FAST_2 417 +#define _LOAD_FAST_3 418 +#define _LOAD_FAST_4 419 +#define _LOAD_FAST_5 420 +#define _LOAD_FAST_6 421 +#define _LOAD_FAST_7 422 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 418 -#define _LOAD_GLOBAL_BUILTINS 419 -#define _LOAD_GLOBAL_MODULE 420 +#define _LOAD_GLOBAL 423 +#define _LOAD_GLOBAL_BUILTINS 424 +#define _LOAD_GLOBAL_MODULE 425 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -222,50 +226,51 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 421 -#define _POP_JUMP_IF_FALSE 422 -#define _POP_JUMP_IF_TRUE 423 +#define _POP_FRAME 426 +#define _POP_JUMP_IF_FALSE 427 +#define _POP_JUMP_IF_TRUE 428 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 424 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 429 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 425 +#define _PUSH_FRAME 430 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 426 +#define _PY_FRAME_GENERAL 431 +#define _REPLACE_WITH_TRUE 432 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR -#define _SAVE_RETURN_OFFSET 427 -#define _SEND 428 +#define _SAVE_RETURN_OFFSET 433 +#define _SEND 434 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 429 -#define _STORE_ATTR 430 -#define _STORE_ATTR_INSTANCE_VALUE 431 -#define _STORE_ATTR_SLOT 432 +#define _START_EXECUTOR 435 +#define _STORE_ATTR 436 +#define _STORE_ATTR_INSTANCE_VALUE 437 +#define _STORE_ATTR_SLOT 438 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 433 -#define _STORE_FAST_0 434 -#define _STORE_FAST_1 435 -#define _STORE_FAST_2 436 -#define _STORE_FAST_3 437 -#define _STORE_FAST_4 438 -#define _STORE_FAST_5 439 -#define _STORE_FAST_6 440 -#define _STORE_FAST_7 441 +#define _STORE_FAST 439 +#define _STORE_FAST_0 440 +#define _STORE_FAST_1 441 +#define _STORE_FAST_2 442 +#define _STORE_FAST_3 443 +#define _STORE_FAST_4 444 +#define _STORE_FAST_5 445 +#define _STORE_FAST_6 446 +#define _STORE_FAST_7 447 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 442 +#define _STORE_SUBSCR 448 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 443 -#define _TO_BOOL 444 +#define _TIER2_RESUME_CHECK 449 +#define _TO_BOOL 450 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -275,13 +280,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 445 +#define _UNPACK_SEQUENCE 451 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 445 +#define MAX_UOP_ID 451 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 475924dcd9c6b83..470e95e2b3b0415 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -193,7 +193,13 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CHECK_ATTR_METHOD_LAZY_DICT] = HAS_DEOPT_FLAG, [_LOAD_ATTR_METHOD_LAZY_DICT] = HAS_ARG_FLAG, [_CHECK_PERIODIC] = HAS_EVAL_BREAK_FLAG, - [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, + [_PY_FRAME_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_FUNCTION_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CHECK_METHOD_VERSION] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_EXPAND_METHOD] = HAS_ARG_FLAG, + [_CHECK_IS_NOT_PY_CALLABLE] = HAS_ARG_FLAG | HAS_EXIT_FLAG, + [_CALL_NON_PY_GENERAL] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, + [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = HAS_ARG_FLAG, [_CHECK_PEP_523] = HAS_DEOPT_FLAG, [_CHECK_FUNCTION_EXACT_ARGS] = HAS_ARG_FLAG | HAS_EXIT_FLAG, @@ -295,6 +301,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", [_CALL_METHOD_DESCRIPTOR_NOARGS] = "_CALL_METHOD_DESCRIPTOR_NOARGS", [_CALL_METHOD_DESCRIPTOR_O] = "_CALL_METHOD_DESCRIPTOR_O", + [_CALL_NON_PY_GENERAL] = "_CALL_NON_PY_GENERAL", [_CALL_STR_1] = "_CALL_STR_1", [_CALL_TUPLE_1] = "_CALL_TUPLE_1", [_CALL_TYPE_1] = "_CALL_TYPE_1", @@ -307,7 +314,10 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_CHECK_EXC_MATCH] = "_CHECK_EXC_MATCH", [_CHECK_FUNCTION] = "_CHECK_FUNCTION", [_CHECK_FUNCTION_EXACT_ARGS] = "_CHECK_FUNCTION_EXACT_ARGS", + [_CHECK_FUNCTION_VERSION] = "_CHECK_FUNCTION_VERSION", + [_CHECK_IS_NOT_PY_CALLABLE] = "_CHECK_IS_NOT_PY_CALLABLE", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", + [_CHECK_METHOD_VERSION] = "_CHECK_METHOD_VERSION", [_CHECK_PEP_523] = "_CHECK_PEP_523", [_CHECK_PERIODIC] = "_CHECK_PERIODIC", [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", @@ -339,6 +349,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_ERROR_POP_N] = "_ERROR_POP_N", [_EXIT_INIT_CHECK] = "_EXIT_INIT_CHECK", [_EXIT_TRACE] = "_EXIT_TRACE", + [_EXPAND_METHOD] = "_EXPAND_METHOD", [_FATAL_ERROR] = "_FATAL_ERROR", [_FORMAT_SIMPLE] = "_FORMAT_SIMPLE", [_FORMAT_WITH_SPEC] = "_FORMAT_WITH_SPEC", @@ -449,6 +460,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_NULL] = "_PUSH_NULL", + [_PY_FRAME_GENERAL] = "_PY_FRAME_GENERAL", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_RETURN_GENERATOR] = "_RETURN_GENERATOR", @@ -850,6 +862,18 @@ int _PyUop_num_popped(int opcode, int oparg) return 1; case _CHECK_PERIODIC: return 0; + case _PY_FRAME_GENERAL: + return 2 + oparg; + case _CHECK_FUNCTION_VERSION: + return 2 + oparg; + case _CHECK_METHOD_VERSION: + return 2 + oparg; + case _EXPAND_METHOD: + return 2 + oparg; + case _CHECK_IS_NOT_PY_CALLABLE: + return 2 + oparg; + case _CALL_NON_PY_GENERAL: + return 2 + oparg; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: return 2 + oparg; case _INIT_CALL_BOUND_METHOD_EXACT_ARGS: diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 185205c6870edcc..647f7c0ecb1ec83 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -144,63 +144,65 @@ extern "C" { #define BINARY_SUBSCR_TUPLE_INT 161 #define CALL_ALLOC_AND_ENTER_INIT 162 #define CALL_BOUND_METHOD_EXACT_ARGS 163 -#define CALL_BUILTIN_CLASS 164 -#define CALL_BUILTIN_FAST 165 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 166 -#define CALL_BUILTIN_O 167 -#define CALL_ISINSTANCE 168 -#define CALL_LEN 169 -#define CALL_LIST_APPEND 170 -#define CALL_METHOD_DESCRIPTOR_FAST 171 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 172 -#define CALL_METHOD_DESCRIPTOR_NOARGS 173 -#define CALL_METHOD_DESCRIPTOR_O 174 -#define CALL_PY_EXACT_ARGS 175 -#define CALL_PY_WITH_DEFAULTS 176 -#define CALL_STR_1 177 -#define CALL_TUPLE_1 178 -#define CALL_TYPE_1 179 -#define COMPARE_OP_FLOAT 180 -#define COMPARE_OP_INT 181 -#define COMPARE_OP_STR 182 -#define CONTAINS_OP_DICT 183 -#define CONTAINS_OP_SET 184 -#define FOR_ITER_GEN 185 -#define FOR_ITER_LIST 186 -#define FOR_ITER_RANGE 187 -#define FOR_ITER_TUPLE 188 -#define LOAD_ATTR_CLASS 189 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 190 -#define LOAD_ATTR_INSTANCE_VALUE 191 -#define LOAD_ATTR_METHOD_LAZY_DICT 192 -#define LOAD_ATTR_METHOD_NO_DICT 193 -#define LOAD_ATTR_METHOD_WITH_VALUES 194 -#define LOAD_ATTR_MODULE 195 -#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 196 -#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 197 -#define LOAD_ATTR_PROPERTY 198 -#define LOAD_ATTR_SLOT 199 -#define LOAD_ATTR_WITH_HINT 200 -#define LOAD_GLOBAL_BUILTIN 201 -#define LOAD_GLOBAL_MODULE 202 -#define LOAD_SUPER_ATTR_ATTR 203 -#define LOAD_SUPER_ATTR_METHOD 204 -#define RESUME_CHECK 205 -#define SEND_GEN 206 -#define STORE_ATTR_INSTANCE_VALUE 207 -#define STORE_ATTR_SLOT 208 -#define STORE_ATTR_WITH_HINT 209 -#define STORE_SUBSCR_DICT 210 -#define STORE_SUBSCR_LIST_INT 211 -#define TO_BOOL_ALWAYS_TRUE 212 -#define TO_BOOL_BOOL 213 -#define TO_BOOL_INT 214 -#define TO_BOOL_LIST 215 -#define TO_BOOL_NONE 216 -#define TO_BOOL_STR 217 -#define UNPACK_SEQUENCE_LIST 218 -#define UNPACK_SEQUENCE_TUPLE 219 -#define UNPACK_SEQUENCE_TWO_TUPLE 220 +#define CALL_BOUND_METHOD_GENERAL 164 +#define CALL_BUILTIN_CLASS 165 +#define CALL_BUILTIN_FAST 166 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 167 +#define CALL_BUILTIN_O 168 +#define CALL_ISINSTANCE 169 +#define CALL_LEN 170 +#define CALL_LIST_APPEND 171 +#define CALL_METHOD_DESCRIPTOR_FAST 172 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 173 +#define CALL_METHOD_DESCRIPTOR_NOARGS 174 +#define CALL_METHOD_DESCRIPTOR_O 175 +#define CALL_NON_PY_GENERAL 176 +#define CALL_PY_EXACT_ARGS 177 +#define CALL_PY_GENERAL 178 +#define CALL_STR_1 179 +#define CALL_TUPLE_1 180 +#define CALL_TYPE_1 181 +#define COMPARE_OP_FLOAT 182 +#define COMPARE_OP_INT 183 +#define COMPARE_OP_STR 184 +#define CONTAINS_OP_DICT 185 +#define CONTAINS_OP_SET 186 +#define FOR_ITER_GEN 187 +#define FOR_ITER_LIST 188 +#define FOR_ITER_RANGE 189 +#define FOR_ITER_TUPLE 190 +#define LOAD_ATTR_CLASS 191 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 192 +#define LOAD_ATTR_INSTANCE_VALUE 193 +#define LOAD_ATTR_METHOD_LAZY_DICT 194 +#define LOAD_ATTR_METHOD_NO_DICT 195 +#define LOAD_ATTR_METHOD_WITH_VALUES 196 +#define LOAD_ATTR_MODULE 197 +#define LOAD_ATTR_NONDESCRIPTOR_NO_DICT 198 +#define LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 199 +#define LOAD_ATTR_PROPERTY 200 +#define LOAD_ATTR_SLOT 201 +#define LOAD_ATTR_WITH_HINT 202 +#define LOAD_GLOBAL_BUILTIN 203 +#define LOAD_GLOBAL_MODULE 204 +#define LOAD_SUPER_ATTR_ATTR 205 +#define LOAD_SUPER_ATTR_METHOD 206 +#define RESUME_CHECK 207 +#define SEND_GEN 208 +#define STORE_ATTR_INSTANCE_VALUE 209 +#define STORE_ATTR_SLOT 210 +#define STORE_ATTR_WITH_HINT 211 +#define STORE_SUBSCR_DICT 212 +#define STORE_SUBSCR_LIST_INT 213 +#define TO_BOOL_ALWAYS_TRUE 214 +#define TO_BOOL_BOOL 215 +#define TO_BOOL_INT 216 +#define TO_BOOL_LIST 217 +#define TO_BOOL_NONE 218 +#define TO_BOOL_STR 219 +#define UNPACK_SEQUENCE_LIST 220 +#define UNPACK_SEQUENCE_TUPLE 221 +#define UNPACK_SEQUENCE_TWO_TUPLE 222 #define INSTRUMENTED_RESUME 236 #define INSTRUMENTED_END_FOR 237 #define INSTRUMENTED_END_SEND 238 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index b5bafe6302bc9ea..b3d7b8103e86c4b 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -88,7 +88,6 @@ "CALL": [ "CALL_BOUND_METHOD_EXACT_ARGS", "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", "CALL_TYPE_1", "CALL_STR_1", "CALL_TUPLE_1", @@ -104,6 +103,9 @@ "CALL_METHOD_DESCRIPTOR_NOARGS", "CALL_METHOD_DESCRIPTOR_FAST", "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", ], } @@ -123,63 +125,65 @@ 'BINARY_SUBSCR_TUPLE_INT': 161, 'CALL_ALLOC_AND_ENTER_INIT': 162, 'CALL_BOUND_METHOD_EXACT_ARGS': 163, - 'CALL_BUILTIN_CLASS': 164, - 'CALL_BUILTIN_FAST': 165, - 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 166, - 'CALL_BUILTIN_O': 167, - 'CALL_ISINSTANCE': 168, - 'CALL_LEN': 169, - 'CALL_LIST_APPEND': 170, - 'CALL_METHOD_DESCRIPTOR_FAST': 171, - 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 172, - 'CALL_METHOD_DESCRIPTOR_NOARGS': 173, - 'CALL_METHOD_DESCRIPTOR_O': 174, - 'CALL_PY_EXACT_ARGS': 175, - 'CALL_PY_WITH_DEFAULTS': 176, - 'CALL_STR_1': 177, - 'CALL_TUPLE_1': 178, - 'CALL_TYPE_1': 179, - 'COMPARE_OP_FLOAT': 180, - 'COMPARE_OP_INT': 181, - 'COMPARE_OP_STR': 182, - 'CONTAINS_OP_DICT': 183, - 'CONTAINS_OP_SET': 184, - 'FOR_ITER_GEN': 185, - 'FOR_ITER_LIST': 186, - 'FOR_ITER_RANGE': 187, - 'FOR_ITER_TUPLE': 188, - 'LOAD_ATTR_CLASS': 189, - 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 190, - 'LOAD_ATTR_INSTANCE_VALUE': 191, - 'LOAD_ATTR_METHOD_LAZY_DICT': 192, - 'LOAD_ATTR_METHOD_NO_DICT': 193, - 'LOAD_ATTR_METHOD_WITH_VALUES': 194, - 'LOAD_ATTR_MODULE': 195, - 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 196, - 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 197, - 'LOAD_ATTR_PROPERTY': 198, - 'LOAD_ATTR_SLOT': 199, - 'LOAD_ATTR_WITH_HINT': 200, - 'LOAD_GLOBAL_BUILTIN': 201, - 'LOAD_GLOBAL_MODULE': 202, - 'LOAD_SUPER_ATTR_ATTR': 203, - 'LOAD_SUPER_ATTR_METHOD': 204, - 'RESUME_CHECK': 205, - 'SEND_GEN': 206, - 'STORE_ATTR_INSTANCE_VALUE': 207, - 'STORE_ATTR_SLOT': 208, - 'STORE_ATTR_WITH_HINT': 209, - 'STORE_SUBSCR_DICT': 210, - 'STORE_SUBSCR_LIST_INT': 211, - 'TO_BOOL_ALWAYS_TRUE': 212, - 'TO_BOOL_BOOL': 213, - 'TO_BOOL_INT': 214, - 'TO_BOOL_LIST': 215, - 'TO_BOOL_NONE': 216, - 'TO_BOOL_STR': 217, - 'UNPACK_SEQUENCE_LIST': 218, - 'UNPACK_SEQUENCE_TUPLE': 219, - 'UNPACK_SEQUENCE_TWO_TUPLE': 220, + 'CALL_BOUND_METHOD_GENERAL': 164, + 'CALL_BUILTIN_CLASS': 165, + 'CALL_BUILTIN_FAST': 166, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, + 'CALL_BUILTIN_O': 168, + 'CALL_ISINSTANCE': 169, + 'CALL_LEN': 170, + 'CALL_LIST_APPEND': 171, + 'CALL_METHOD_DESCRIPTOR_FAST': 172, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, + 'CALL_METHOD_DESCRIPTOR_O': 175, + 'CALL_NON_PY_GENERAL': 176, + 'CALL_PY_EXACT_ARGS': 177, + 'CALL_PY_GENERAL': 178, + 'CALL_STR_1': 179, + 'CALL_TUPLE_1': 180, + 'CALL_TYPE_1': 181, + 'COMPARE_OP_FLOAT': 182, + 'COMPARE_OP_INT': 183, + 'COMPARE_OP_STR': 184, + 'CONTAINS_OP_DICT': 185, + 'CONTAINS_OP_SET': 186, + 'FOR_ITER_GEN': 187, + 'FOR_ITER_LIST': 188, + 'FOR_ITER_RANGE': 189, + 'FOR_ITER_TUPLE': 190, + 'LOAD_ATTR_CLASS': 191, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, + 'LOAD_ATTR_INSTANCE_VALUE': 193, + 'LOAD_ATTR_METHOD_LAZY_DICT': 194, + 'LOAD_ATTR_METHOD_NO_DICT': 195, + 'LOAD_ATTR_METHOD_WITH_VALUES': 196, + 'LOAD_ATTR_MODULE': 197, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, + 'LOAD_ATTR_PROPERTY': 200, + 'LOAD_ATTR_SLOT': 201, + 'LOAD_ATTR_WITH_HINT': 202, + 'LOAD_GLOBAL_BUILTIN': 203, + 'LOAD_GLOBAL_MODULE': 204, + 'LOAD_SUPER_ATTR_ATTR': 205, + 'LOAD_SUPER_ATTR_METHOD': 206, + 'RESUME_CHECK': 207, + 'SEND_GEN': 208, + 'STORE_ATTR_INSTANCE_VALUE': 209, + 'STORE_ATTR_SLOT': 210, + 'STORE_ATTR_WITH_HINT': 211, + 'STORE_SUBSCR_DICT': 212, + 'STORE_SUBSCR_LIST_INT': 213, + 'TO_BOOL_ALWAYS_TRUE': 214, + 'TO_BOOL_BOOL': 215, + 'TO_BOOL_INT': 216, + 'TO_BOOL_LIST': 217, + 'TO_BOOL_NONE': 218, + 'TO_BOOL_STR': 219, + 'UNPACK_SEQUENCE_LIST': 220, + 'UNPACK_SEQUENCE_TUPLE': 221, + 'UNPACK_SEQUENCE_TWO_TUPLE': 222, } opmap = { diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index d3f4d6c29c55367..7ea27929138da3c 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -46,11 +46,16 @@ def test_frames_are_popped_after_failed_calls(self): # recovering from failed calls: def f(): pass - for _ in range(1000): - try: - f(None) - except TypeError: + class C: + def m(self): pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass # BOOM! diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 747a73829fa7058..b68ed3baadc652e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -840,7 +840,7 @@ def loop_test(): %3d LOAD_GLOBAL_MODULE 1 (load_test + NULL) LOAD_FAST 0 (i) - CALL_PY_WITH_DEFAULTS 1 + CALL_PY_GENERAL 1 POP_TOP JUMP_BACKWARD 16 (to L1) diff --git a/Lib/test/test_dynamic.py b/Lib/test/test_dynamic.py index 0aa3be6a1bde6ae..21bece26b893c66 100644 --- a/Lib/test/test_dynamic.py +++ b/Lib/test/test_dynamic.py @@ -4,7 +4,7 @@ import sys import unittest -from test.support import swap_item, swap_attr +from test.support import swap_item, swap_attr, is_wasi, Py_DEBUG class RebindBuiltinsTests(unittest.TestCase): @@ -134,6 +134,7 @@ def test_eval_gives_lambda_custom_globals(self): self.assertEqual(foo(), 7) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_load_global_specialization_failure_keeps_oparg(self): # https://github.com/python/cpython/issues/91625 class MyGlobals(dict): diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 70ee35ed2850bc0..b72640bd871ba65 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -6,6 +6,7 @@ import unittest import warnings +from test.support import is_wasi, Py_DEBUG from test.support.os_helper import (TESTFN, skip_unless_symlink, can_symlink, create_empty_file, change_cwd) @@ -366,6 +367,8 @@ def test_glob_named_pipe(self): self.assertEqual(self.rglob('mypipe', 'sub'), []) self.assertEqual(self.rglob('mypipe', '*'), []) + + @unittest.skipIf(is_wasi and Py_DEBUG, "requires too much stack") def test_glob_many_open_files(self): depth = 30 base = os.path.join(self.tempdir, 'deep') diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ddada96bea71b8c..b2a0dc030e20cc7 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3042,7 +3042,6 @@ dummy_func( family(CALL, INLINE_CACHE_ENTRIES_CALL) = { CALL_BOUND_METHOD_EXACT_ARGS, CALL_PY_EXACT_ARGS, - CALL_PY_WITH_DEFAULTS, CALL_TYPE_1, CALL_STR_1, CALL_TUPLE_1, @@ -3058,6 +3057,9 @@ dummy_func( CALL_METHOD_DESCRIPTOR_NOARGS, CALL_METHOD_DESCRIPTOR_FAST, CALL_ALLOC_AND_ENTER_INIT, + CALL_PY_GENERAL, + CALL_BOUND_METHOD_GENERAL, + CALL_NON_PY_GENERAL, }; specializing op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { @@ -3147,9 +3149,108 @@ dummy_func( macro(CALL) = _SPECIALIZE_CALL + unused/2 + _CALL + _CHECK_PERIODIC; + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _PyInterpreterFrame*)) { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + SYNC_SP(); + if (new_frame == NULL) { + ERROR_NO_POP(); + } + } + + op(_CHECK_FUNCTION_VERSION, (func_version/2, callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + EXIT_IF(!PyFunction_Check(callable)); + PyFunctionObject *func = (PyFunctionObject *)callable; + EXIT_IF(func->func_version != func_version); + } + + macro(CALL_PY_GENERAL) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_FUNCTION_VERSION + + _PY_FRAME_GENERAL + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_METHOD_VERSION, (func_version/2, callable, null, unused[oparg] -- callable, null, unused[oparg])) { + EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); + PyObject *func = ((PyMethodObject *)callable)->im_func; + EXIT_IF(!PyFunction_Check(func)); + EXIT_IF(((PyFunctionObject *)func)->func_version != func_version); + EXIT_IF(null != NULL); + } + + op(_EXPAND_METHOD, (callable, null, unused[oparg] -- method, self, unused[oparg])) { + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + } + + macro(CALL_BOUND_METHOD_GENERAL) = + unused/1 + // Skip over the counter + _CHECK_PEP_523 + + _CHECK_METHOD_VERSION + + _EXPAND_METHOD + + _PY_FRAME_GENERAL + + _SAVE_RETURN_OFFSET + + _PUSH_FRAME; + + op(_CHECK_IS_NOT_PY_CALLABLE, (callable, unused, unused[oparg] -- callable, unused, unused[oparg])) { + EXIT_IF(PyFunction_Check(callable)); + EXIT_IF(Py_TYPE(callable) == &PyMethod_Type); + } + + op(_CALL_NON_PY_GENERAL, (callable, self_or_null, args[oparg] -- res)) { +#if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); +#endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + ERROR_IF(res == NULL, error); + } + + macro(CALL_NON_PY_GENERAL) = + unused/1 + // Skip over the counter + unused/2 + + _CHECK_IS_NOT_PY_CALLABLE + + _CALL_NON_PY_GENERAL + + _CHECK_PERIODIC; + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { - DEOPT_IF(null != NULL); - DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); + EXIT_IF(null != NULL); + EXIT_IF(Py_TYPE(callable) != &PyMethod_Type); } op(_INIT_CALL_BOUND_METHOD_EXACT_ARGS, (callable, unused, unused[oparg] -- func, self, unused[oparg])) { @@ -3227,40 +3328,6 @@ dummy_func( _SAVE_RETURN_OFFSET + _PUSH_FRAME; - inst(CALL_PY_WITH_DEFAULTS, (unused/1, func_version/2, callable, self_or_null, args[oparg] -- unused)) { - DEOPT_IF(tstate->interp->eval_frame); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable)); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount); - DEOPT_IF(argcount < min_args); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; - } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); - } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); - } - inst(CALL_TYPE_1, (unused/1, unused/2, callable, null, arg -- res)) { assert(oparg == 1); DEOPT_IF(null != NULL); diff --git a/Python/ceval.c b/Python/ceval.c index 118746909902f3b..3626ffbd02ff401 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -247,10 +247,6 @@ static PyObject * import_name(PyThreadState *, _PyInterpreterFrame *, static PyObject * import_from(PyThreadState *, PyObject *, PyObject *); static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); -static _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, - PyObject *locals, PyObject* const* args, - size_t argcount, PyObject *kwnames); static _PyInterpreterFrame * _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs); @@ -1716,7 +1712,7 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) } /* Consumes references to func, locals and all the args */ -static _PyInterpreterFrame * +_PyInterpreterFrame * _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, PyObject *locals, PyObject* const* args, size_t argcount, PyObject *kwnames) @@ -1736,6 +1732,8 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return frame; fail: /* Consume the references */ + Py_DECREF(func); + Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { Py_DECREF(args[i]); } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d0b794c61ef4a82..5f15f67324292b6 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3032,6 +3032,153 @@ break; } + case _PY_FRAME_GENERAL: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + _PyInterpreterFrame *new_frame; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + JUMP_TO_ERROR(); + } + stack_pointer[0] = (PyObject *)new_frame; + stack_pointer += 1; + break; + } + + case _CHECK_FUNCTION_VERSION: { + PyObject *callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + if (!PyFunction_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyFunctionObject *func = (PyFunctionObject *)callable; + if (func->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CHECK_METHOD_VERSION: { + PyObject *null; + PyObject *callable; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + uint32_t func_version = (uint32_t)CURRENT_OPERAND(); + if (Py_TYPE(callable) != &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + PyObject *func = ((PyMethodObject *)callable)->im_func; + if (!PyFunction_Check(func)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (((PyFunctionObject *)func)->func_version != func_version) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (null != NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _EXPAND_METHOD: { + PyObject *null; + PyObject *callable; + PyObject *method; + PyObject *self; + oparg = CURRENT_OPARG(); + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + PyObject *callable; + oparg = CURRENT_OPARG(); + callable = stack_pointer[-2 - oparg]; + if (PyFunction_Check(callable)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + if (Py_TYPE(callable) == &PyMethod_Type) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + break; + } + + case _CALL_NON_PY_GENERAL: { + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + oparg = CURRENT_OPARG(); + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { PyObject *null; PyObject *callable; @@ -3276,8 +3423,6 @@ break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 because it uses the 'this_instr' variable */ - case _CALL_TYPE_1: { PyObject *arg; PyObject *null; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 800d19229e3d6a5..87098b0506522f0 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -1002,6 +1002,97 @@ DISPATCH(); } + TARGET(CALL_BOUND_METHOD_GENERAL) { + _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_BOUND_METHOD_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *null; + PyObject *callable; + PyObject *method; + PyObject *self; + PyObject **args; + PyObject *self_or_null; + _PyInterpreterFrame *new_frame; + /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_METHOD_VERSION + null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type, CALL); + PyObject *func = ((PyMethodObject *)callable)->im_func; + DEOPT_IF(!PyFunction_Check(func), CALL); + DEOPT_IF(((PyFunctionObject *)func)->func_version != func_version, CALL); + DEOPT_IF(null != NULL, CALL); + } + // _EXPAND_METHOD + { + assert(null == NULL); + assert(Py_TYPE(callable) == &PyMethod_Type); + self = ((PyMethodObject *)callable)->im_self; + Py_INCREF(self); + stack_pointer[-1 - oparg] = self; // Patch stack as it is used by _PY_FRAME_GENERAL + method = ((PyMethodObject *)callable)->im_func; + assert(PyFunction_Check(method)); + Py_INCREF(method); + Py_DECREF(callable); + } + // _PY_FRAME_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = self; + callable = method; + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + goto error; + } + } + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif + } + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); + } + TARGET(CALL_BUILTIN_CLASS) { frame->instr_ptr = next_instr; next_instr += 4; @@ -1713,6 +1804,56 @@ DISPATCH(); } + TARGET(CALL_NON_PY_GENERAL) { + frame->instr_ptr = next_instr; + next_instr += 4; + INSTRUCTION_STATS(CALL_NON_PY_GENERAL); + static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *callable; + PyObject **args; + PyObject *self_or_null; + PyObject *res; + /* Skip 1 cache entry */ + /* Skip 2 cache entries */ + // _CHECK_IS_NOT_PY_CALLABLE + callable = stack_pointer[-2 - oparg]; + { + DEOPT_IF(PyFunction_Check(callable), CALL); + DEOPT_IF(Py_TYPE(callable) == &PyMethod_Type, CALL); + } + // _CALL_NON_PY_GENERAL + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + { + #if TIER_ONE + assert(opcode != INSTRUMENTED_CALL); + #endif + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { stack_pointer += -2 - oparg; goto error; } + } + // _CHECK_PERIODIC + { + } + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + CHECK_EVAL_BREAKER(); + DISPATCH(); + } + TARGET(CALL_PY_EXACT_ARGS) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; @@ -1786,50 +1927,76 @@ DISPATCH(); } - TARGET(CALL_PY_WITH_DEFAULTS) { + TARGET(CALL_PY_GENERAL) { _Py_CODEUNIT *this_instr = frame->instr_ptr = next_instr; next_instr += 4; - INSTRUCTION_STATS(CALL_PY_WITH_DEFAULTS); + INSTRUCTION_STATS(CALL_PY_GENERAL); static_assert(INLINE_CACHE_ENTRIES_CALL == 3, "incorrect cache size"); + PyObject *callable; PyObject **args; PyObject *self_or_null; - PyObject *callable; + _PyInterpreterFrame *new_frame; /* Skip 1 cache entry */ + // _CHECK_PEP_523 + { + DEOPT_IF(tstate->interp->eval_frame, CALL); + } + // _CHECK_FUNCTION_VERSION + callable = stack_pointer[-2 - oparg]; + { + uint32_t func_version = read_u32(&this_instr[2].cache); + DEOPT_IF(!PyFunction_Check(callable), CALL); + PyFunctionObject *func = (PyFunctionObject *)callable; + DEOPT_IF(func->func_version != func_version, CALL); + } + // _PY_FRAME_GENERAL args = &stack_pointer[-oparg]; self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - uint32_t func_version = read_u32(&this_instr[2].cache); - DEOPT_IF(tstate->interp->eval_frame, CALL); - int argcount = oparg; - if (self_or_null != NULL) { - args--; - argcount++; - } - DEOPT_IF(!PyFunction_Check(callable), CALL); - PyFunctionObject *func = (PyFunctionObject *)callable; - DEOPT_IF(func->func_version != func_version, CALL); - PyCodeObject *code = (PyCodeObject *)func->func_code; - assert(func->func_defaults); - assert(PyTuple_CheckExact(func->func_defaults)); - int defcount = (int)PyTuple_GET_SIZE(func->func_defaults); - assert(defcount <= code->co_argcount); - int min_args = code->co_argcount - defcount; - DEOPT_IF(argcount > code->co_argcount, CALL); - DEOPT_IF(argcount < min_args, CALL); - DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), CALL); - STAT_INC(CALL, hit); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, func, code->co_argcount); - for (int i = 0; i < argcount; i++) { - new_frame->localsplus[i] = args[i]; + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + assert(Py_TYPE(callable) == &PyFunction_Type); + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + stack_pointer += -2 - oparg; + if (new_frame == NULL) { + goto error; + } } - for (int i = argcount; i < code->co_argcount; i++) { - PyObject *def = PyTuple_GET_ITEM(func->func_defaults, i - min_args); - new_frame->localsplus[i] = Py_NewRef(def); + // _SAVE_RETURN_OFFSET + { + #if TIER_ONE + frame->return_offset = (uint16_t)(next_instr - this_instr); + #endif + #if TIER_TWO + frame->return_offset = oparg; + #endif } - // Manipulate stack and cache directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - frame->return_offset = (uint16_t)(next_instr - this_instr); - DISPATCH_INLINED(new_frame); + // _PUSH_FRAME + { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + LLTRACE_RESUME_FRAME(); + } + DISPATCH(); } TARGET(CALL_STR_1) { diff --git a/Python/opcode_targets.h b/Python/opcode_targets.h index 4061ba33cea53ea..fa4f1f8cbb475a0 100644 --- a/Python/opcode_targets.h +++ b/Python/opcode_targets.h @@ -163,6 +163,7 @@ static void *opcode_targets[256] = { &&TARGET_BINARY_SUBSCR_TUPLE_INT, &&TARGET_CALL_ALLOC_AND_ENTER_INIT, &&TARGET_CALL_BOUND_METHOD_EXACT_ARGS, + &&TARGET_CALL_BOUND_METHOD_GENERAL, &&TARGET_CALL_BUILTIN_CLASS, &&TARGET_CALL_BUILTIN_FAST, &&TARGET_CALL_BUILTIN_FAST_WITH_KEYWORDS, @@ -174,8 +175,9 @@ static void *opcode_targets[256] = { &&TARGET_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS, &&TARGET_CALL_METHOD_DESCRIPTOR_NOARGS, &&TARGET_CALL_METHOD_DESCRIPTOR_O, + &&TARGET_CALL_NON_PY_GENERAL, &&TARGET_CALL_PY_EXACT_ARGS, - &&TARGET_CALL_PY_WITH_DEFAULTS, + &&TARGET_CALL_PY_GENERAL, &&TARGET_CALL_STR_1, &&TARGET_CALL_TUPLE_1, &&TARGET_CALL_TYPE_1, @@ -233,8 +235,6 @@ static void *opcode_targets[256] = { &&_unknown_opcode, &&_unknown_opcode, &&_unknown_opcode, - &&_unknown_opcode, - &&_unknown_opcode, &&TARGET_INSTRUMENTED_RESUME, &&TARGET_INSTRUMENTED_END_FOR, &&TARGET_INSTRUMENTED_END_SEND, diff --git a/Python/optimizer.c b/Python/optimizer.c index c0e1be96353d3fb..8be2c0ffbd78e91 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -987,6 +987,7 @@ static void make_exit(_PyUOpInstruction *inst, int opcode, int target) { inst->opcode = opcode; inst->oparg = 0; + inst->operand = 0; inst->format = UOP_FORMAT_TARGET; inst->target = target; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 60763286178c71d..928bc03382b8fbc 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -629,6 +629,15 @@ dummy_func(void) { frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); } + op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + first_valid_check_stack = NULL; + goto done; + } + op(_POP_FRAME, (retval -- res)) { SYNC_SP(); ctx->frame->stack_pointer = stack_pointer; @@ -718,7 +727,7 @@ dummy_func(void) { if (first_valid_check_stack == NULL) { first_valid_check_stack = corresponding_check_stack; } - else { + else if (corresponding_check_stack) { // delete all but the first valid _CHECK_STACK_SPACE corresponding_check_stack->opcode = _NOP; } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b602d663b08ba6c..2a4efd73d794df5 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1559,6 +1559,58 @@ break; } + case _PY_FRAME_GENERAL: { + _Py_UopsSymbol **args; + _Py_UopsSymbol *self_or_null; + _Py_UopsSymbol *callable; + _Py_UOpsAbstractFrame *new_frame; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* The _Py_UOpsAbstractFrame design assumes that we can copy arguments across directly */ + (void)callable; + (void)self_or_null; + (void)args; + first_valid_check_stack = NULL; + goto done; + stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; + stack_pointer += -1 - oparg; + break; + } + + case _CHECK_FUNCTION_VERSION: { + break; + } + + case _CHECK_METHOD_VERSION: { + break; + } + + case _EXPAND_METHOD: { + _Py_UopsSymbol *method; + _Py_UopsSymbol *self; + method = sym_new_not_null(ctx); + if (method == NULL) goto out_of_space; + self = sym_new_not_null(ctx); + if (self == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = method; + stack_pointer[-1 - oparg] = self; + break; + } + + case _CHECK_IS_NOT_PY_CALLABLE: { + break; + } + + case _CALL_NON_PY_GENERAL: { + _Py_UopsSymbol *res; + res = sym_new_not_null(ctx); + if (res == NULL) goto out_of_space; + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + break; + } + case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: { _Py_UopsSymbol *null; _Py_UopsSymbol *callable; @@ -1692,7 +1744,7 @@ if (first_valid_check_stack == NULL) { first_valid_check_stack = corresponding_check_stack; } - else { + else if (corresponding_check_stack) { // delete all but the first valid _CHECK_STACK_SPACE corresponding_check_stack->opcode = _NOP; } @@ -1700,8 +1752,6 @@ break; } - /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ - case _CALL_TYPE_1: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); diff --git a/Python/specialize.c b/Python/specialize.c index 72114f27f69c52a..9ac428c3593f563 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -1789,8 +1789,7 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return -1; } if (Py_TYPE(tp) != &PyType_Type) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_METACLASS); - return -1; + goto generic; } if (tp->tp_new == PyBaseObject_Type.tp_new) { PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); @@ -1807,58 +1806,11 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); return 0; } - return -1; - } - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_CLASS_MUTABLE); - return -1; -} - -#ifdef Py_STATS -static int -builtin_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_CFUNC_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_VARARGS_KEYWORDS; - case METH_NOARGS: - return SPEC_FAIL_CALL_CFUNC_NOARGS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_CFUNC_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; - } -} - -static int -meth_descr_call_fail_kind(int ml_flags) -{ - switch (ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | - METH_KEYWORDS | METH_METHOD)) { - case METH_VARARGS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS; - case METH_VARARGS | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_VARARGS_KEYWORDS; - case METH_METHOD | METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_CALL_METH_DESCR_METHOD_FASTCALL_KEYWORDS; - /* These cases should be optimized, but return "other" just in case */ - case METH_NOARGS: - case METH_O: - case METH_FASTCALL: - case METH_FASTCALL | METH_KEYWORDS: - return SPEC_FAIL_OTHER; - default: - return SPEC_FAIL_CALL_BAD_CALL_FLAGS; } +generic: + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } -#endif // Py_STATS static int specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, @@ -1901,8 +1853,8 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, return 0; } } - SPECIALIZATION_FAIL(CALL, meth_descr_call_fail_kind(descr->d_method->ml_flags)); - return -1; + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } static int @@ -1917,36 +1869,25 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } - if (kind != SIMPLE_FUNCTION) { - SPECIALIZATION_FAIL(CALL, kind); + int argcount = -1; + if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; } - int argcount = code->co_argcount; - int defcount = func->func_defaults == NULL ? 0 : (int)PyTuple_GET_SIZE(func->func_defaults); - int min_args = argcount-defcount; - // GH-105840: min_args is negative when somebody sets too many __defaults__! - if (min_args < 0 || nargs > argcount || nargs < min_args) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); - return -1; + if (kind == SIMPLE_FUNCTION) { + argcount = code->co_argcount; } - assert(nargs <= argcount && nargs >= min_args); - assert(min_args >= 0 && defcount >= 0); - assert(defcount == 0 || func->func_defaults != NULL); int version = _PyFunction_GetVersionForCurrentState(func); if (version == 0) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); return -1; } write_u32(cache->func_version, version); - if (argcount == nargs) { + if (argcount == nargs + bound_method) { instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } - else if (bound_method) { - SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); - return -1; - } else { - instr->op.code = CALL_PY_WITH_DEFAULTS; + instr->op.code = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; } return 0; } @@ -1955,6 +1896,7 @@ static int specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) { if (PyCFunction_GET_FUNCTION(callable) == NULL) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OTHER); return 1; } switch (PyCFunction_GET_FLAGS(callable) & @@ -1991,38 +1933,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) return 0; } default: - SPECIALIZATION_FAIL(CALL, - builtin_call_fail_kind(PyCFunction_GET_FLAGS(callable))); - return 1; - } -} - -#ifdef Py_STATS -static int -call_fail_kind(PyObject *callable) -{ - assert(!PyCFunction_CheckExact(callable)); - assert(!PyFunction_Check(callable)); - assert(!PyType_Check(callable)); - assert(!Py_IS_TYPE(callable, &PyMethodDescr_Type)); - assert(!PyMethod_Check(callable)); - if (PyInstanceMethod_Check(callable)) { - return SPEC_FAIL_CALL_INSTANCE_METHOD; - } - // builtin method - else if (PyCMethod_Check(callable)) { - return SPEC_FAIL_CALL_CMETHOD; - } - else if (Py_TYPE(callable) == &PyWrapperDescr_Type) { - return SPEC_FAIL_CALL_OPERATOR_WRAPPER; - } - else if (Py_TYPE(callable) == &_PyMethodWrapper_Type) { - return SPEC_FAIL_CALL_METHOD_WRAPPER; + instr->op.code = CALL_NON_PY_GENERAL; + return 0; } - return SPEC_FAIL_OTHER; } -#endif // Py_STATS - void _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) @@ -2047,7 +1961,7 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) else if (PyMethod_Check(callable)) { PyObject *func = ((PyMethodObject *)callable)->im_func; if (PyFunction_Check(func)) { - fail = specialize_py_call((PyFunctionObject *)func, instr, nargs+1, true); + fail = specialize_py_call((PyFunctionObject *)func, instr, nargs, true); } else { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_BOUND_METHOD); @@ -2055,8 +1969,8 @@ _Py_Specialize_Call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) } } else { - SPECIALIZATION_FAIL(CALL, call_fail_kind(callable)); - fail = -1; + instr->op.code = CALL_NON_PY_GENERAL; + fail = 0; } if (fail) { STAT_INC(CALL, failure); From b034f14a4b6e9197d3926046721b8b4b4b4f5b3d Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 04:12:10 -0700 Subject: [PATCH 107/220] gh-74929: Implement PEP 667 (GH-115153) --- Doc/data/stable_abi.dat | 3 + Include/ceval.h | 4 + Include/cpython/frameobject.h | 6 + Include/cpython/pyframe.h | 2 + Include/internal/pycore_frame.h | 13 +- Lib/test/test_frame.py | 186 +++- Lib/test/test_listcomps.py | 7 +- Lib/test/test_peepholer.py | 17 - Lib/test/test_stable_abi_ctypes.py | 3 + Lib/test/test_sys.py | 2 +- ...4-04-27-21-44-40.gh-issue-74929.C2nESp.rst | 1 + Misc/stable_abi.toml | 6 + Objects/frameobject.c | 860 ++++++++++++++---- Objects/object.c | 1 + PC/python3dll.c | 3 + Python/ceval.c | 48 +- Python/intrinsics.c | 7 +- Python/sysmodule.c | 8 - Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 + 19 files changed, 921 insertions(+), 257 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 8c8a378f52bd5d5..76a035f194d9115 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -188,6 +188,9 @@ function,PyEval_EvalFrame,3.2,, function,PyEval_EvalFrameEx,3.2,, function,PyEval_GetBuiltins,3.2,, function,PyEval_GetFrame,3.2,, +function,PyEval_GetFrameBuiltins,3.13,, +function,PyEval_GetFrameGlobals,3.13,, +function,PyEval_GetFrameLocals,3.13,, function,PyEval_GetFuncDesc,3.2,, function,PyEval_GetFuncName,3.2,, function,PyEval_GetGlobals,3.2,, diff --git a/Include/ceval.h b/Include/ceval.h index 8ea9da8d134ee0d..1ec746c3708220e 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -22,6 +22,10 @@ PyAPI_FUNC(PyObject *) PyEval_GetGlobals(void); PyAPI_FUNC(PyObject *) PyEval_GetLocals(void); PyAPI_FUNC(PyFrameObject *) PyEval_GetFrame(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameBuiltins(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameGlobals(void); +PyAPI_FUNC(PyObject *) PyEval_GetFrameLocals(void); + PyAPI_FUNC(int) Py_AddPendingCall(int (*func)(void *), void *arg); PyAPI_FUNC(int) Py_MakePendingCalls(void); diff --git a/Include/cpython/frameobject.h b/Include/cpython/frameobject.h index 4e19535c656f2cb..dbbfbb5105ba7aa 100644 --- a/Include/cpython/frameobject.h +++ b/Include/cpython/frameobject.h @@ -27,3 +27,9 @@ PyAPI_FUNC(int) _PyFrame_IsEntryFrame(PyFrameObject *frame); PyAPI_FUNC(int) PyFrame_FastToLocalsWithError(PyFrameObject *f); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); + + +typedef struct { + PyObject_HEAD + PyFrameObject* frame; +} PyFrameLocalsProxyObject; diff --git a/Include/cpython/pyframe.h b/Include/cpython/pyframe.h index c5adbbe4868f698..eeafbb17a56badd 100644 --- a/Include/cpython/pyframe.h +++ b/Include/cpython/pyframe.h @@ -3,8 +3,10 @@ #endif PyAPI_DATA(PyTypeObject) PyFrame_Type; +PyAPI_DATA(PyTypeObject) PyFrameLocalsProxy_Type; #define PyFrame_Check(op) Py_IS_TYPE((op), &PyFrame_Type) +#define PyFrameLocalsProxy_Check(op) Py_IS_TYPE((op), &PyFrameLocalsProxy_Type) PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame); PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame); diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index e13fdd9bb2e01cc..994900c007f4bd0 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -25,7 +25,7 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ + PyObject *f_extra_locals; /* Dict for locals set by users using f_locals, could be NULL */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -245,14 +245,11 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame * frame); int _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg); -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden); - -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame); +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame); -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear); +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame); static inline bool _PyThreadState_HasStackSpace(PyThreadState *tstate, int size) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 8e744a1223e86f8..93d0ea839d16eb1 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -1,3 +1,4 @@ +import copy import gc import operator import re @@ -13,7 +14,7 @@ _testcapi = None from test import support -from test.support import threading_helper, Py_GIL_DISABLED +from test.support import import_helper, threading_helper, Py_GIL_DISABLED from test.support.script_helper import assert_python_ok @@ -198,14 +199,6 @@ def inner(): tb = tb.tb_next return frames - def test_locals(self): - f, outer, inner = self.make_frames() - outer_locals = outer.f_locals - self.assertIsInstance(outer_locals.pop('inner'), types.FunctionType) - self.assertEqual(outer_locals, {'x': 5, 'y': 6}) - inner_locals = inner.f_locals - self.assertEqual(inner_locals, {'x': 5, 'z': 7}) - def test_clear_locals(self): # Test f_locals after clear() (issue #21897) f, outer, inner = self.make_frames() @@ -217,8 +210,8 @@ def test_clear_locals(self): def test_locals_clear_locals(self): # Test f_locals before and after clear() (to exercise caching) f, outer, inner = self.make_frames() - outer.f_locals - inner.f_locals + self.assertNotEqual(outer.f_locals, {}) + self.assertNotEqual(inner.f_locals, {}) outer.clear() inner.clear() self.assertEqual(outer.f_locals, {}) @@ -269,6 +262,177 @@ def inner(): r"^$" % (file_repr, offset + 5)) +class TestFrameLocals(unittest.TestCase): + def test_scope(self): + class A: + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + + self.assertEqual(A.x, 2) + self.assertEqual(A.y, 2) + + def f(): + x = 1 + sys._getframe().f_locals['x'] = 2 + sys._getframe().f_locals['y'] = 2 + self.assertEqual(x, 2) + self.assertEqual(locals()['y'], 2) + f() + + def test_closure(self): + x = 1 + y = 2 + + def f(): + z = x + y + d = sys._getframe().f_locals + self.assertEqual(d['x'], 1) + self.assertEqual(d['y'], 2) + d['x'] = 2 + d['y'] = 3 + + f() + self.assertEqual(x, 2) + self.assertEqual(y, 3) + + def test_as_dict(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + # self, x, y, d + self.assertEqual(len(d), 4) + self.assertIs(d['d'], d) + self.assertEqual(set(d.keys()), set(['x', 'y', 'd', 'self'])) + self.assertEqual(len(d.values()), 4) + self.assertIn(1, d.values()) + self.assertEqual(len(d.items()), 4) + self.assertIn(('x', 1), d.items()) + self.assertEqual(d.__getitem__('x'), 1) + d.__setitem__('x', 2) + self.assertEqual(d['x'], 2) + self.assertEqual(d.get('x'), 2) + self.assertIs(d.get('non_exist', None), None) + self.assertEqual(d.__len__(), 4) + self.assertEqual(set([key for key in d]), set(['x', 'y', 'd', 'self'])) + self.assertIn('x', d) + self.assertTrue(d.__contains__('x')) + + self.assertEqual(reversed(d), list(reversed(d.keys()))) + + d.update({'x': 3, 'z': 4}) + self.assertEqual(d['x'], 3) + self.assertEqual(d['z'], 4) + + with self.assertRaises(TypeError): + d.update([1, 2]) + + self.assertEqual(d.setdefault('x', 5), 3) + self.assertEqual(d.setdefault('new', 5), 5) + self.assertEqual(d['new'], 5) + + with self.assertRaises(KeyError): + d['non_exist'] + + def test_as_number(self): + x = 1 + y = 2 + d = sys._getframe().f_locals + self.assertIn('z', d | {'z': 3}) + d |= {'z': 3} + self.assertEqual(d['z'], 3) + d |= {'y': 3} + self.assertEqual(d['y'], 3) + with self.assertRaises(TypeError): + d |= 3 + with self.assertRaises(TypeError): + _ = d | [3] + + def test_non_string_key(self): + d = sys._getframe().f_locals + d[1] = 2 + self.assertEqual(d[1], 2) + + def test_write_with_hidden(self): + def f(): + f_locals = [sys._getframe().f_locals for b in [0]][0] + f_locals['b'] = 2 + f_locals['c'] = 3 + self.assertEqual(b, 2) + self.assertEqual(c, 3) + b = 0 + c = 0 + f() + + def test_repr(self): + x = 1 + # Introduce a reference cycle + frame = sys._getframe() + self.assertEqual(repr(frame.f_locals), repr(dict(frame.f_locals))) + + def test_delete(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(TypeError): + del d['x'] + + with self.assertRaises(AttributeError): + d.clear() + + with self.assertRaises(AttributeError): + d.pop('x') + + @support.cpython_only + def test_sizeof(self): + proxy = sys._getframe().f_locals + support.check_sizeof(self, proxy, support.calcobjsize("P")) + + def test_unsupport(self): + x = 1 + d = sys._getframe().f_locals + with self.assertRaises(AttributeError): + d.copy() + + with self.assertRaises(TypeError): + copy.copy(d) + + with self.assertRaises(TypeError): + copy.deepcopy(d) + + +class TestFrameCApi(unittest.TestCase): + def test_basic(self): + x = 1 + ctypes = import_helper.import_module('ctypes') + PyEval_GetFrameLocals = ctypes.pythonapi.PyEval_GetFrameLocals + PyEval_GetFrameLocals.restype = ctypes.py_object + frame_locals = PyEval_GetFrameLocals() + self.assertTrue(type(frame_locals), dict) + self.assertEqual(frame_locals['x'], 1) + frame_locals['x'] = 2 + self.assertEqual(x, 1) + + PyEval_GetFrameGlobals = ctypes.pythonapi.PyEval_GetFrameGlobals + PyEval_GetFrameGlobals.restype = ctypes.py_object + frame_globals = PyEval_GetFrameGlobals() + self.assertTrue(type(frame_globals), dict) + self.assertIs(frame_globals, globals()) + + PyEval_GetFrameBuiltins = ctypes.pythonapi.PyEval_GetFrameBuiltins + PyEval_GetFrameBuiltins.restype = ctypes.py_object + frame_builtins = PyEval_GetFrameBuiltins() + self.assertEqual(frame_builtins, __builtins__) + + PyFrame_GetLocals = ctypes.pythonapi.PyFrame_GetLocals + PyFrame_GetLocals.argtypes = [ctypes.py_object] + PyFrame_GetLocals.restype = ctypes.py_object + frame = sys._getframe() + f_locals = PyFrame_GetLocals(frame) + self.assertTrue(f_locals['x'], 1) + f_locals['x'] = 2 + self.assertEqual(x, 2) + + class TestIncompleteFrameAreInvisible(unittest.TestCase): def test_issue95818(self): diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index df1debf35210caf..ec2aac81682db85 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -622,9 +622,14 @@ def test_exception_in_post_comp_call(self): def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) def _recursive_replace(self, maybe_code): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 6989aafd293138f..6c27ee4db97af32 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -933,23 +933,6 @@ def f(): self.assertNotInBytecode(f, "LOAD_FAST_CHECK") return f - def test_deleting_local_warns_and_assigns_none(self): - f = self.make_function_with_no_checks() - co_code = f.__code__.co_code - def trace(frame, event, arg): - if event == 'line' and frame.f_lineno == 4: - del frame.f_locals["x"] - sys.settrace(None) - return None - return trace - e = r"assigning None to unbound local 'x'" - with self.assertWarnsRegex(RuntimeWarning, e): - sys.settrace(trace) - f() - self.assertInBytecode(f, "LOAD_FAST") - self.assertNotInBytecode(f, "LOAD_FAST_CHECK") - self.assertEqual(f.__code__.co_code, co_code) - def test_modifying_local_does_not_add_check(self): f = self.make_function_with_no_checks() def trace(frame, event, arg): diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index d22698168615e28..c06c285c5013a61 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -227,6 +227,9 @@ def test_windows_feature_macros(self): "PyEval_EvalFrameEx", "PyEval_GetBuiltins", "PyEval_GetFrame", + "PyEval_GetFrameBuiltins", + "PyEval_GetFrameGlobals", + "PyEval_GetFrameLocals", "PyEval_GetFuncDesc", "PyEval_GetFuncName", "PyEval_GetGlobals", diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 73912767ae25b74..944e84e88c8a35d 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1561,7 +1561,7 @@ class C(object): pass def func(): return sys._getframe() x = func() - check(x, size('3Pi3c7P2ic??2P')) + check(x, size('3Pi2cP7P2ic??2P')) # function def func(): pass check(func, size('15Pi')) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst new file mode 100644 index 000000000000000..46e628f7fa7dbe3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst @@ -0,0 +1 @@ +Implement PEP 667 - converted ``frame.f_locals`` to a write through proxy diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index 5c29e98705aeaf0..77473662aaa76ca 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2501,3 +2501,9 @@ added = '3.13' [function.PyType_GetModuleByDef] added = '3.13' +[function.PyEval_GetFrameBuiltins] + added = '3.13' +[function.PyEval_GetFrameGlobals] + added = '3.13' +[function.PyEval_GetFrameLocals] + added = '3.13' diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 36538b1f6d53fe1..8030ecb68536747 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -16,12 +16,633 @@ #define OFF(x) offsetof(PyFrameObject, x) + +// Returns borrowed reference or NULL +static PyObject * +framelocalsproxy_getval(_PyInterpreterFrame *frame, PyCodeObject *co, int i) +{ + PyObject **fast = _PyFrame_GetLocalsArray(frame); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *value = fast[i]; + PyObject *cell = NULL; + + if (value == NULL) { + return NULL; + } + + if (kind == CO_FAST_FREE || kind & CO_FAST_CELL) { + // The cell was set when the frame was created from + // the function's closure. + assert(PyCell_Check(value)); + cell = value; + } + + if (cell != NULL) { + value = PyCell_GET(cell); + } + + if (value == NULL) { + return NULL; + } + + return value; +} + +static int +framelocalsproxy_getkeyindex(PyFrameObject *frame, PyObject* key, bool read) +{ + /* + * Returns the fast locals index of the key + * - if read == true, returns the index if the value is not NULL + * - if read == false, returns the index if the value is not hidden + */ + + assert(PyUnicode_CheckExact(key)); + + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + int found_key = false; + + // We do 2 loops here because it's highly possible the key is interned + // and we can do a pointer comparison. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (name == key) { + found_key = true; + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + + if (!found_key) { + // This is unlikely, but we need to make sure. This means the key + // is not interned. + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + if (_PyUnicode_EQ(name, key)) { + if (read) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + return i; + } + } else { + if (!(_PyLocals_GetKind(co->co_localspluskinds, i) & CO_FAST_HIDDEN)) { + return i; + } + } + } + } + } + + return -1; +} + +static PyObject * +framelocalsproxy_getitem(PyObject *self, PyObject *key) +{ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + assert(value != NULL); + return Py_NewRef(value); + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + if (extra != NULL) { + PyObject *value = PyDict_GetItem(extra, key); + if (value != NULL) { + return Py_NewRef(value); + } + } + + PyErr_Format(PyExc_KeyError, "local variable '%R' is not defined", key); + return NULL; +} + +static int +framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) +{ + /* Merge locals into fast locals */ + PyFrameObject* frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyObject** fast = _PyFrame_GetLocalsArray(frame->f_frame); + PyCodeObject* co = _PyFrame_GetCode(frame->f_frame); + + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "cannot remove variables from FrameLocalsProxy"); + return -1; + } + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, false); + if (i >= 0) { + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + + PyObject *oldvalue = fast[i]; + PyObject *cell = NULL; + if (kind == CO_FAST_FREE) { + // The cell was set when the frame was created from + // the function's closure. + assert(oldvalue != NULL && PyCell_Check(oldvalue)); + cell = oldvalue; + } else if (kind & CO_FAST_CELL && oldvalue != NULL) { + if (PyCell_Check(oldvalue)) { + cell = oldvalue; + } + } + if (cell != NULL) { + oldvalue = PyCell_GET(cell); + if (value != oldvalue) { + PyCell_SET(cell, Py_XNewRef(value)); + Py_XDECREF(oldvalue); + } + } else if (value != oldvalue) { + Py_XSETREF(fast[i], Py_NewRef(value)); + } + Py_XDECREF(value); + return 0; + } + } + + // Okay not in the fast locals, try extra locals + + PyObject *extra = frame->f_extra_locals; + + if (extra == NULL) { + extra = PyDict_New(); + if (extra == NULL) { + return -1; + } + frame->f_extra_locals = extra; + } + + assert(PyDict_Check(extra)); + + return PyDict_SetItem(extra, key, value) < 0; +} + +static int +framelocalsproxy_merge(PyObject* self, PyObject* other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + return -1; + } + + PyObject *keys = PyMapping_Keys(other); + PyObject *iter = NULL; + PyObject *key = NULL; + PyObject *value = NULL; + + assert(keys != NULL); + + iter = PyObject_GetIter(keys); + Py_DECREF(keys); + + if (iter == NULL) { + return -1; + } + + while ((key = PyIter_Next(iter)) != NULL) { + value = PyObject_GetItem(other, key); + if (value == NULL) { + Py_DECREF(key); + Py_DECREF(iter); + return -1; + } + + if (framelocalsproxy_setitem(self, key, value) < 0) { + Py_DECREF(key); + Py_DECREF(value); + Py_DECREF(iter); + return -1; + } + + Py_DECREF(key); + Py_DECREF(value); + } + + return 0; +} + +static PyObject * +framelocalsproxy_keys(PyObject *self, PyObject *__unused) +{ + PyObject *names = PyList_New(0); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); + if (val) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyList_Append(names, name); + } + } + + // Iterate through the extra locals + Py_ssize_t i = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + if (frame->f_extra_locals) { + assert(PyDict_Check(frame->f_extra_locals)); + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { + PyList_Append(names, key); + } + } + + return names; +} + +static void +framelocalsproxy_dealloc(PyObject *self) +{ + PyObject_GC_UnTrack(self); + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +framelocalsproxy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyFrameLocalsProxyObject *self = (PyFrameLocalsProxyObject *)type->tp_alloc(type, 0); + if (self == NULL) { + return NULL; + } + + PyFrameObject *frame = (PyFrameObject*)PyTuple_GET_ITEM(args, 0); + assert(PyFrame_Check(frame)); + + ((PyFrameLocalsProxyObject*)self)->frame = (PyFrameObject*)Py_NewRef(frame); + + return (PyObject *)self; +} + +static int +framelocalsproxy_tp_clear(PyObject *self) +{ + Py_CLEAR(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static int +framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(((PyFrameLocalsProxyObject*)self)->frame); + return 0; +} + +static PyObject * +framelocalsproxy_iter(PyObject *self) +{ + return PyObject_GetIter(framelocalsproxy_keys(self, NULL)); +} + +static PyObject * +framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) +{ + if (PyFrameLocalsProxy_Check(other)) { + bool result = ((PyFrameLocalsProxyObject*)self)->frame == ((PyFrameLocalsProxyObject*)other)->frame; + if (op == Py_EQ) { + return PyBool_FromLong(result); + } else if (op == Py_NE) { + return PyBool_FromLong(!result); + } + } else if (PyDict_Check(other)) { + PyObject *dct = PyDict_New(); + PyObject *result = NULL; + PyDict_Update(dct, self); + result = PyObject_RichCompare(dct, other, op); + Py_DECREF(dct); + return result; + } + + Py_RETURN_NOTIMPLEMENTED; +} + +static PyObject * +framelocalsproxy_repr(PyObject *self) +{ + int i = Py_ReprEnter(self); + if (i != 0) { + return i > 0 ? PyUnicode_FromString("{...}") : NULL; + } + + PyObject *dct = PyDict_New(); + PyObject *repr = NULL; + + if (PyDict_Update(dct, self) == 0) { + repr = PyObject_Repr(dct); + } + Py_ReprLeave(self); + + Py_DECREF(dct); + return repr; +} + +static PyObject* +framelocalsproxy_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + PyObject *result = PyDict_New(); + if (PyDict_Update(result, self) < 0) { + Py_DECREF(result); + return NULL; + } + + if (PyDict_Update(result, other) < 0) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_inplace_or(PyObject *self, PyObject *other) +{ + if (!PyDict_Check(other) && !PyFrameLocalsProxy_Check(other)) { + Py_RETURN_NOTIMPLEMENTED; + } + + if (framelocalsproxy_merge(self, other) < 0) { + Py_RETURN_NOTIMPLEMENTED; + } + + return Py_NewRef(self); +} + +static PyObject* +framelocalsproxy_values(PyObject *self, PyObject *__unused) +{ + PyObject *values = PyList_New(0); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + if (value) { + PyList_Append(values, value); + } + } + + // Iterate through the extra locals + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + if (frame->f_extra_locals) { + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyList_Append(values, value); + } + } + + return values; +} + +static PyObject * +framelocalsproxy_items(PyObject *self, PyObject *__unused) +{ + PyObject *items = PyList_New(0); + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + + for (int i = 0; i < co->co_nlocalsplus; i++) { + PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); + PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); + + if (value) { + PyObject *pair = PyTuple_Pack(2, name, value); + PyList_Append(items, pair); + Py_DECREF(pair); + } + } + + // Iterate through the extra locals + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; + + if (frame->f_extra_locals) { + while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { + PyObject *pair = PyTuple_Pack(2, key, value); + PyList_Append(items, pair); + Py_DECREF(pair); + } + } + + return items; +} + +static Py_ssize_t +framelocalsproxy_length(PyObject *self) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + Py_ssize_t size = 0; + + if (frame->f_extra_locals != NULL) { + assert(PyDict_Check(frame->f_extra_locals)); + size += PyDict_Size(frame->f_extra_locals); + } + + for (int i = 0; i < co->co_nlocalsplus; i++) { + if (framelocalsproxy_getval(frame->f_frame, co, i) != NULL) { + size++; + } + } + return size; +} + +static PyObject* +framelocalsproxy_contains(PyObject *self, PyObject *key) +{ + PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; + + if (PyUnicode_CheckExact(key)) { + int i = framelocalsproxy_getkeyindex(frame, key, true); + if (i >= 0) { + Py_RETURN_TRUE; + } + } + + PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; + if (extra != NULL) { + int result = PyDict_Contains(extra, key); + if (result < 0) { + return NULL; + } else if (result > 0) { + Py_RETURN_TRUE; + } + } + + Py_RETURN_FALSE; +} + +static PyObject* +framelocalsproxy_update(PyObject *self, PyObject *other) +{ + if (framelocalsproxy_merge(self, other) < 0) { + PyErr_SetString(PyExc_TypeError, "update() argument must be dict or another FrameLocalsProxy"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +framelocalsproxy_get(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "get expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t nargs) +{ + if (nargs < 1 || nargs > 2) { + PyErr_SetString(PyExc_TypeError, "setdefault expected 1 or 2 arguments"); + return NULL; + } + + PyObject *key = args[0]; + PyObject *default_value = Py_None; + + if (nargs == 2) { + default_value = args[1]; + } + + PyObject *result = framelocalsproxy_getitem(self, key); + + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Clear(); + if (framelocalsproxy_setitem(self, key, default_value) < 0) { + return NULL; + } + return Py_XNewRef(default_value); + } + return NULL; + } + + return result; +} + +static PyObject* +framelocalsproxy_reversed(PyObject *self, PyObject *__unused) +{ + PyObject *result = framelocalsproxy_keys(self, NULL); + if (PyList_Reverse(result) < 0) { + Py_DECREF(result); + return NULL; + } + return result; +} + +static PyNumberMethods framelocalsproxy_as_number = { + .nb_or = framelocalsproxy_or, + .nb_inplace_or = framelocalsproxy_inplace_or, +}; + +static PyMappingMethods framelocalsproxy_as_mapping = { + framelocalsproxy_length, // mp_length + framelocalsproxy_getitem, // mp_subscript + framelocalsproxy_setitem, // mp_ass_subscript +}; + +static PyMethodDef framelocalsproxy_methods[] = { + {"__contains__", framelocalsproxy_contains, METH_O | METH_COEXIST, + NULL}, + {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, + NULL}, + {"__reversed__", framelocalsproxy_reversed, METH_NOARGS, + NULL}, + {"keys", framelocalsproxy_keys, METH_NOARGS, + NULL}, + {"values", framelocalsproxy_values, METH_NOARGS, + NULL}, + {"items", framelocalsproxy_items, METH_NOARGS, + NULL}, + {"update", framelocalsproxy_update, METH_O, + NULL}, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + NULL}, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + NULL}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyFrameLocalsProxy_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + .tp_name = "FrameLocalsProxy", + .tp_basicsize = sizeof(PyFrameLocalsProxyObject), + .tp_dealloc = (destructor)framelocalsproxy_dealloc, + .tp_repr = &framelocalsproxy_repr, + .tp_as_number = &framelocalsproxy_as_number, + .tp_as_mapping = &framelocalsproxy_as_mapping, + .tp_getattro = PyObject_GenericGetAttr, + .tp_setattro = PyObject_GenericSetAttr, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, + .tp_traverse = framelocalsproxy_visit, + .tp_clear = framelocalsproxy_tp_clear, + .tp_richcompare = framelocalsproxy_richcompare, + .tp_iter = framelocalsproxy_iter, + .tp_methods = framelocalsproxy_methods, + .tp_alloc = PyType_GenericAlloc, + .tp_new = framelocalsproxy_new, + .tp_free = PyObject_GC_Del, +}; + +PyObject * +_PyFrameLocalsProxy_New(PyFrameObject *frame) +{ + PyObject* args = PyTuple_Pack(1, frame); + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); + Py_DECREF(args); + return proxy; +} + static PyMemberDef frame_memberlist[] = { {"f_trace_lines", Py_T_BOOL, OFF(f_trace_lines), 0}, {NULL} /* Sentinel */ }; - static PyObject * frame_getlocals(PyFrameObject *f, void *closure) { @@ -30,11 +651,14 @@ frame_getlocals(PyFrameObject *f, void *closure) return NULL; } assert(!_PyFrame_IsIncomplete(f->f_frame)); - PyObject *locals = _PyFrame_GetLocals(f->f_frame, 1); - if (locals) { - f->f_fast_as_locals = 1; + + PyCodeObject *co = _PyFrame_GetCode(f->f_frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { + return Py_NewRef(f->f_frame->f_locals); } - return locals; + + return _PyFrameLocalsProxy_New(f); } int @@ -595,20 +1219,6 @@ first_line_not_before(int *lines, int len, int line) return result; } -static bool -frame_is_cleared(PyFrameObject *frame) -{ - assert(!_PyFrame_IsIncomplete(frame->f_frame)); - if (frame->f_frame->stacktop == 0) { - return true; - } - if (frame->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { - PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); - return gen->gi_frame_state == FRAME_CLEARED; - } - return false; -} - static bool frame_is_suspended(PyFrameObject *frame) { assert(!_PyFrame_IsIncomplete(frame->f_frame)); @@ -900,6 +1510,7 @@ frame_dealloc(PyFrameObject *f) } Py_CLEAR(f->f_back); Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); PyObject_GC_Del(f); Py_XDECREF(co); Py_TRASHCAN_END; @@ -910,6 +1521,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); + Py_VISIT(f->f_extra_locals); if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } @@ -921,6 +1533,7 @@ static int frame_tp_clear(PyFrameObject *f) { Py_CLEAR(f->f_trace); + Py_CLEAR(f->f_extra_locals); /* locals and stack */ PyObject **locals = _PyFrame_GetLocalsArray(f->f_frame); @@ -1056,8 +1669,8 @@ _PyFrame_New_NoTrack(PyCodeObject *code) f->f_trace = NULL; f->f_trace_lines = 1; f->f_trace_opcodes = 0; - f->f_fast_as_locals = 0; f->f_lineno = 0; + f->f_extra_locals = NULL; return f; } @@ -1204,103 +1817,45 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, } -PyObject * -_PyFrame_GetLocals(_PyInterpreterFrame *frame, int include_hidden) +bool +_PyFrame_HasHiddenLocals(_PyInterpreterFrame *frame) { - /* Merge fast locals into f->f_locals */ - PyObject *locals = frame->f_locals; - if (locals == NULL) { - locals = frame->f_locals = PyDict_New(); - if (locals == NULL) { - return NULL; - } - } - PyObject *hidden = NULL; - - /* If include_hidden, "hidden" fast locals (from inlined comprehensions in - module/class scopes) will be included in the returned dict, but not in - frame->f_locals; the returned dict will be a modified copy. Non-hidden - locals will still be updated in frame->f_locals. */ - if (include_hidden) { - hidden = PyDict_New(); - if (hidden == NULL) { - return NULL; - } - } - - frame_init_get_vars(frame); + /* + * This function returns if there are hidden locals introduced by PEP 709, + * which are the isolated fast locals for inline comprehensions + */ + PyCodeObject* co = _PyFrame_GetCode(frame); - PyCodeObject *co = _PyFrame_GetCode(frame); for (int i = 0; i < co->co_nlocalsplus; i++) { - PyObject *value; // borrowed reference - if (!frame_get_var(frame, co, i, &value)) { - continue; - } - - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + if (kind & CO_FAST_HIDDEN) { - if (include_hidden && value != NULL) { - if (PyObject_SetItem(hidden, name, value) != 0) { - goto error; - } - } - continue; - } - if (value == NULL) { - if (PyObject_DelItem(locals, name) != 0) { - if (PyErr_ExceptionMatches(PyExc_KeyError)) { - PyErr_Clear(); - } - else { - goto error; - } - } - } - else { - if (PyObject_SetItem(locals, name, value) != 0) { - goto error; - } - } - } + PyObject* value = framelocalsproxy_getval(frame, co, i); - if (include_hidden && PyDict_Size(hidden)) { - PyObject *innerlocals = PyDict_New(); - if (innerlocals == NULL) { - goto error; - } - if (PyDict_Merge(innerlocals, locals, 1) != 0) { - Py_DECREF(innerlocals); - goto error; - } - if (PyDict_Merge(innerlocals, hidden, 1) != 0) { - Py_DECREF(innerlocals); - goto error; + if (value != NULL) { + return true; + } } - locals = innerlocals; - } - else { - Py_INCREF(locals); } - Py_CLEAR(hidden); - return locals; - - error: - Py_XDECREF(hidden); - return NULL; + return false; } -int -_PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) +PyObject * +_PyFrame_GetLocals(_PyInterpreterFrame *frame) { - PyObject *locals = _PyFrame_GetLocals(frame, 0); - if (locals == NULL) { - return -1; + // We should try to avoid creating the FrameObject if possible. + // So we check if the frame is a module or class level scope + PyCodeObject *co = _PyFrame_GetCode(frame); + + if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + return Py_NewRef(frame->f_locals); } - Py_DECREF(locals); - return 0; + + PyFrameObject* f = _PyFrame_GetFrameObject(frame); + + return _PyFrameLocalsProxy_New(f); } @@ -1354,112 +1909,19 @@ PyFrame_GetVarString(PyFrameObject *frame, const char *name) int PyFrame_FastToLocalsWithError(PyFrameObject *f) { - if (f == NULL) { - PyErr_BadInternalCall(); - return -1; - } - assert(!_PyFrame_IsIncomplete(f->f_frame)); - int err = _PyFrame_FastToLocalsWithError(f->f_frame); - if (err == 0) { - f->f_fast_as_locals = 1; - } - return err; + return 0; } void PyFrame_FastToLocals(PyFrameObject *f) { - int res; - assert(!_PyFrame_IsIncomplete(f->f_frame)); - assert(!PyErr_Occurred()); - - res = PyFrame_FastToLocalsWithError(f); - if (res < 0) - PyErr_Clear(); -} - -void -_PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) -{ - /* Merge locals into fast locals */ - PyObject *locals; - PyObject **fast; - PyCodeObject *co; - locals = frame->f_locals; - if (locals == NULL) { - return; - } - fast = _PyFrame_GetLocalsArray(frame); - co = _PyFrame_GetCode(frame); - - PyObject *exc = PyErr_GetRaisedException(); - for (int i = 0; i < co->co_nlocalsplus; i++) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); - - /* Same test as in PyFrame_FastToLocals() above. */ - if (kind & CO_FAST_FREE && !(co->co_flags & CO_OPTIMIZED)) { - continue; - } - PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyObject *value = PyObject_GetItem(locals, name); - /* We only care about NULLs if clear is true. */ - if (value == NULL) { - PyErr_Clear(); - if (!clear) { - continue; - } - } - PyObject *oldvalue = fast[i]; - PyObject *cell = NULL; - if (kind == CO_FAST_FREE) { - // The cell was set when the frame was created from - // the function's closure. - assert(oldvalue != NULL && PyCell_Check(oldvalue)); - cell = oldvalue; - } - else if (kind & CO_FAST_CELL && oldvalue != NULL) { - /* Same test as in PyFrame_FastToLocals() above. */ - if (PyCell_Check(oldvalue) && - _PyFrame_OpAlreadyRan(frame, MAKE_CELL, i)) { - // (likely) MAKE_CELL must have executed already. - cell = oldvalue; - } - // (unlikely) Otherwise, it must have been set to some - // initial value by an earlier call to PyFrame_LocalsToFast(). - } - if (cell != NULL) { - oldvalue = PyCell_GET(cell); - if (value != oldvalue) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue); - } - } - else if (value != oldvalue) { - if (value == NULL) { - // Probably can't delete this, since the compiler's flow - // analysis may have already "proven" that it exists here: - const char *e = "assigning None to unbound local %R"; - if (PyErr_WarnFormat(PyExc_RuntimeWarning, 0, e, name)) { - // It's okay if frame_obj is NULL, just try anyways: - PyErr_WriteUnraisable((PyObject *)frame->frame_obj); - } - value = Py_NewRef(Py_None); - } - Py_XSETREF(fast[i], Py_NewRef(value)); - } - Py_XDECREF(value); - } - PyErr_SetRaisedException(exc); + return; } void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - assert(!_PyFrame_IsIncomplete(f->f_frame)); - if (f && f->f_fast_as_locals && !frame_is_cleared(f)) { - _PyFrame_LocalsToFast(f->f_frame, clear); - f->f_fast_as_locals = 0; - } + return; } int diff --git a/Objects/object.c b/Objects/object.c index 79e4fb4dbbf7c69..1bf0e65ec60ce4b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2235,6 +2235,7 @@ static PyTypeObject* static_types[] = { &PyFilter_Type, &PyFloat_Type, &PyFrame_Type, + &PyFrameLocalsProxy_Type, &PyFrozenSet_Type, &PyFunction_Type, &PyGen_Type, diff --git a/PC/python3dll.c b/PC/python3dll.c index c6fdc0bd73b9fe7..86c888430891c9a 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -253,6 +253,9 @@ EXPORT_FUNC(PyEval_EvalFrame) EXPORT_FUNC(PyEval_EvalFrameEx) EXPORT_FUNC(PyEval_GetBuiltins) EXPORT_FUNC(PyEval_GetFrame) +EXPORT_FUNC(PyEval_GetFrameBuiltins) +EXPORT_FUNC(PyEval_GetFrameGlobals) +EXPORT_FUNC(PyEval_GetFrameLocals) EXPORT_FUNC(PyEval_GetFuncDesc) EXPORT_FUNC(PyEval_GetFuncName) EXPORT_FUNC(PyEval_GetGlobals) diff --git a/Python/ceval.c b/Python/ceval.c index 3626ffbd02ff401..0d02a9887bef7ad 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2475,12 +2475,7 @@ PyEval_GetLocals(void) return NULL; } - if (_PyFrame_FastToLocalsWithError(current_frame) < 0) { - return NULL; - } - - PyObject *locals = current_frame->f_locals; - assert(locals != NULL); + PyObject *locals = _PyEval_GetFrameLocals(); return locals; } @@ -2494,7 +2489,24 @@ _PyEval_GetFrameLocals(void) return NULL; } - return _PyFrame_GetLocals(current_frame, 1); + PyObject *locals = _PyFrame_GetLocals(current_frame); + if (locals == NULL) { + return NULL; + } + + if (PyFrameLocalsProxy_Check(locals)) { + PyObject* ret = PyDict_New(); + if (PyDict_Update(ret, locals)) { + Py_DECREF(ret); + return NULL; + } + Py_DECREF(locals); + return ret; + } else if (PyMapping_Check(locals)) { + return locals; + } + + return NULL; } PyObject * @@ -2508,6 +2520,28 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject* +PyEval_GetFrameLocals(void) +{ + return _PyEval_GetFrameLocals(); +} + +PyObject* PyEval_GetFrameGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); + if (current_frame == NULL) { + return NULL; + } + return Py_XNewRef(current_frame->f_globals); +} + +PyObject* PyEval_GetFrameBuiltins(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return Py_XNewRef(_PyEval_GetBuiltins(tstate)); +} + int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) { diff --git a/Python/intrinsics.c b/Python/intrinsics.c index 5b10c3ce7d5d775..a6b2c108b671756 100644 --- a/Python/intrinsics.c +++ b/Python/intrinsics.c @@ -123,18 +123,15 @@ static PyObject * import_star(PyThreadState* tstate, PyObject *from) { _PyInterpreterFrame *frame = tstate->current_frame; - if (_PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - PyObject *locals = frame->f_locals; + PyObject *locals = _PyFrame_GetLocals(frame); if (locals == NULL) { _PyErr_SetString(tstate, PyExc_SystemError, "no locals found during 'import *'"); return NULL; } int err = import_all_from(tstate, locals, from); - _PyFrame_LocalsToFast(frame, 0); + Py_DECREF(locals); if (err < 0) { return NULL; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 645b76fccf602c5..bd7f821931da424 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1022,13 +1022,6 @@ static PyObject * call_trampoline(PyThreadState *tstate, PyObject* callback, PyFrameObject *frame, int what, PyObject *arg) { - /* Discard any previous modifications the frame's fast locals */ - if (frame->f_fast_as_locals) { - if (PyFrame_FastToLocalsWithError(frame) < 0) { - return NULL; - } - } - /* call the Python-level function */ if (arg == NULL) { arg = Py_None; @@ -1036,7 +1029,6 @@ call_trampoline(PyThreadState *tstate, PyObject* callback, PyObject *args[3] = {(PyObject *)frame, whatstrings[what], arg}; PyObject *result = _PyObject_VectorcallTstate(tstate, callback, args, 3, NULL); - PyFrame_LocalsToFast(frame, 1); return result; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 285129fd361665f..1b8cccf80872c8c 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -43,6 +43,7 @@ Objects/enumobject.c - PyReversed_Type - Objects/fileobject.c - PyStdPrinter_Type - Objects/floatobject.c - PyFloat_Type - Objects/frameobject.c - PyFrame_Type - +Objects/frameobject.c - PyFrameLocalsProxy_Type - Objects/funcobject.c - PyClassMethod_Type - Objects/funcobject.c - PyFunction_Type - Objects/funcobject.c - PyStaticMethod_Type - From d5e6c7cb667465125ff7fe2c643ccb2abb40fcd0 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Sat, 4 May 2024 08:46:32 -0500 Subject: [PATCH 108/220] fix comment typo in importlib (#118567) --- Lib/importlib/_bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6d6292f95592534..de5651f0a7fc361 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1134,7 +1134,7 @@ def find_spec(cls, fullname, path=None, target=None): # part of the importer), instead of here (the finder part). # The loader is the usual place to get the data that will # be loaded into the module. (For example, see _LoaderBasics - # in _bootstra_external.py.) Most importantly, this importer + # in _bootstrap_external.py.) Most importantly, this importer # is simpler if we wait to get the data. # However, getting as much data in the finder as possible # to later load the module is okay, and sometimes important. From f6b5d3bdc83f8daca05e8b379190587a236585ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 14:47:48 +0100 Subject: [PATCH 109/220] build(deps): bump hypothesis from 6.100.0 to 6.100.2 in /Tools (#118462) Bumps [hypothesis](https://github.com/HypothesisWorks/hypothesis) from 6.100.0 to 6.100.2. - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/hypothesis-python-6.100.0...hypothesis-python-6.100.2) --- Tools/requirements-hypothesis.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 8cfa0df523dd168..9d5a18c881bf362 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.100.0 +hypothesis==6.100.2 From f34e965e52b9bdf157b829371870edfde45b80bf Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 07:44:49 -0700 Subject: [PATCH 110/220] GH-111744: Support opcode events in bdb (GH-111834) --- Lib/bdb.py | 72 +++++++++++++++---- Lib/test/test_bdb.py | 15 +++- Lib/test/test_pdb.py | 1 - ...-11-07-22-41-42.gh-issue-111744.TbLxF0.rst | 1 + 4 files changed, 72 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-11-07-22-41-42.gh-issue-111744.TbLxF0.rst diff --git a/Lib/bdb.py b/Lib/bdb.py index 1acf7957f0d6693..675c8ae51df4c3d 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -32,8 +32,10 @@ def __init__(self, skip=None): self.skip = set(skip) if skip else None self.breaks = {} self.fncache = {} - self.frame_trace_lines = {} + self.frame_trace_lines_opcodes = {} self.frame_returning = None + self.trace_opcodes = False + self.enterframe = None self._load_breaks() @@ -85,6 +87,9 @@ def trace_dispatch(self, frame, event, arg): The arg parameter depends on the previous event. """ + + self.enterframe = frame + if self.quitting: return # None if event == 'line': @@ -101,6 +106,8 @@ def trace_dispatch(self, frame, event, arg): return self.trace_dispatch if event == 'c_return': return self.trace_dispatch + if event == 'opcode': + return self.dispatch_opcode(frame, arg) print('bdb.Bdb.dispatch: unknown debugging event:', repr(event)) return self.trace_dispatch @@ -187,6 +194,17 @@ def dispatch_exception(self, frame, arg): return self.trace_dispatch + def dispatch_opcode(self, frame, arg): + """Invoke user function and return trace function for opcode event. + If the debugger stops on the current opcode, invoke + self.user_opcode(). Raise BdbQuit if self.quitting is set. + Return self.trace_dispatch to continue tracing in this scope. + """ + if self.stop_here(frame) or self.break_here(frame): + self.user_opcode(frame) + if self.quitting: raise BdbQuit + return self.trace_dispatch + # Normally derived classes don't override the following # methods, but they may if they want to redefine the # definition of stopping and breakpoints. @@ -273,7 +291,21 @@ def user_exception(self, frame, exc_info): """Called when we stop on an exception.""" pass - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): + def user_opcode(self, frame): + """Called when we are about to execute an opcode.""" + pass + + def _set_trace_opcodes(self, trace_opcodes): + if trace_opcodes != self.trace_opcodes: + self.trace_opcodes = trace_opcodes + frame = self.enterframe + while frame is not None: + frame.f_trace_opcodes = trace_opcodes + if frame is self.botframe: + break + frame = frame.f_back + + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -286,6 +318,17 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + self._set_trace_opcodes(opcode) + + def _set_caller_tracefunc(self): + # Issue #13183: pdb skips frames after hitting a breakpoint and running + # step commands. + # Restore the trace function in the caller (that may not have been set + # for performance reasons) when returning from the current frame. + if self.frame_returning: + caller_frame = self.frame_returning.f_back + if caller_frame and not caller_frame.f_trace: + caller_frame.f_trace = self.trace_dispatch # Derived classes and clients can call the following methods # to affect the stepping state. @@ -300,16 +343,14 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - # Issue #13183: pdb skips frames after hitting a breakpoint and running - # step commands. - # Restore the trace function in the caller (that may not have been set - # for performance reasons) when returning from the current frame. - if self.frame_returning: - caller_frame = self.frame_returning.f_back - if caller_frame and not caller_frame.f_trace: - caller_frame.f_trace = self.trace_dispatch + self._set_caller_tracefunc() self._set_stopinfo(None, None) + def set_stepinstr(self): + """Stop before the next instruction.""" + self._set_caller_tracefunc() + self._set_stopinfo(None, None, opcode=True) + def set_next(self, frame): """Stop on the next line in or below the given frame.""" self._set_stopinfo(frame, None) @@ -329,11 +370,12 @@ def set_trace(self, frame=None): if frame is None: frame = sys._getframe().f_back self.reset() + self.enterframe = frame while frame: frame.f_trace = self.trace_dispatch self.botframe = frame - # We need f_trace_liens == True for the debugger to work - self.frame_trace_lines[frame] = frame.f_trace_lines + self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes) + # We need f_trace_lines == True for the debugger to work frame.f_trace_lines = True frame = frame.f_back self.set_step() @@ -353,9 +395,9 @@ def set_continue(self): while frame and frame is not self.botframe: del frame.f_trace frame = frame.f_back - for frame, prev_trace_lines in self.frame_trace_lines.items(): - frame.f_trace_lines = prev_trace_lines - self.frame_trace_lines = {} + for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items(): + frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes + self.frame_trace_lines_opcodes = {} def set_quit(self): """Set quitting attribute to True. diff --git a/Lib/test/test_bdb.py b/Lib/test/test_bdb.py index 568c88e326c0873..ed1a63daea11869 100644 --- a/Lib/test/test_bdb.py +++ b/Lib/test/test_bdb.py @@ -228,6 +228,10 @@ def user_exception(self, frame, exc_info): self.process_event('exception', frame) self.next_set_method() + def user_opcode(self, frame): + self.process_event('opcode', frame) + self.next_set_method() + def do_clear(self, arg): # The temporary breakpoints are deleted in user_line(). bp_list = [self.currentbp] @@ -366,7 +370,7 @@ def next_set_method(self): set_method = getattr(self, 'set_' + set_type) # The following set methods give back control to the tracer. - if set_type in ('step', 'continue', 'quit'): + if set_type in ('step', 'stepinstr', 'continue', 'quit'): set_method() return elif set_type in ('next', 'return'): @@ -610,6 +614,15 @@ def test_step_next_on_last_statement(self): with TracerRun(self) as tracer: tracer.runcall(tfunc_main) + def test_stepinstr(self): + self.expect_set = [ + ('line', 2, 'tfunc_main'), ('stepinstr', ), + ('opcode', 2, 'tfunc_main'), ('next', ), + ('line', 3, 'tfunc_main'), ('quit', ), + ] + with TracerRun(self) as tracer: + tracer.runcall(tfunc_main) + def test_next(self): self.expect_set = [ ('line', 2, 'tfunc_main'), ('step', ), diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index c9ea52717490b1d..001562acae74586 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2456,7 +2456,6 @@ def test_pdb_issue_gh_108976(): ... 'continue' ... ]): ... test_function() - bdb.Bdb.dispatch: unknown debugging event: 'opcode' > (5)test_function() -> a = 1 (Pdb) continue diff --git a/Misc/NEWS.d/next/Library/2023-11-07-22-41-42.gh-issue-111744.TbLxF0.rst b/Misc/NEWS.d/next/Library/2023-11-07-22-41-42.gh-issue-111744.TbLxF0.rst new file mode 100644 index 000000000000000..ed856e7667a3729 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-11-07-22-41-42.gh-issue-111744.TbLxF0.rst @@ -0,0 +1 @@ +Support opcode events in :mod:`bdb` From 5f547585fa56c94c5d836b5313a7200f4937ebc4 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 4 May 2024 18:08:38 +0300 Subject: [PATCH 111/220] gh-118569: Add a test for dynamic PEP695 classes (#118570) --- Lib/test/test_type_params.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 4c5bf6bfd33c75a..82f1007f9ac97b4 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -776,6 +776,31 @@ class D[U](T): self.assertIn(int, C.D.__bases__) self.assertIs(C.D.x, str) + +class DynamicClassTest(unittest.TestCase): + def _set_type_params(self, ns, params): + ns['__type_params__'] = params + + def test_types_new_class_with_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}, + lambda ns: self._set_type_params(ns, (T,))) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, (T,)) + self.assertEqual(Klass.__parameters__, (T,)) + + def test_types_new_class_no_callback(self): + T = TypeVar('T', infer_variance=True) + Klass = types.new_class('Klass', (Generic[T],), {}) + + self.assertEqual(Klass.__bases__, (Generic,)) + self.assertEqual(Klass.__orig_bases__, (Generic[T],)) + self.assertEqual(Klass.__type_params__, ()) # must be explicitly set + self.assertEqual(Klass.__parameters__, (T,)) + + class TypeParamsManglingTest(unittest.TestCase): def test_mangling(self): class Foo[__T]: From 978fba58aef347de4a1376e525df2dacc7b2fff3 Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Sat, 4 May 2024 23:45:49 +0800 Subject: [PATCH 112/220] gh-117139: Fix missing semicolon (GH-118573) --- Include/internal/pycore_stackref.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index fd929cd4873a8bd..93898174789f7b3 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -114,7 +114,7 @@ _Py_untag_stack_steal(PyObject **dst, const _PyStackRef *src, size_t length) #define PyStackRef_XSETREF(dst, src) \ do { \ - _PyStackRef *_tmp_dst_ptr = &(dst) \ + _PyStackRef *_tmp_dst_ptr = &(dst); \ _PyStackRef _tmp_old_dst = (*_tmp_dst_ptr); \ *_tmp_dst_ptr = (src); \ PyStackRef_XDECREF(_tmp_old_dst); \ From 291cfa454b9c5b677c955aaf53fab91f0186b6fa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 4 May 2024 15:24:02 -0600 Subject: [PATCH 113/220] gh-117953: Track Extra Details in Global Extensions Cache (gh-118532) We have only been tracking each module's PyModuleDef. However, there are some problems with that. For example, in some cases we load single-phase init extension modules from def->m_base.m_init or def->m_base.m_copy, but if multiple modules share a def then we can end up with unexpected behavior. With this change, we track the following: * PyModuleDef (same as before) * for some modules, its init function or a copy of its __dict__, but specific to that module * whether it is a builtin/core module or a "dynamic" extension * the interpreter (ID) that owns the cached __dict__ (only if cached) This also makes it easier to remember the module's kind (e.g. single-phase init) and if loading it previously failed, which I'm doing separately. --- Include/internal/pycore_importdl.h | 12 +- Python/import.c | 680 +++++++++++++++++++++++------ Python/importdl.c | 24 +- 3 files changed, 578 insertions(+), 138 deletions(-) diff --git a/Include/internal/pycore_importdl.h b/Include/internal/pycore_importdl.h index b0af28da34e0871..e5f222b371a1138 100644 --- a/Include/internal/pycore_importdl.h +++ b/Include/internal/pycore_importdl.h @@ -22,6 +22,11 @@ typedef enum ext_module_kind { _Py_ext_module_kind_INVALID = 3, } _Py_ext_module_kind; +typedef enum ext_module_origin { + _Py_ext_module_origin_CORE = 1, + _Py_ext_module_origin_BUILTIN = 2, + _Py_ext_module_origin_DYNAMIC = 3, +} _Py_ext_module_origin; /* Input for loading an extension module. */ struct _Py_ext_module_loader_info { @@ -34,6 +39,7 @@ struct _Py_ext_module_loader_info { /* path is always a borrowed ref of name or filename, * depending on if it's builtin or not. */ PyObject *path; + _Py_ext_module_origin origin; const char *hook_prefix; const char *newcontext; }; @@ -42,7 +48,11 @@ extern void _Py_ext_module_loader_info_clear( extern int _Py_ext_module_loader_info_init( struct _Py_ext_module_loader_info *info, PyObject *name, - PyObject *filename); + PyObject *filename, + _Py_ext_module_origin origin); +extern int _Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *p_info, + PyObject *name); extern int _Py_ext_module_loader_info_init_for_builtin( struct _Py_ext_module_loader_info *p_info, PyObject *name); diff --git a/Python/import.c b/Python/import.c index 4f91f03f091edd6..fa0e548c9bf4c41 100644 --- a/Python/import.c +++ b/Python/import.c @@ -34,6 +34,17 @@ module _imp #include "clinic/import.c.h" +#ifndef NDEBUG +static bool +is_interpreter_isolated(PyInterpreterState *interp) +{ + return !_Py_IsMainInterpreter(interp) + && !(interp->feature_flags & Py_RTFLAGS_USE_MAIN_OBMALLOC) + && interp->ceval.own_gil; +} +#endif + + /*******************************/ /* process-global import state */ /*******************************/ @@ -435,6 +446,45 @@ _PyImport_GetNextModuleIndex(void) return _Py_atomic_add_ssize(&LAST_MODULE_INDEX, 1) + 1; } +#ifndef NDEBUG +struct extensions_cache_value; +static struct extensions_cache_value * _find_cached_def(PyModuleDef *); +static Py_ssize_t _get_cached_module_index(struct extensions_cache_value *); +#endif + +static Py_ssize_t +_get_module_index_from_def(PyModuleDef *def) +{ + Py_ssize_t index = def->m_base.m_index; + assert(index > 0); +#ifndef NDEBUG + struct extensions_cache_value *cached = _find_cached_def(def); + assert(cached == NULL || index == _get_cached_module_index(cached)); +#endif + return index; +} + +static void +_set_module_index(PyModuleDef *def, Py_ssize_t index) +{ + assert(index > 0); + if (index == def->m_base.m_index) { + /* There's nothing to do. */ + } + else if (def->m_base.m_index == 0) { + /* It should have been initialized by PyModuleDef_Init(). + * We assert here to catch this in dev, but keep going otherwise. */ + assert(def->m_base.m_index != 0); + def->m_base.m_index = index; + } + else { + /* It was already set for a different module. + * We replace the old value. */ + assert(def->m_base.m_index > 0); + def->m_base.m_index = index; + } +} + static const char * _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) { @@ -451,9 +501,8 @@ _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) } static PyObject * -_modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_get(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; if (_modules_by_index_check(interp, index) != NULL) { return NULL; } @@ -463,11 +512,9 @@ _modules_by_index_get(PyInterpreterState *interp, PyModuleDef *def) static int _modules_by_index_set(PyInterpreterState *interp, - PyModuleDef *def, PyObject *module) + Py_ssize_t index, PyObject *module) { - assert(def != NULL); - assert(def->m_slots == NULL); - assert(def->m_base.m_index > 0); + assert(index > 0); if (MODULES_BY_INDEX(interp) == NULL) { MODULES_BY_INDEX(interp) = PyList_New(0); @@ -476,7 +523,6 @@ _modules_by_index_set(PyInterpreterState *interp, } } - Py_ssize_t index = def->m_base.m_index; while (PyList_GET_SIZE(MODULES_BY_INDEX(interp)) <= index) { if (PyList_Append(MODULES_BY_INDEX(interp), Py_None) < 0) { return -1; @@ -487,9 +533,8 @@ _modules_by_index_set(PyInterpreterState *interp, } static int -_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_clear_one(PyInterpreterState *interp, Py_ssize_t index) { - Py_ssize_t index = def->m_base.m_index; const char *err = _modules_by_index_check(interp, index); if (err != NULL) { Py_FatalError(err); @@ -506,7 +551,8 @@ PyState_FindModule(PyModuleDef* module) if (module->m_slots) { return NULL; } - return _modules_by_index_get(interp, module); + Py_ssize_t index = _get_module_index_from_def(module); + return _modules_by_index_get(interp, index); } /* _PyState_AddModule() has been completely removed from the C-API @@ -526,7 +572,9 @@ _PyState_AddModule(PyThreadState *tstate, PyObject* module, PyModuleDef* def) "PyState_AddModule called on module with slots"); return -1; } - return _modules_by_index_set(tstate->interp, def, module); + assert(def->m_slots == NULL); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_set(tstate->interp, index, module); } int @@ -546,7 +594,7 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) } PyInterpreterState *interp = tstate->interp; - Py_ssize_t index = def->m_base.m_index; + Py_ssize_t index = _get_module_index_from_def(def); if (MODULES_BY_INDEX(interp) && index < PyList_GET_SIZE(MODULES_BY_INDEX(interp)) && module == PyList_GET_ITEM(MODULES_BY_INDEX(interp), index)) @@ -555,7 +603,8 @@ PyState_AddModule(PyObject* module, PyModuleDef* def) return -1; } - return _modules_by_index_set(interp, def, module); + assert(def->m_slots == NULL); + return _modules_by_index_set(interp, index, module); } int @@ -568,7 +617,8 @@ PyState_RemoveModule(PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - return _modules_by_index_clear_one(tstate->interp, def); + Py_ssize_t index = _get_module_index_from_def(def); + return _modules_by_index_clear_one(tstate->interp, index); } @@ -587,6 +637,8 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* cleanup the saved copy of module dicts */ PyModuleDef *md = PyModule_GetDef(m); if (md) { + // XXX Do this more carefully. The dict might be owned + // by another interpreter. Py_CLEAR(md->m_base.m_copy); } } @@ -924,6 +976,7 @@ extensions_lock_release(void) PyMutex_Unlock(&_PyRuntime.imports.extensions.mutex); } + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by the tuple @@ -940,6 +993,217 @@ extensions_lock_release(void) dictionary, to avoid loading shared libraries twice. */ +typedef struct cached_m_dict { + /* A shallow copy of the original module's __dict__. */ + PyObject *copied; + /* The interpreter that owns the copy. */ + int64_t interpid; +} *cached_m_dict_t; + +struct extensions_cache_value { + PyModuleDef *def; + + /* The function used to re-initialize the module. + This is only set for legacy (single-phase init) extension modules + and only used for those that support multiple initializations + (m_size >= 0). + It is set by update_global_state_for_extension(). */ + PyModInitFunction m_init; + + /* The module's index into its interpreter's modules_by_index cache. + This is set for all extension modules but only used for legacy ones. + (See PyInterpreterState.modules_by_index for more info.) */ + Py_ssize_t m_index; + + /* A copy of the module's __dict__ after the first time it was loaded. + This is only set/used for legacy modules that do not support + multiple initializations. + It is set exclusively by fixup_cached_def(). */ + cached_m_dict_t m_dict; + struct cached_m_dict _m_dict; + + _Py_ext_module_origin origin; +}; + +static struct extensions_cache_value * +alloc_extensions_cache_value(void) +{ + struct extensions_cache_value *value + = PyMem_RawMalloc(sizeof(struct extensions_cache_value)); + if (value == NULL) { + PyErr_NoMemory(); + return NULL; + } + *value = (struct extensions_cache_value){0}; + return value; +} + +static void +free_extensions_cache_value(struct extensions_cache_value *value) +{ + PyMem_RawFree(value); +} + +static Py_ssize_t +_get_cached_module_index(struct extensions_cache_value *cached) +{ + assert(cached->m_index > 0); + return cached->m_index; +} + +static void +fixup_cached_def(struct extensions_cache_value *value) +{ + /* For the moment, the values in the def's m_base may belong + * to another module, and we're replacing them here. This can + * cause problems later if the old module is reloaded. + * + * Also, we don't decref any old cached values first when we + * replace them here, in case we need to restore them in the + * near future. Instead, the caller is responsible for wrapping + * this up by calling cleanup_old_cached_def() or + * restore_old_cached_def() if there was an error. */ + PyModuleDef *def = value->def; + assert(def != NULL); + + /* We assume that all module defs are statically allocated + and will never be freed. Otherwise, we would incref here. */ + _Py_SetImmortalUntracked((PyObject *)def); + + def->m_base.m_init = value->m_init; + + assert(value->m_index > 0); + _set_module_index(def, value->m_index); + + /* Different modules can share the same def, so we can't just + * expect m_copy to be NULL. */ + assert(def->m_base.m_copy == NULL + || def->m_base.m_init == NULL + || value->m_dict != NULL); + if (value->m_dict != NULL) { + assert(value->m_dict->copied != NULL); + /* As noted above, we don't first decref the old value, if any. */ + def->m_base.m_copy = Py_NewRef(value->m_dict->copied); + } +} + +static void +restore_old_cached_def(PyModuleDef *def, PyModuleDef_Base *oldbase) +{ + def->m_base = *oldbase; +} + +static void +cleanup_old_cached_def(PyModuleDef_Base *oldbase) +{ + Py_XDECREF(oldbase->m_copy); +} + +static void +del_cached_def(struct extensions_cache_value *value) +{ + /* If we hadn't made the stored defs immortal, we would decref here. + However, this decref would be problematic if the module def were + dynamically allocated, it were the last ref, and this function + were called with an interpreter other than the def's owner. */ + assert(value->def == NULL || _Py_IsImmortal(value->def)); + + Py_XDECREF(value->def->m_base.m_copy); + value->def->m_base.m_copy = NULL; +} + +static int +init_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict) +{ + assert(value != NULL); + /* This should only have been called without an m_dict already set. */ + assert(value->m_dict == NULL); + if (m_dict == NULL) { + return 0; + } + assert(PyDict_Check(m_dict)); + assert(value->origin != _Py_ext_module_origin_CORE); + + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!is_interpreter_isolated(interp)); + + /* XXX gh-88216: The copied dict is owned by the current + * interpreter. That's a problem if the interpreter has + * its own obmalloc state or if the module is successfully + * imported into such an interpreter. If the interpreter + * has its own GIL then there may be data races and + * PyImport_ClearModulesByIndex() can crash. Normally, + * a single-phase init module cannot be imported in an + * isolated interpreter, but there are ways around that. + * Hence, heere be dragons! Ideally we would instead do + * something like make a read-only, immortal copy of the + * dict using PyMem_RawMalloc() and store *that* in m_copy. + * Then we'd need to make sure to clear that when the + * runtime is finalized, rather than in + * PyImport_ClearModulesByIndex(). */ + PyObject *copied = PyDict_Copy(m_dict); + if (copied == NULL) { + /* We expect this can only be "out of memory". */ + return -1; + } + // XXX We may want to make the copy immortal. + // XXX This incref shouldn't be necessary. We are decref'ing + // one to many times somewhere. + Py_INCREF(copied); + + value->_m_dict = (struct cached_m_dict){ + .copied=copied, + .interpid=PyInterpreterState_GetID(interp), + }; + + value->m_dict = &value->_m_dict; + return 0; +} + +static void +del_cached_m_dict(struct extensions_cache_value *value) +{ + if (value->m_dict != NULL) { + assert(value->m_dict == &value->_m_dict); + assert(value->m_dict->copied != NULL); + /* In the future we can take advantage of m_dict->interpid + * to decref the dict using the owning interpreter. */ + Py_XDECREF(value->m_dict->copied); + value->m_dict = NULL; + } +} + +static PyObject * get_core_module_dict( + PyInterpreterState *interp, PyObject *name, PyObject *path); + +static PyObject * +get_cached_m_dict(struct extensions_cache_value *value, + PyObject *name, PyObject *path) +{ + assert(value != NULL); + PyInterpreterState *interp = _PyInterpreterState_GET(); + /* It might be a core module (e.g. sys & builtins), + for which we don't cache m_dict. */ + if (value->origin == _Py_ext_module_origin_CORE) { + return get_core_module_dict(interp, name, path); + } + assert(value->def != NULL); + // XXX Switch to value->m_dict. + PyObject *m_dict = value->def->m_base.m_copy; + Py_XINCREF(m_dict); + return m_dict; +} + +static void +del_extensions_cache_value(struct extensions_cache_value *value) +{ + if (value != NULL) { + del_cached_m_dict(value); + del_cached_def(value); + free_extensions_cache_value(value); + } +} + static void * hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) { @@ -953,6 +1217,7 @@ hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) assert(SIZE_MAX - str1_len - str2_len > 2); size_t size = str1_len + 1 + str2_len + 1; + // XXX Use a buffer if it's a temp value (every case but "set"). char *key = PyMem_RawMalloc(size); if (key == NULL) { PyErr_NoMemory(); @@ -984,6 +1249,41 @@ hashtable_destroy_str(void *ptr) PyMem_RawFree(ptr); } +#ifndef NDEBUG +struct hashtable_next_match_def_data { + PyModuleDef *def; + struct extensions_cache_value *matched; +}; + +static int +hashtable_next_match_def(_Py_hashtable_t *ht, + const void *key, const void *value, void *user_data) +{ + if (value == NULL) { + /* It was previously deleted. */ + return 0; + } + struct hashtable_next_match_def_data *data + = (struct hashtable_next_match_def_data *)user_data; + struct extensions_cache_value *cur + = (struct extensions_cache_value *)value; + if (cur->def == data->def) { + data->matched = cur; + return 1; + } + return 0; +} + +static struct extensions_cache_value * +_find_cached_def(PyModuleDef *def) +{ + struct hashtable_next_match_def_data data = {0}; + (void)_Py_hashtable_foreach( + EXTENSIONS.hashtable, hashtable_next_match_def, &data); + return data.matched; +} +#endif + #define HTSEP ':' static int @@ -994,8 +1294,7 @@ _extensions_cache_init(void) hashtable_hash_str, hashtable_compare_str, hashtable_destroy_str, // key - /* There's no need to decref the def since it's immortal. */ - NULL, // value + (_Py_hashtable_destroy_func)del_extensions_cache_value, // value &alloc ); if (EXTENSIONS.hashtable == NULL) { @@ -1027,10 +1326,11 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name, return entry; } -static PyModuleDef * +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * _extensions_cache_get(PyObject *path, PyObject *name) { - PyModuleDef *def = NULL; + struct extensions_cache_value *value = NULL; extensions_lock_acquire(); _Py_hashtable_entry_t *entry = @@ -1039,18 +1339,35 @@ _extensions_cache_get(PyObject *path, PyObject *name) /* It was never added. */ goto finally; } - def = (PyModuleDef *)entry->value; + value = (struct extensions_cache_value *)entry->value; finally: extensions_lock_release(); - return def; + return value; } -static int -_extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def) +/* This can only fail with "out of memory". */ +static struct extensions_cache_value * +_extensions_cache_set(PyObject *path, PyObject *name, + PyModuleDef *def, PyModInitFunction m_init, + Py_ssize_t m_index, PyObject *m_dict, + _Py_ext_module_origin origin) { - int res = -1; + struct extensions_cache_value *value = NULL; + void *key = NULL; + struct extensions_cache_value *newvalue = NULL; + PyModuleDef_Base olddefbase = def->m_base; + assert(def != NULL); + assert(m_init == NULL || m_dict == NULL); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(def->m_base.m_init == NULL || def->m_base.m_init == m_init); + /* For now we don't worry about comparing value->m_copy. */ + assert(def->m_base.m_copy == NULL || m_dict != NULL); + assert((origin == _Py_ext_module_origin_DYNAMIC) == (name != path)); + assert(origin != _Py_ext_module_origin_CORE || m_dict == NULL); + extensions_lock_acquire(); if (EXTENSIONS.hashtable == NULL) { @@ -1059,43 +1376,82 @@ _extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def) } } - int already_set = 0; - void *key = NULL; + /* Create a cached value to populate for the module. */ _Py_hashtable_entry_t *entry = _extensions_cache_find_unlocked(path, name, &key); + value = entry == NULL + ? NULL + : (struct extensions_cache_value *)entry->value; + /* We should never be updating an existing cache value. */ + assert(value == NULL); + if (value != NULL) { + PyErr_Format(PyExc_SystemError, + "extension module %R is already cached", name); + goto finally; + } + newvalue = alloc_extensions_cache_value(); + if (newvalue == NULL) { + goto finally; + } + + /* Populate the new cache value data. */ + *newvalue = (struct extensions_cache_value){ + .def=def, + .m_init=m_init, + .m_index=m_index, + /* m_dict is set by set_cached_m_dict(). */ + .origin=origin, + }; + if (init_cached_m_dict(newvalue, m_dict) < 0) { + goto finally; + } + fixup_cached_def(newvalue); + if (entry == NULL) { /* It was never added. */ - if (_Py_hashtable_set(EXTENSIONS.hashtable, key, def) < 0) { + if (_Py_hashtable_set(EXTENSIONS.hashtable, key, newvalue) < 0) { PyErr_NoMemory(); goto finally; } /* The hashtable owns the key now. */ key = NULL; } - else if (entry->value == NULL) { + else if (value == NULL) { /* It was previously deleted. */ - entry->value = def; + entry->value = newvalue; } else { - /* We expect it to be static, so it must be the same pointer. */ - assert((PyModuleDef *)entry->value == def); - /* It was already added. */ - already_set = 1; + /* We are updating the entry for an existing module. */ + /* We expect def to be static, so it must be the same pointer. */ + assert(value->def == def); + /* We expect the same symbol to be used and the shared object file + * to have remained loaded, so it must be the same pointer. */ + assert(value->m_init == m_init); + /* The same module can't switch between caching __dict__ and not. */ + assert((value->m_dict == NULL) == (m_dict == NULL)); + /* This shouldn't ever happen. */ + Py_UNREACHABLE(); } - if (!already_set) { - /* We assume that all module defs are statically allocated - and will never be freed. Otherwise, we would incref here. */ - _Py_SetImmortal((PyObject *)def); - } - res = 0; + value = newvalue; finally: + if (value == NULL) { + restore_old_cached_def(def, &olddefbase); + if (newvalue != NULL) { + del_extensions_cache_value(newvalue); + } + } + else { + cleanup_old_cached_def(&olddefbase); + } + extensions_lock_release(); if (key != NULL) { hashtable_destroy_str(key); } - return res; + + return value; } static void @@ -1118,13 +1474,11 @@ _extensions_cache_delete(PyObject *path, PyObject *name) /* It was already removed. */ goto finally; } - /* If we hadn't made the stored defs immortal, we would decref here. - However, this decref would be problematic if the module def were - dynamically allocated, it were the last ref, and this function - were called with an interpreter other than the def's owner. */ - assert(_Py_IsImmortal(entry->value)); + struct extensions_cache_value *value = entry->value; entry->value = NULL; + del_extensions_cache_value(value); + finally: extensions_lock_release(); } @@ -1180,11 +1534,11 @@ get_core_module_dict(PyInterpreterState *interp, if (path == name) { assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "sys") == 0) { - return interp->sysdict_copy; + return Py_NewRef(interp->sysdict_copy); } assert(!PyErr_Occurred()); if (PyUnicode_CompareWithASCIIString(name, "builtins") == 0) { - return interp->builtins_copy; + return Py_NewRef(interp->builtins_copy); } assert(!PyErr_Occurred()); } @@ -1207,6 +1561,7 @@ is_core_module(PyInterpreterState *interp, PyObject *name, PyObject *path) return 0; } + #ifndef NDEBUG static _Py_ext_module_kind _get_extension_kind(PyModuleDef *def, bool check_size) @@ -1252,29 +1607,39 @@ _get_extension_kind(PyModuleDef *def, bool check_size) || kind == _Py_ext_module_kind_UNKNOWN); \ } while (0) -#define assert_singlephase(def) \ - assert_singlephase_def(def) +#define assert_singlephase(cached) \ + do { \ + _Py_ext_module_kind kind = _get_extension_kind(cached->def, true); \ + assert(kind == _Py_ext_module_kind_SINGLEPHASE); \ + } while (0) #else /* defined(NDEBUG) */ #define assert_multiphase_def(def) #define assert_singlephase_def(def) -#define assert_singlephase(def) +#define assert_singlephase(cached) #endif struct singlephase_global_update { - PyObject *m_dict; PyModInitFunction m_init; + Py_ssize_t m_index; + PyObject *m_dict; + _Py_ext_module_origin origin; }; -static int +static struct extensions_cache_value * update_global_state_for_extension(PyThreadState *tstate, PyObject *path, PyObject *name, PyModuleDef *def, struct singlephase_global_update *singlephase) { - /* Copy the module's __dict__, if applicable. */ + struct extensions_cache_value *cached = NULL; + PyModInitFunction m_init = NULL; + PyObject *m_dict = NULL; + + /* Set up for _extensions_cache_set(). */ if (singlephase == NULL) { + assert(def->m_base.m_init == NULL); assert(def->m_base.m_copy == NULL); } else { @@ -1288,7 +1653,7 @@ update_global_state_for_extension(PyThreadState *tstate, // We should prevent this somehow. The simplest solution // is probably to store m_copy/m_init in the cache along // with the def, rather than within the def. - def->m_base.m_init = singlephase->m_init; + m_init = singlephase->m_init; } else if (singlephase->m_dict == NULL) { /* It must be a core builtin module. */ @@ -1305,32 +1670,7 @@ update_global_state_for_extension(PyThreadState *tstate, assert(!is_core_module(tstate->interp, name, path)); assert(PyUnicode_CompareWithASCIIString(name, "sys") != 0); assert(PyUnicode_CompareWithASCIIString(name, "builtins") != 0); - /* XXX gh-88216: The copied dict is owned by the current - * interpreter. That's a problem if the interpreter has - * its own obmalloc state or if the module is successfully - * imported into such an interpreter. If the interpreter - * has its own GIL then there may be data races and - * PyImport_ClearModulesByIndex() can crash. Normally, - * a single-phase init module cannot be imported in an - * isolated interpreter, but there are ways around that. - * Hence, heere be dragons! Ideally we would instead do - * something like make a read-only, immortal copy of the - * dict using PyMem_RawMalloc() and store *that* in m_copy. - * Then we'd need to make sure to clear that when the - * runtime is finalized, rather than in - * PyImport_ClearModulesByIndex(). */ - if (def->m_base.m_copy) { - /* Somebody already imported the module, - likely under a different name. - XXX this should really not happen. */ - Py_CLEAR(def->m_base.m_copy); - } - def->m_base.m_copy = PyDict_Copy(singlephase->m_dict); - if (def->m_base.m_copy == NULL) { - // XXX Ignore this error? Doing so would effectively - // mark the module as not loadable. */ - return -1; - } + m_dict = singlephase->m_dict; } } @@ -1338,28 +1678,34 @@ update_global_state_for_extension(PyThreadState *tstate, // XXX Why special-case the main interpreter? if (_Py_IsMainInterpreter(tstate->interp) || def->m_size == -1) { #ifndef NDEBUG - PyModuleDef *cached = _extensions_cache_get(path, name); - assert(cached == NULL || cached == def); + cached = _extensions_cache_get(path, name); + assert(cached == NULL || cached->def == def); #endif - if (_extensions_cache_set(path, name, def) < 0) { - return -1; + cached = _extensions_cache_set( + path, name, def, m_init, singlephase->m_index, m_dict, + singlephase->origin); + if (cached == NULL) { + // XXX Ignore this error? Doing so would effectively + // mark the module as not loadable. + return NULL; } } - return 0; + return cached; } /* For multi-phase init modules, the module is finished * by PyModule_FromDefAndSpec(). */ static int -finish_singlephase_extension(PyThreadState *tstate, - PyObject *mod, PyModuleDef *def, +finish_singlephase_extension(PyThreadState *tstate, PyObject *mod, + struct extensions_cache_value *cached, PyObject *name, PyObject *modules) { assert(mod != NULL && PyModule_Check(mod)); - assert(def == _PyModule_GetDef(mod)); + assert(cached->def == _PyModule_GetDef(mod)); - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { return -1; } @@ -1374,10 +1720,13 @@ finish_singlephase_extension(PyThreadState *tstate, static PyObject * -reload_singlephase_extension(PyThreadState *tstate, PyModuleDef *def, +reload_singlephase_extension(PyThreadState *tstate, + struct extensions_cache_value *cached, struct _Py_ext_module_loader_info *info) { - assert_singlephase(def); + PyModuleDef *def = cached->def; + assert(def != NULL); + assert_singlephase(cached); PyObject *mod = NULL; /* It may have been successfully imported previously @@ -1391,46 +1740,53 @@ reload_singlephase_extension(PyThreadState *tstate, PyModuleDef *def, PyObject *modules = get_modules_dict(tstate, true); if (def->m_size == -1) { - PyObject *m_copy = def->m_base.m_copy; /* Module does not support repeated initialization */ + assert(cached->m_init == NULL); + assert(def->m_base.m_init == NULL); + // XXX Copying the cached dict may break interpreter isolation. + // We could solve this by temporarily acquiring the original + // interpreter's GIL. + PyObject *m_copy = get_cached_m_dict(cached, info->name, info->path); if (m_copy == NULL) { - /* It might be a core module (e.g. sys & builtins), - for which we don't set m_copy. */ - m_copy = get_core_module_dict( - tstate->interp, info->name, info->path); - if (m_copy == NULL) { - assert(!PyErr_Occurred()); - return NULL; - } + assert(!PyErr_Occurred()); + return NULL; } mod = import_add_module(tstate, info->name); if (mod == NULL) { + Py_DECREF(m_copy); return NULL; } PyObject *mdict = PyModule_GetDict(mod); if (mdict == NULL) { + Py_DECREF(m_copy); Py_DECREF(mod); return NULL; } - if (PyDict_Update(mdict, m_copy)) { + int rc = PyDict_Update(mdict, m_copy); + Py_DECREF(m_copy); + if (rc < 0) { Py_DECREF(mod); return NULL; } /* We can't set mod->md_def if it's missing, * because _PyImport_ClearModulesByIndex() might break - * due to violating interpreter isolation. See the note - * in fix_up_extension_for_interpreter(). Until that - * is solved, we leave md_def set to NULL. */ + * due to violating interpreter isolation. + * See the note in set_cached_m_dict(). + * Until that is solved, we leave md_def set to NULL. */ assert(_PyModule_GetDef(mod) == NULL || _PyModule_GetDef(mod) == def); } else { - if (def->m_base.m_init == NULL) { + assert(cached->m_dict == NULL); + assert(def->m_base.m_copy == NULL); + // XXX Use cached->m_init. + PyModInitFunction p0 = def->m_base.m_init; + if (p0 == NULL) { assert(!PyErr_Occurred()); return NULL; } struct _Py_ext_module_loader_result res; - if (_PyImport_RunModInitFunc(def->m_base.m_init, info, &res) < 0) { + if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { _Py_ext_module_loader_result_apply_error(&res, name_buf); return NULL; } @@ -1457,7 +1813,8 @@ reload_singlephase_extension(PyThreadState *tstate, PyModuleDef *def, } } - if (_modules_by_index_set(tstate->interp, def, mod) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_set(tstate->interp, index, mod) < 0) { PyMapping_DelItem(modules, info->name); Py_DECREF(mod); return NULL; @@ -1468,14 +1825,18 @@ reload_singlephase_extension(PyThreadState *tstate, PyModuleDef *def, static PyObject * import_find_extension(PyThreadState *tstate, - struct _Py_ext_module_loader_info *info) + struct _Py_ext_module_loader_info *info, + struct extensions_cache_value **p_cached) { /* Only single-phase init modules will be in the cache. */ - PyModuleDef *def = _extensions_cache_get(info->path, info->name); - if (def == NULL) { + struct extensions_cache_value *cached + = _extensions_cache_get(info->path, info->name); + if (cached == NULL) { return NULL; } - assert_singlephase(def); + assert(cached->def != NULL); + assert_singlephase(cached); + *p_cached = cached; /* It may have been successfully imported previously in an interpreter that allows legacy modules @@ -1486,7 +1847,7 @@ import_find_extension(PyThreadState *tstate, return NULL; } - PyObject *mod = reload_singlephase_extension(tstate, def, info); + PyObject *mod = reload_singlephase_extension(tstate, cached, info); if (mod == NULL) { return NULL; } @@ -1496,6 +1857,7 @@ import_find_extension(PyThreadState *tstate, PySys_FormatStderr("import %U # previously loaded (%R)\n", info->name, info->path); } + return mod; } @@ -1509,6 +1871,7 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, PyObject *mod = NULL; PyModuleDef *def = NULL; + struct extensions_cache_value *cached = NULL; const char *name_buf = PyBytes_AS_STRING(info->name_encoded); struct _Py_ext_module_loader_result res; @@ -1551,7 +1914,14 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, } /* Update global import state. */ - struct singlephase_global_update singlephase = {0}; + assert(def->m_base.m_index != 0); + struct singlephase_global_update singlephase = { + // XXX Modules that share a def should each get their own index, + // whereas currently they share (which means the per-interpreter + // cache is less reliable than it should be). + .m_index=def->m_base.m_index, + .origin=info->origin, + }; // gh-88216: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. if (def->m_size == -1) { @@ -1563,18 +1933,19 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, else { /* We will reload via the init function. */ assert(def->m_size >= 0); + assert(def->m_base.m_copy == NULL); singlephase.m_init = p0; } - if (update_global_state_for_extension( - tstate, info->path, info->name, def, &singlephase) < 0) - { + cached = update_global_state_for_extension( + tstate, info->path, info->name, def, &singlephase); + if (cached == NULL) { goto error; } /* Update per-interpreter import state. */ PyObject *modules = get_modules_dict(tstate, true); if (finish_singlephase_extension( - tstate, mod, def, info->name, modules) < 0) + tstate, mod, cached, info->name, modules) < 0) { goto error; } @@ -1594,13 +1965,14 @@ static int clear_singlephase_extension(PyInterpreterState *interp, PyObject *name, PyObject *path) { - PyModuleDef *def = _extensions_cache_get(path, name); - if (def == NULL) { + struct extensions_cache_value *cached = _extensions_cache_get(path, name); + if (cached == NULL) { if (PyErr_Occurred()) { return -1; } return 0; } + PyModuleDef *def = cached->def; /* Clear data set when the module was initially loaded. */ def->m_base.m_init = NULL; @@ -1608,8 +1980,9 @@ clear_singlephase_extension(PyInterpreterState *interp, // We leave m_index alone since there's no reason to reset it. /* Clear the PyState_*Module() cache entry. */ - if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { - if (_modules_by_index_clear_one(interp, def) < 0) { + Py_ssize_t index = _get_cached_module_index(cached); + if (_modules_by_index_check(interp, index) == NULL) { + if (_modules_by_index_clear_one(interp, index) < 0) { return -1; } } @@ -1652,18 +2025,29 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, assert_singlephase_def(def); assert(def->m_size == -1); assert(def->m_base.m_copy == NULL); - - struct singlephase_global_update singlephase = { - /* We don't want def->m_base.m_copy populated. */ - .m_dict=NULL, - }; - if (update_global_state_for_extension( - tstate, nameobj, nameobj, def, &singlephase) < 0) - { - goto finally; + assert(def->m_base.m_index >= 0); + + /* We aren't using import_find_extension() for core modules, + * so we have to do the extra check to make sure the module + * isn't already in the global cache before calling + * update_global_state_for_extension(). */ + struct extensions_cache_value *cached + = _extensions_cache_get(nameobj, nameobj); + if (cached == NULL) { + struct singlephase_global_update singlephase = { + .m_index=def->m_base.m_index, + /* We don't want def->m_base.m_copy populated. */ + .m_dict=NULL, + .origin=_Py_ext_module_origin_CORE, + }; + cached = update_global_state_for_extension( + tstate, nameobj, nameobj, def, &singlephase); + if (cached == NULL) { + goto finally; + } } - if (finish_singlephase_extension(tstate, mod, def, nameobj, modules) < 0) { + if (finish_singlephase_extension(tstate, mod, cached, nameobj, modules) < 0) { goto finally; } @@ -1700,16 +2084,30 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return NULL; } - PyObject *mod = import_find_extension(tstate, &info); + struct extensions_cache_value *cached = NULL; + PyObject *mod = import_find_extension(tstate, &info, &cached); if (mod != NULL) { assert(!_PyErr_Occurred(tstate)); - assert_singlephase(_PyModule_GetDef(mod)); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); goto finally; } else if (_PyErr_Occurred(tstate)) { goto finally; } + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } + struct _inittab *found = NULL; for (struct _inittab *p = INITTAB; p->name != NULL; p++) { if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { @@ -4054,10 +4452,15 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) return NULL; } - mod = import_find_extension(tstate, &info); + struct extensions_cache_value *cached = NULL; + mod = import_find_extension(tstate, &info, &cached); if (mod != NULL) { assert(!_PyErr_Occurred(tstate)); - assert_singlephase(_PyModule_GetDef(mod)); + assert(cached != NULL); + /* The module might not have md_def set in certain reload cases. */ + assert(_PyModule_GetDef(mod) == NULL + || cached->def == _PyModule_GetDef(mod)); + assert_singlephase(cached); goto finally; } else if (_PyErr_Occurred(tstate)) { @@ -4065,6 +4468,15 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } /* Otherwise it must be multi-phase init or the first time it's loaded. */ + /* If the module was added to the global cache + * but def->m_base.m_copy was cleared (e.g. subinterp fini) + * then we have to do a little dance here. */ + if (cached != NULL) { + assert(cached->def->m_base.m_copy == NULL); + /* For now we clear the cache and move on. */ + _extensions_cache_delete(info.path, info.name); + } + if (PySys_Audit("import", "OOOOO", info.name, info.filename, Py_None, Py_None, Py_None) < 0) { diff --git a/Python/importdl.c b/Python/importdl.c index a3091946bc94bfe..38f56db4b4cc96a 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -111,9 +111,12 @@ _Py_ext_module_loader_info_clear(struct _Py_ext_module_loader_info *info) int _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, - PyObject *name, PyObject *filename) + PyObject *name, PyObject *filename, + _Py_ext_module_origin origin) { - struct _Py_ext_module_loader_info info = {0}; + struct _Py_ext_module_loader_info info = { + .origin=origin, + }; assert(name != NULL); if (!PyUnicode_Check(name)) { @@ -183,12 +186,25 @@ _Py_ext_module_loader_info_init_for_builtin( .name_encoded=name_encoded, /* We won't need filename. */ .path=name, + .origin=_Py_ext_module_origin_BUILTIN, .hook_prefix=ascii_only_prefix, .newcontext=NULL, }; return 0; } +int +_Py_ext_module_loader_info_init_for_core( + struct _Py_ext_module_loader_info *info, + PyObject *name) +{ + if (_Py_ext_module_loader_info_init_for_builtin(info, name) < 0) { + return -1; + } + info->origin = _Py_ext_module_origin_CORE; + return 0; +} + int _Py_ext_module_loader_info_init_from_spec( struct _Py_ext_module_loader_info *p_info, @@ -203,7 +219,9 @@ _Py_ext_module_loader_info_init_from_spec( Py_DECREF(name); return -1; } - int err = _Py_ext_module_loader_info_init(p_info, name, filename); + /* We could also accommodate builtin modules here without much trouble. */ + _Py_ext_module_origin origin = _Py_ext_module_origin_DYNAMIC; + int err = _Py_ext_module_loader_info_init(p_info, name, filename, origin); Py_DECREF(name); Py_DECREF(filename); return err; From 08d169f14a715ceaae3d563ced2ff1633d009359 Mon Sep 17 00:00:00 2001 From: Davide Rizzo Date: Sat, 4 May 2024 23:41:47 +0200 Subject: [PATCH 114/220] gh-109617: fix ncurses incompatibility on macOS with Xcode 15 (#111258) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Łukasz Langa --- Include/py_curses.h | 19 +++++-- ...-10-24-12-39-04.gh-issue-109617.YoI8TV.rst | 2 + Modules/_cursesmodule.c | 17 ++++-- configure | 55 +++++++++++++++---- configure.ac | 5 +- 5 files changed, 77 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-24-12-39-04.gh-issue-109617.YoI8TV.rst diff --git a/Include/py_curses.h b/Include/py_curses.h index e46b08e9cc414e7..a51d9980eee401b 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -23,10 +23,16 @@ # endif #endif -#if !defined(HAVE_CURSES_IS_PAD) && defined(WINDOW_HAS_FLAGS) -/* The following definition is necessary for ncurses 5.7; without it, - some of [n]curses.h set NCURSES_OPAQUE to 1, and then Python - can't get at the WINDOW flags field. */ +#if defined(WINDOW_HAS_FLAGS) && defined(__APPLE__) +/* gh-109617, gh-115383: we can rely on the default value for NCURSES_OPAQUE on + most platforms, but not on macOS. This is because, starting with Xcode 15, + Apple-provided ncurses.h comes from ncurses 6 (which defaults to opaque + structs) but can still be linked to older versions of ncurses dynamic + libraries which don't provide functions such as is_pad() to deal with opaque + structs. Setting NCURSES_OPAQUE to 0 is harmless in all ncurses releases to + this date (provided that a thread-safe implementation is not required), but + this might change in the future. This fix might become irrelevant once + support for macOS 13 or earlier is dropped. */ #define NCURSES_OPAQUE 0 #endif @@ -39,7 +45,10 @@ #ifdef HAVE_NCURSES_H /* configure was checking , but we will use , which has some or all these features. */ -#if !defined(WINDOW_HAS_FLAGS) && !(NCURSES_OPAQUE+0) +#if !defined(WINDOW_HAS_FLAGS) && \ + (NCURSES_VERSION_PATCH+0 < 20070303 || !(NCURSES_OPAQUE+0)) +/* the WINDOW flags field was always accessible in ncurses prior to 20070303; + after that, it depends on the value of NCURSES_OPAQUE. */ #define WINDOW_HAS_FLAGS 1 #endif #if !defined(HAVE_CURSES_IS_PAD) && NCURSES_VERSION_PATCH+0 >= 20090906 diff --git a/Misc/NEWS.d/next/Library/2023-10-24-12-39-04.gh-issue-109617.YoI8TV.rst b/Misc/NEWS.d/next/Library/2023-10-24-12-39-04.gh-issue-109617.YoI8TV.rst new file mode 100644 index 000000000000000..4fda69d332d50a7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-24-12-39-04.gh-issue-109617.YoI8TV.rst @@ -0,0 +1,2 @@ +:mod:`ncurses`: fixed a crash that could occur on macOS 13 or earlier when +Python was built with Apple Xcode 15's SDK. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 8bf6824b6d83ffa..69b97042b89dc77 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1156,8 +1156,10 @@ int py_mvwdelch(WINDOW *w, int y, int x) #endif #if defined(HAVE_CURSES_IS_PAD) +// is_pad() is defined, either as a macro or as a function #define py_is_pad(win) is_pad(win) #elif defined(WINDOW_HAS_FLAGS) +// is_pad() is not defined, but we can inspect WINDOW structure members #define py_is_pad(win) ((win) ? ((win)->_flags & _ISPAD) != 0 : FALSE) #endif @@ -4586,7 +4588,14 @@ make_ncurses_version(PyTypeObject *type) if (ncurses_version == NULL) { return NULL; } - + const char *str = curses_version(); + unsigned long major = 0, minor = 0, patch = 0; + if (!str || sscanf(str, "%*[^0-9]%lu.%lu.%lu", &major, &minor, &patch) < 3) { + // Fallback to header version, which cannot be that wrong + major = NCURSES_VERSION_MAJOR; + minor = NCURSES_VERSION_MINOR; + patch = NCURSES_VERSION_PATCH; + } #define SetIntItem(flag) \ PyStructSequence_SET_ITEM(ncurses_version, pos++, PyLong_FromLong(flag)); \ if (PyErr_Occurred()) { \ @@ -4594,9 +4603,9 @@ make_ncurses_version(PyTypeObject *type) return NULL; \ } - SetIntItem(NCURSES_VERSION_MAJOR) - SetIntItem(NCURSES_VERSION_MINOR) - SetIntItem(NCURSES_VERSION_PATCH) + SetIntItem(major) + SetIntItem(minor) + SetIntItem(patch) #undef SetIntItem return ncurses_version; diff --git a/configure b/configure index cc85aed2aa51c2f..0cd137198d0bb71 100755 --- a/configure +++ b/configure @@ -26781,7 +26781,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26824,7 +26827,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26867,7 +26873,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26910,7 +26919,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26953,7 +26965,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -26996,7 +27011,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27039,7 +27057,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27082,7 +27103,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27125,7 +27149,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27168,7 +27195,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { @@ -27211,7 +27241,10 @@ then : else $as_nop cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include + + #define NCURSES_OPAQUE 0 + #include + int main (void) { diff --git a/configure.ac b/configure.ac index c55e33add20fdec..ae594517df5365b 100644 --- a/configure.ac +++ b/configure.ac @@ -6713,7 +6713,10 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], [py_var], [AC_COMPILE_IFELSE( [AC_LANG_PROGRAM( - [@%:@include ], [ + [ + #define NCURSES_OPAQUE 0 + #include + ], [ #ifndef $1 void *x=$1 #endif From 999f0c512281995fb61a0d9eda075fd846e8c505 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Sat, 4 May 2024 18:22:33 -0500 Subject: [PATCH 115/220] gh-118164: str(10**10000) hangs if the C _decimal module is missing (#118503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Initial stab. * Test the tentative fix. Hangs "forever" without this change. * Move the new test to a better spot. * New comment to explain why _convert_to_str allows any poewr of 10. * Fixed a comment, and fleshed out an existing test that appeared unfinished. * Added temporary asserts. Or maybe permanent ;-) * Update Lib/_pydecimal.py Co-authored-by: Serhiy Storchaka * Remove the new _convert_to_str(). Serhiy and I independently concluded that exact powers of 10 aren't possible in these contexts, so just checking the string length is sufficient. * At least for now, add the asserts to the other block too. * 📜🤖 Added by blurb_it. --------- Co-authored-by: Serhiy Storchaka Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Lib/_pydecimal.py | 19 +++++++++++--- Lib/test/test_decimal.py | 25 ++++++++++++++++++- ...-05-04-20-22-59.gh-issue-118164.9D02MQ.rst | 1 + 3 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index de4561a5ee050b8..613123ec7b43299 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -2131,10 +2131,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2184,13 +2190,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 7010c34792e0937..e927e24b582a5dd 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -4716,9 +4716,33 @@ def test_py_exact_power(self): c.prec = 1 x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('3e-6')) + c.prec = 2 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.6e-6')) + c.prec = 3 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 28 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**(MAX_EMAX + 1). + # Fixed via https://github.com/python/cpython/pull/118503. + c.prec = P.MAX_PREC + c.Emax = P.MAX_EMAX + c.Emin = P.MIN_EMIN + c.traps[P.Inexact] = 1 + D2 = Decimal(2) + # If the bug is still present, the next statement won't complete. + res = D2 ** 117 + self.assertEqual(res, 1 << 117) + def test_py_immutability_operations(self): # Do operations and check that it didn't change internal objects. Decimal = P.Decimal @@ -5705,7 +5729,6 @@ def test_format_fallback_rounding(self): with C.localcontext(rounding=C.ROUND_DOWN): self.assertEqual(format(y, '#.1f'), '6.0') - @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst b/Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst new file mode 100644 index 000000000000000..80dc868540418fa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-04-20-22-59.gh-issue-118164.9D02MQ.rst @@ -0,0 +1 @@ +The Python implementation of the ``decimal`` module could appear to hang in relatively small power cases (like ``2**117``) if context precision was set to a very high value. A different method to check for exactly representable results is used now that doesn't rely on computing ``10**precision`` (which could be effectively too large to compute). From 1b22d801b86ed314c4804b19a1fc4b13484e3cea Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 5 May 2024 03:07:29 +0200 Subject: [PATCH 116/220] gh-118518: Allow perf to work without frame pointers (#112254) --- Doc/c-api/init_config.rst | 5 +- Doc/howto/perf_profiling.rst | 33 + Doc/using/cmdline.rst | 24 + Doc/whatsnew/3.13.rst | 5 + Include/internal/pycore_ceval.h | 1 + Include/internal/pycore_ceval_state.h | 2 + Lib/test/test_perf_profiler.py | 146 ++++- Makefile.pre.in | 1 + ...-05-02-20-32-42.gh-issue-118518.m-JbTi.rst | 4 + PCbuild/_freeze_module.vcxproj | 1 + PCbuild/_freeze_module.vcxproj.filters | 3 + PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 + Python/initconfig.c | 14 + Python/perf_jit_trampoline.c | 615 ++++++++++++++++++ Python/perf_trampoline.c | 52 +- Python/pylifecycle.c | 9 +- Python/sysmodule.c | 10 + Tools/c-analyzer/cpython/ignored.tsv | 2 + 19 files changed, 892 insertions(+), 39 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst create mode 100644 Python/perf_jit_trampoline.c diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 47a8fbb2cd9c97e..63ec25b8e60badb 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1251,7 +1251,10 @@ PyConfig for more information. Set by :option:`-X perf <-X>` command line option and by the - :envvar:`PYTHONPERFSUPPORT` environment variable. + :envvar:`PYTHONPERFSUPPORT` environment variable for perf support + with stack pointers and :option:`-X perfjit <-X>` command line option + and by the :envvar:`PYTHONPERFJITSUPPORT` environment variable for perf + support with DWARF JIT information. Default: ``-1``. diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index bb1c00e0aa51d5f..0ce66f31274a5f8 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -205,3 +205,36 @@ You can check if your system has been compiled with this flag by running:: If you don't see any output it means that your interpreter has not been compiled with frame pointers and therefore it may not be able to show Python functions in the output of ``perf``. + + +How to work without frame pointers +---------------------------------- + +If you are working with a Python interpreter that has been compiled without frame pointers +you can still use the ``perf`` profiler but the overhead will be a bit higher because Python +needs to generate unwinding information for every Python function call on the fly. Additionally, +``perf`` will take more time to process the data because it will need to use the DWARF debugging +information to unwind the stack and this is a slow process. + +To enable this mode, you can use the environment variable :envvar:`PYTHONPERFJITSUPPORT` or the +:option:`-X perfjit <-X>` option, which will enable the JIT mode for the ``perf`` profiler. + +When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to +call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file. + + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperfjit my_script.py + $ perf inject -i perf.data --jit + $ perf report -g -i perf.data + +or using the environment variable:: + + $ PYTHONPERFJITSUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ perf inject -i perf.data --jit + $ perf report -g -i perf.data + +Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take snapshots of the stack of +the process being profiled and save the information in the ``perf.data`` file. By default the size of +the stack dump is 8192 bytes but the user can change the size by passing the size after comma like +``--call-graph dwarf,4096``. The size of the stack dump is important because if the size is too small +``perf`` will not be able to unwind the stack and the output will be incomplete. + diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 1f5d02d1ea54a23..051dbf94a953666 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -586,6 +586,15 @@ Miscellaneous options .. versionadded:: 3.12 + * ``-X perfjit`` enables support for the Linux ``perf`` profiler with DWARF + support. When this option is provided, the ``perf`` profiler will be able + to report Python calls using DWARF ifnormation. This option is only available on + some platforms and will do nothing if is not supported on the current + system. The default value is "off". See also :envvar:`PYTHONPERFJITSUPPORT` + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + * :samp:`-X cpu_count={n}` overrides :func:`os.cpu_count`, :func:`os.process_cpu_count`, and :func:`multiprocessing.cpu_count`. *n* must be greater than or equal to 1. @@ -1127,6 +1136,21 @@ conflict. .. versionadded:: 3.12 +.. envvar:: PYTHONPERFJITSUPPORT + + If this variable is set to a nonzero value, it enables support for + the Linux ``perf`` profiler so Python calls can be detected by it + using DWARF information. + + If set to ``0``, disable Linux ``perf`` profiler support. + + See also the :option:`-X perfjit <-X>` command-line option + and :ref:`perf_profiling`. + + .. versionadded:: 3.13 + + + .. envvar:: PYTHON_CPU_COUNT If this variable is set to a positive integer, it overrides the return diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 269a7cc985ad198..a5f5ba41ebf66eb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -231,6 +231,11 @@ Other Language Changes equivalent of the :option:`-X frozen_modules <-X>` command-line option. (Contributed by Yilei Yang in :gh:`111374`.) +* Add :ref:`support for the perf profiler ` working without + frame pointers through the new environment variable + :envvar:`PYTHONPERFJITSUPPORT` and command-line option :option:`-X perfjit + <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) + * The new :envvar:`PYTHON_HISTORY` environment variable can be used to change the location of a ``.python_history`` file. (Contributed by Levi Sabah, Zackery Spytz and Hugo van Kemenade in diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index cfb88c3f4c8e156..904fbd1a389f41e 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -108,6 +108,7 @@ extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE extern _PyPerf_Callbacks _Py_perfmap_callbacks; +extern _PyPerf_Callbacks _Py_perfmap_jit_callbacks; #endif static inline PyObject* diff --git a/Include/internal/pycore_ceval_state.h b/Include/internal/pycore_ceval_state.h index 376d96ad5d334cc..009a1ea41eb9857 100644 --- a/Include/internal/pycore_ceval_state.h +++ b/Include/internal/pycore_ceval_state.h @@ -75,6 +75,7 @@ struct trampoline_api_st { unsigned int code_size, PyCodeObject* code); int (*free_state)(void* state); void *state; + Py_ssize_t code_padding; }; #endif @@ -83,6 +84,7 @@ struct _ceval_runtime_state { struct { #ifdef PY_HAVE_PERF_TRAMPOLINE perf_status_t status; + int perf_trampoline_type; Py_ssize_t extra_code_index; struct code_arena_st *code_arena; struct trampoline_api_st trampoline_api; diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index e7c03b990860136..9f72a46dc91c35a 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -5,6 +5,7 @@ import sysconfig import os import pathlib +import shutil from test import support from test.support.script_helper import ( make_script, @@ -76,14 +77,27 @@ def baz(): perf_file = pathlib.Path(f"/tmp/perf-{process.pid}.map") self.assertTrue(perf_file.exists()) perf_file_contents = perf_file.read_text() - perf_lines = perf_file_contents.splitlines(); - expected_symbols = [f"py::foo:{script}", f"py::bar:{script}", f"py::baz:{script}"] + perf_lines = perf_file_contents.splitlines() + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] for expected_symbol in expected_symbols: - perf_line = next((line for line in perf_lines if expected_symbol in line), None) - self.assertIsNotNone(perf_line, f"Could not find {expected_symbol} in perf file") + perf_line = next( + (line for line in perf_lines if expected_symbol in line), None + ) + self.assertIsNotNone( + perf_line, f"Could not find {expected_symbol} in perf file" + ) perf_addr = perf_line.split(" ")[0] - self.assertFalse(perf_addr.startswith("0x"), "Address should not be prefixed with 0x") - self.assertTrue(set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters") + self.assertFalse( + perf_addr.startswith("0x"), "Address should not be prefixed with 0x" + ) + self.assertTrue( + set(perf_addr).issubset(string.hexdigits), + "Address should contain only hex characters", + ) def test_trampoline_works_with_forks(self): code = """if 1: @@ -212,7 +226,7 @@ def test_sys_api_get_status(self): assert_python_ok("-c", code) -def is_unwinding_reliable(): +def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") if not cflags: return False @@ -259,14 +273,27 @@ def perf_command_works(): return True -def run_perf(cwd, *args, **env_vars): +def run_perf(cwd, *args, use_jit=False, **env_vars): if env_vars: env = os.environ.copy() env.update(env_vars) else: env = None output_file = cwd + "/perf_output.perf" - base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + if not use_jit: + base_cmd = ("perf", "record", "-g", "--call-graph=fp", "-o", output_file, "--") + else: + base_cmd = ( + "perf", + "record", + "-g", + "--call-graph=dwarf,65528", + "-F99", + "-k1", + "-o", + output_file, + "--", + ) proc = subprocess.run( base_cmd + args, stdout=subprocess.PIPE, @@ -274,9 +301,21 @@ def run_perf(cwd, *args, **env_vars): env=env, ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") + if use_jit: + jit_output_file = cwd + "/jit_output.dump" + command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) + proc = subprocess.run( + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + ) + if proc.returncode: + print(proc.stderr) + raise ValueError(f"Perf failed with return code {proc.returncode}") + # Copy the jit_output_file to the output_file + os.rename(jit_output_file, output_file) + base_cmd = ("perf", "script") proc = subprocess.run( ("perf", "script", "-i", output_file), @@ -290,20 +329,9 @@ def run_perf(cwd, *args, **env_vars): ) -@unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(is_unwinding_reliable(), "Unwinding is unreliable") -class TestPerfProfiler(unittest.TestCase): - def setUp(self): - super().setUp() - self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) - - def tearDown(self) -> None: - super().tearDown() - files_to_delete = ( - set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files - ) - for file in files_to_delete: - file.unlink() +class TestPerfProfilerMixin: + def run_perf(self, script_dir, perf_mode, script): + raise NotImplementedError() def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: @@ -322,14 +350,14 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, "-Xperf", script) + stdout, stderr = self.run_perf(script_dir, script) self.assertEqual(stderr, "") self.assertIn(f"py::foo:{script}", stdout) self.assertIn(f"py::bar:{script}", stdout) self.assertIn(f"py::baz:{script}", stdout) - def test_python_calls_do_not_appear_in_the_stack_if_perf_activated(self): + def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: def foo(n): @@ -346,13 +374,38 @@ def baz(n): baz(10000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = run_perf(script_dir, sys.executable, script) + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=False + ) self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) self.assertNotIn(f"py::baz:{script}", stdout) +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless( + is_unwinding_reliable_with_frame_pointers(), + "Unwinding is unreliable with frame pointers", +) +class TestPerfProfiler(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf(script_dir, sys.executable, "-Xperf", script) + return run_perf(script_dir, sys.executable, script) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("perf-*.map")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = ( + set(pathlib.Path("/tmp/").glob("perf-*.map")) - self.perf_files + ) + for file in files_to_delete: + file.unlink() + def test_pre_fork_compile(self): code = """if 1: import sys @@ -370,7 +423,7 @@ def bar_fork(): foo_fork() def foo(): - pass + import time; time.sleep(1) def bar(): foo() @@ -423,12 +476,41 @@ def compile_trampolines_for_all_functions(): # identical in both the parent and child perf-map files. perf_file_lines = perf_file_contents.split("\n") for line in perf_file_lines: - if ( - f"py::foo_fork:{script}" in line - or f"py::bar_fork:{script}" in line - ): + if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) +def _is_kernel_version_at_least(major, minor): + try: + with open("/proc/version") as f: + version = f.readline().split()[2] + except FileNotFoundError: + return False + version = version.split(".") + return int(version[0]) > major or (int(version[0]) == major and int(version[1]) >= minor) + +@unittest.skipUnless(perf_command_works(), "perf command doesn't work") +@unittest.skipUnless(_is_kernel_version_at_least(6, 6), "perf command may not work due to a perf bug") +class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): + def run_perf(self, script_dir, script, activate_trampoline=True): + if activate_trampoline: + return run_perf( + script_dir, sys.executable, "-Xperfjit", script, use_jit=True + ) + return run_perf(script_dir, sys.executable, script, use_jit=True) + + def setUp(self): + super().setUp() + self.perf_files = set(pathlib.Path("/tmp/").glob("jit*.dump")) + self.perf_files |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + + def tearDown(self) -> None: + super().tearDown() + files_to_delete = set(pathlib.Path("/tmp/").glob("jit*.dump")) + files_to_delete |= set(pathlib.Path("/tmp/").glob("jitted-*.so")) + files_to_delete = files_to_delete - self.perf_files + for file in files_to_delete: + file.unlink() + if __name__ == "__main__": unittest.main() diff --git a/Makefile.pre.in b/Makefile.pre.in index bd17debf309f0ee..924ed1f5621255d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -488,6 +488,7 @@ PYTHON_OBJS= \ Python/fileutils.o \ Python/suggestions.o \ Python/perf_trampoline.o \ + Python/perf_jit_trampoline.o \ Python/$(DYNLOADFILE) \ $(LIBOBJS) \ $(MACHDEP_OBJS) \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst new file mode 100644 index 000000000000000..7d4c003019bdef0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst @@ -0,0 +1,4 @@ +Allow the Linux perf support to work without frame pointers using perf's +advanced JIT support. The feature is activated when using the +``PYTHONPERFJITSUPPORT`` environment variable or when running Python with +``-Xperfjit``. Patch by Pablo Galindo diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 9717d89b54d828b..e5e18de60ec3490 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -240,6 +240,7 @@ + diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 9b106bea601e347..9630f54ae4ea29b 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -94,6 +94,9 @@ Source Files + + Source Files + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index a24667dc74064aa..b17e782a21421ed 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -609,6 +609,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 0b858cf1eb46a8a..cf9bc0f4bc1c707 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -1403,6 +1403,9 @@ Python + + Python + Python diff --git a/Python/initconfig.c b/Python/initconfig.c index d91a8199b544dc6..1880a28fea81f69 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1703,6 +1703,20 @@ config_init_perf_profiling(PyConfig *config) if (xoption) { config->perf_profiling = 1; } + env = config_get_env(config, "PYTHONPERFJITSUPPORT"); + if (env) { + if (_Py_str_to_int(env, &active) != 0) { + active = 0; + } + if (active) { + config->perf_profiling = 2; + } + } + xoption = config_get_xoption(config, L"perfjit"); + if (xoption) { + config->perf_profiling = 2; + } + return _PyStatus_OK(); } diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c new file mode 100644 index 000000000000000..fdce0da3eded720 --- /dev/null +++ b/Python/perf_jit_trampoline.c @@ -0,0 +1,615 @@ +#include "Python.h" +#include "pycore_ceval.h" // _PyPerf_Callbacks +#include "pycore_frame.h" +#include "pycore_interp.h" + + +#ifdef PY_HAVE_PERF_TRAMPOLINE + +#include +#include +#include +#include // mmap() +#include +#include // sysconf() +#include // gettimeofday() + +// ---------------------------------- +// Perf jitdump API +// ---------------------------------- + +typedef struct { + FILE* perf_map; + PyThread_type_lock map_lock; + void* mapped_buffer; + size_t mapped_size; + int code_id; +} PerfMapJitState; + +static PerfMapJitState perf_jit_map_state; + +/* +Usually the binary and libraries are mapped in separate region like below: + + address -> + --+---------------------+--//--+---------------------+-- + | .text | .data | ... | | .text | .data | ... | + --+---------------------+--//--+---------------------+-- + myprog libc.so + +So it'd be easy and straight-forward to find a mapped binary or library from an +address. + +But for JIT code, the code arena only cares about the code section. But the +resulting DSOs (which is generated by perf inject -j) contain ELF headers and +unwind info too. Then it'd generate following address space with synthesized +MMAP events. Let's say it has a sample between address B and C. + + sample + | + address -> A B v C + --------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + --------------------------------------------------------------------------------------------------- + +If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see +the unwind info. If it maps both .text section and unwind sections, the sample +could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing +which one is right. So to make perf happy we have non-overlapping ranges for each +DSO: + + address -> + ------------------------------------------------------------------------------------------------------- + /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + ... + ------------------------------------------------------------------------------------------------------- + +As the trampolines are constant, we add a constant padding but in general the padding needs to have the +size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + +#define PERF_JIT_CODE_PADDING 0x100 +#define trampoline_api _PyRuntime.ceval.perf.trampoline_api + +typedef uint64_t uword; +typedef const char* CodeComments; + +#define Pd "d" +#define MB (1024 * 1024) + +#define EM_386 3 +#define EM_X86_64 62 +#define EM_ARM 40 +#define EM_AARCH64 183 +#define EM_RISCV 243 + +#define TARGET_ARCH_IA32 0 +#define TARGET_ARCH_X64 0 +#define TARGET_ARCH_ARM 0 +#define TARGET_ARCH_ARM64 0 +#define TARGET_ARCH_RISCV32 0 +#define TARGET_ARCH_RISCV64 0 + +#define FLAG_generate_perf_jitdump 0 +#define FLAG_write_protect_code 0 +#define FLAG_write_protect_vm_isolate 0 +#define FLAG_code_comments 0 + +#define UNREACHABLE() + +static uword GetElfMachineArchitecture(void) { +#if TARGET_ARCH_IA32 + return EM_386; +#elif TARGET_ARCH_X64 + return EM_X86_64; +#elif TARGET_ARCH_ARM + return EM_ARM; +#elif TARGET_ARCH_ARM64 + return EM_AARCH64; +#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 + return EM_RISCV; +#else + UNREACHABLE(); + return 0; +#endif +} + +typedef struct { + uint32_t magic; + uint32_t version; + uint32_t size; + uint32_t elf_mach_target; + uint32_t reserved; + uint32_t process_id; + uint64_t time_stamp; + uint64_t flags; +} Header; + + enum PerfEvent { + PerfLoad = 0, + PerfMove = 1, + PerfDebugInfo = 2, + PerfClose = 3, + PerfUnwindingInfo = 4 +}; + +struct BaseEvent { + uint32_t event; + uint32_t size; + uint64_t time_stamp; + }; + +typedef struct { + struct BaseEvent base; + uint32_t process_id; + uint32_t thread_id; + uint64_t vma; + uint64_t code_address; + uint64_t code_size; + uint64_t code_id; +} CodeLoadEvent; + +typedef struct { + struct BaseEvent base; + uint64_t unwind_data_size; + uint64_t eh_frame_hdr_size; + uint64_t mapped_size; +} CodeUnwindingInfoEvent; + +static const intptr_t nanoseconds_per_second = 1000000000; + +// Dwarf encoding constants + +static const uint8_t DwarfUData4 = 0x03; +static const uint8_t DwarfSData4 = 0x0b; +static const uint8_t DwarfPcRel = 0x10; +static const uint8_t DwarfDataRel = 0x30; +// static uint8_t DwarfOmit = 0xff; +typedef struct { + unsigned char version; + unsigned char eh_frame_ptr_enc; + unsigned char fde_count_enc; + unsigned char table_enc; + int32_t eh_frame_ptr; + int32_t eh_fde_count; + int32_t from; + int32_t to; +} EhFrameHeader; + +static int64_t get_current_monotonic_ticks(void) { + struct timespec ts; + if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { + UNREACHABLE(); + return 0; + } + // Convert to nanoseconds. + int64_t result = ts.tv_sec; + result *= nanoseconds_per_second; + result += ts.tv_nsec; + return result; +} + +static int64_t get_current_time_microseconds(void) { + // gettimeofday has microsecond resolution. + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + UNREACHABLE(); + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; +} + + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + + +static void perf_map_jit_write_fully(const void* buffer, size_t size) { + FILE* out_file = perf_jit_map_state.perf_map; + const char* ptr = (const char*)(buffer); + while (size > 0) { + const size_t written = fwrite(ptr, 1, size, out_file); + if (written == 0) { + UNREACHABLE(); + break; + } + size -= written; + ptr += written; + } +} + +static void perf_map_jit_write_header(int pid, FILE* out_file) { + Header header; + header.magic = 0x4A695444; + header.version = 1; + header.size = sizeof(Header); + header.elf_mach_target = GetElfMachineArchitecture(); + header.process_id = pid; + header.time_stamp = get_current_time_microseconds(); + header.flags = 0; + perf_map_jit_write_fully(&header, sizeof(header)); +} + +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; + } + + const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) + if (page_size == -1) { + close(fd); + return NULL; + } + + // The perf jit interface forces us to map the first page of the file + // to signal that we are using the interface. + perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); + if (perf_jit_map_state.mapped_buffer == NULL) { + close(fd); + return NULL; + } + perf_jit_map_state.mapped_size = page_size; + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; + } + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; + } + perf_jit_map_state.code_id = 0; + + // trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + return &perf_jit_map_state; +} + +/* DWARF definitions. */ + +#define DWRF_CIE_VERSION 1 + +enum { + DWRF_CFA_nop = 0x0, + DWRF_CFA_offset_extended = 0x5, + DWRF_CFA_def_cfa = 0xc, + DWRF_CFA_def_cfa_offset = 0xe, + DWRF_CFA_offset_extended_sf = 0x11, + DWRF_CFA_advance_loc = 0x40, + DWRF_CFA_offset = 0x80 +}; + +enum + { + DWRF_EH_PE_absptr = 0x00, + DWRF_EH_PE_omit = 0xff, + + /* FDE data encoding. */ + DWRF_EH_PE_uleb128 = 0x01, + DWRF_EH_PE_udata2 = 0x02, + DWRF_EH_PE_udata4 = 0x03, + DWRF_EH_PE_udata8 = 0x04, + DWRF_EH_PE_sleb128 = 0x09, + DWRF_EH_PE_sdata2 = 0x0a, + DWRF_EH_PE_sdata4 = 0x0b, + DWRF_EH_PE_sdata8 = 0x0c, + DWRF_EH_PE_signed = 0x08, + + /* FDE flags. */ + DWRF_EH_PE_pcrel = 0x10, + DWRF_EH_PE_textrel = 0x20, + DWRF_EH_PE_datarel = 0x30, + DWRF_EH_PE_funcrel = 0x40, + DWRF_EH_PE_aligned = 0x50, + + DWRF_EH_PE_indirect = 0x80 + }; + +enum { DWRF_TAG_compile_unit = 0x11 }; + +enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; + +enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; + +enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; + +enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; + +enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; + +enum { +#ifdef __x86_64__ + /* Yes, the order is strange, but correct. */ + DWRF_REG_AX, + DWRF_REG_DX, + DWRF_REG_CX, + DWRF_REG_BX, + DWRF_REG_SI, + DWRF_REG_DI, + DWRF_REG_BP, + DWRF_REG_SP, + DWRF_REG_8, + DWRF_REG_9, + DWRF_REG_10, + DWRF_REG_11, + DWRF_REG_12, + DWRF_REG_13, + DWRF_REG_14, + DWRF_REG_15, + DWRF_REG_RA, +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_REG_SP = 31, + DWRF_REG_RA = 30, +#else +# error "Unsupported target architecture" +#endif +}; + +typedef struct ELFObjectContext +{ + uint8_t* p; /* Pointer to next address in obj.space. */ + uint8_t* startp; /* Pointer to start address in obj.space. */ + uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ + uint32_t code_size; /* Size of machine code. */ +} ELFObjectContext; + +/* Append a null-terminated string. */ +static uint32_t +elfctx_append_string(ELFObjectContext* ctx, const char* str) +{ + uint8_t* p = ctx->p; + uint32_t ofs = (uint32_t)(p - ctx->startp); + do { + *p++ = (uint8_t)*str; + } while (*str++); + ctx->p = p; + return ofs; +} + +/* Append a SLEB128 value. */ +static void +elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) +{ + uint8_t* p = ctx->p; + for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { + *p++ = (uint8_t)((v & 0x7f) | 0x80); + } + *p++ = (uint8_t)(v & 0x7f); + ctx->p = p; +} + +/* Append a ULEB128 to buffer. */ +static void +elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) +{ + uint8_t* p = ctx->p; + for (; v >= 0x80; v >>= 7) { + *p++ = (char)((v & 0x7f) | 0x80); + } + *p++ = (char)v; + ctx->p = p; +} + +/* Shortcuts to generate DWARF structures. */ +#define DWRF_U8(x) (*p++ = (x)) +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ + } +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + } + +/* Initialize .eh_frame section. */ +static void +elf_init_ehframe(ELFObjectContext* ctx) +{ + uint8_t* p = ctx->p; + uint8_t* framep = p; + + /* Emit DWARF EH CIE. */ + DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ + DWRF_U8(DWRF_CIE_VERSION); + DWRF_STR("zR"); /* Augmentation. */ + DWRF_UV(1); /* Code alignment factor. */ + DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ + DWRF_U8(DWRF_REG_RA); /* Return address register. */ + DWRF_UV(1); + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ + DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); + DWRF_ALIGNNOP(sizeof(uintptr_t)); + ) + + ctx->eh_frame_p = p; + + /* Emit DWARF EH FDE. */ + DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ + DWRF_U32(-0x30); /* Machine code offset relative to .text. */ + DWRF_U32(ctx->code_size); /* Machine code length. */ + DWRF_U8(0); /* Augmentation data. */ + /* Registers saved in CFRAME. */ +#ifdef __x86_64__ + DWRF_U8(DWRF_CFA_advance_loc | 4); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_advance_loc | 6); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); + /* Extra registers saved for JIT-compiled code. */ +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_U8(DWRF_CFA_advance_loc | 1); + DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); + DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); + DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); + DWRF_U8(DWRF_CFA_advance_loc | 3); + DWRF_U8(DWRF_CFA_offset | -(64 - 29)); + DWRF_U8(DWRF_CFA_offset | -(64 - 30)); + DWRF_U8(DWRF_CFA_def_cfa_offset); + DWRF_UV(0); +#else +# error "Unsupported target architecture" +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t));) + + ctx->p = p; +} + +static void perf_map_jit_write_entry(void *state, const void *code_addr, + unsigned int code_size, PyCodeObject *co) +{ + + if (perf_jit_map_state.perf_map == NULL) { + void* ret = perf_map_jit_init(); + if(ret == NULL){ + return; + } + } + + const char *entry = ""; + if (co->co_qualname != NULL) { + entry = PyUnicode_AsUTF8(co->co_qualname); + } + const char *filename = ""; + if (co->co_filename != NULL) { + filename = PyUnicode_AsUTF8(co->co_filename); + } + + + size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; + char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); + if (perf_map_entry == NULL) { + return; + } + snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); + + const size_t name_length = strlen(perf_map_entry); + uword base = (uword)code_addr; + uword size = code_size; + + // Write the code unwinding info event. + + // Create unwinding information (eh frame) + ELFObjectContext ctx; + char buffer[1024]; + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + elf_init_ehframe(&ctx); + int eh_frame_size = ctx.p - ctx.startp; + + // Populate the unwind info event for perf + CodeUnwindingInfoEvent ev2; + ev2.base.event = PerfUnwindingInfo; + ev2.base.time_stamp = get_current_monotonic_ticks(); + ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + // Ensure we have enough space between DSOs when perf maps them + assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; + int padding_size = round_up(content_size, 8) - content_size; + ev2.base.size = content_size + padding_size; + perf_map_jit_write_fully(&ev2, sizeof(ev2)); + + + // Populate the eh Frame header + EhFrameHeader f; + f.version = 1; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; + f.fde_count_enc = DwarfUData4; + f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); + f.eh_fde_count = 1; + f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; + f.to = -(eh_frame_size - cie_size); + + perf_map_jit_write_fully(ctx.startp, eh_frame_size); + perf_map_jit_write_fully(&f, sizeof(f)); + + char padding_bytes[] = "\0\0\0\0\0\0\0\0"; + perf_map_jit_write_fully(&padding_bytes, padding_size); + + // Write the code load event. + CodeLoadEvent ev; + ev.base.event = PerfLoad; + ev.base.size = sizeof(ev) + (name_length+1) + size; + ev.base.time_stamp = get_current_monotonic_ticks(); + ev.process_id = getpid(); + ev.thread_id = gettid(); + ev.vma = base; + ev.code_address = base; + ev.code_size = size; + perf_jit_map_state.code_id += 1; + ev.code_id = perf_jit_map_state.code_id; + + perf_map_jit_write_fully(&ev, sizeof(ev)); + perf_map_jit_write_fully(perf_map_entry, name_length+1); + perf_map_jit_write_fully((void*)(base), size); + return; +} + +static int perf_map_jit_fini(void* state) { + if (perf_jit_map_state.perf_map != NULL) { + // close the file + PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); + fclose(perf_jit_map_state.perf_map); + PyThread_release_lock(perf_jit_map_state.map_lock); + + // clean up the lock and state + PyThread_free_lock(perf_jit_map_state.map_lock); + perf_jit_map_state.perf_map = NULL; + } + if (perf_jit_map_state.mapped_buffer != NULL) { + munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + } + trampoline_api.state = NULL; + return 0; +} + +_PyPerf_Callbacks _Py_perfmap_jit_callbacks = { + &perf_map_jit_init, + &perf_map_jit_write_entry, + &perf_map_jit_fini, +}; + +#endif diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 750ba18d3510ed5..f144f7d436fe68d 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -143,6 +143,8 @@ any DWARF information available for them). #include // mmap() #include #include // sysconf() +#include // gettimeofday() + #if defined(__arm__) || defined(__arm64__) || defined(__aarch64__) #define PY_HAVE_INVALIDATE_ICACHE @@ -187,12 +189,19 @@ struct code_arena_st { typedef struct code_arena_st code_arena_t; typedef struct trampoline_api_st trampoline_api_t; +enum perf_trampoline_type { + PERF_TRAMPOLINE_UNSET = 0, + PERF_TRAMPOLINE_TYPE_MAP = 1, + PERF_TRAMPOLINE_TYPE_JITDUMP = 2, +}; + #define perf_status _PyRuntime.ceval.perf.status #define extra_code_index _PyRuntime.ceval.perf.extra_code_index #define perf_code_arena _PyRuntime.ceval.perf.code_arena #define trampoline_api _PyRuntime.ceval.perf.trampoline_api #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork +#define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type static void perf_map_write_entry(void *state, const void *code_addr, @@ -220,6 +229,8 @@ static void* perf_map_init_state(void) { PyUnstable_PerfMapState_Init(); + trampoline_api.code_padding = 0; + perf_trampoline_type = PERF_TRAMPOLINE_TYPE_MAP; return NULL; } @@ -236,6 +247,30 @@ _PyPerf_Callbacks _Py_perfmap_callbacks = { &perf_map_free_state, }; + +static size_t round_up(int64_t value, int64_t multiple) { + if (multiple == 0) { + // Avoid division by zero + return value; + } + + int64_t remainder = value % multiple; + if (remainder == 0) { + // Value is already a multiple of 'multiple' + return value; + } + + // Calculate the difference to the next multiple + int64_t difference = multiple - remainder; + + // Add the difference to the value + int64_t rounded_up_value = value + difference; + + return rounded_up_value; +} + +// TRAMPOLINE MANAGEMENT API + static int new_code_arena(void) { @@ -256,6 +291,7 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; + size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -264,9 +300,9 @@ new_code_arena(void) // measurable performance improvement by rounding trampolines up to 32-bit // or 64-bit alignment. - size_t n_copies = mem_size / code_size; + size_t n_copies = mem_size / chunk_size; for (size_t i = 0; i < n_copies; i++) { - memcpy(memory + i * code_size, start, code_size * sizeof(char)); + memcpy(memory + i * chunk_size, start, code_size * sizeof(char)); } // Some systems may prevent us from creating executable code on the fly. int res = mprotect(memory, mem_size, PROT_READ | PROT_EXEC); @@ -320,16 +356,18 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - code_arena->size_left -= code_arena->code_size; - code_arena->current_addr += code_arena->code_size; + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + code_arena->size_left -= total_code_size; + code_arena->current_addr += total_code_size; return trampoline; } static inline py_trampoline compile_trampoline(void) { + size_t total_code_size = round_up(perf_code_arena->code_size + trampoline_api.code_padding, 16); if ((perf_code_arena == NULL) || - (perf_code_arena->size_left <= perf_code_arena->code_size)) { + (perf_code_arena->size_left <= total_code_size)) { if (new_code_arena() < 0) { return NULL; } @@ -480,6 +518,7 @@ _PyPerfTrampoline_Fini(void) } if (perf_status == PERF_STATUS_OK) { trampoline_api.free_state(trampoline_api.state); + perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } extra_code_index = -1; perf_status = PERF_STATUS_NO_INIT; @@ -508,6 +547,9 @@ _PyPerfTrampoline_AfterFork_Child(void) { #ifdef PY_HAVE_PERF_TRAMPOLINE if (persist_after_fork) { + if (perf_trampoline_type != PERF_TRAMPOLINE_TYPE_MAP) { + return PyStatus_Error("Failed to copy perf map file as perf trampoline type is not type map."); + } _PyPerfTrampoline_Fini(); char filename[256]; pid_t parent_pid = getppid(); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9dc6e3f31128c1b..f24b0482c2bc38b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1210,7 +1210,14 @@ init_interp_main(PyThreadState *tstate) #ifdef PY_HAVE_PERF_TRAMPOLINE if (config->perf_profiling) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_callbacks) < 0 || + _PyPerf_Callbacks *cur_cb; + if (config->perf_profiling == 1) { + cur_cb = &_Py_perfmap_callbacks; + } + else { + cur_cb = &_Py_perfmap_jit_callbacks; + } + if (_PyPerfTrampoline_SetCallbacks(cur_cb) < 0 || _PyPerfTrampoline_Init(config->perf_profiling) < 0) { return _PyStatus_ERR("can't initialize the perf trampoline"); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index bd7f821931da424..17c4a5f46c09df1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2282,6 +2282,16 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } + else if (strcmp(backend, "perfjit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; + } + } + } } else { PyErr_Format(PyExc_ValueError, "invalid backend: %s", backend); diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c4dbdcfe80bb4a4..a1c1553d8ddc104 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -365,6 +365,8 @@ Python/intrinsics.c - _PyIntrinsics_BinaryFunctions - Python/lock.c - TIME_TO_BE_FAIR_NS - Python/opcode_targets.h - opcode_targets - Python/perf_trampoline.c - _Py_perfmap_callbacks - +Python/perf_jit_trampoline.c - _Py_perfmap_jit_callbacks - +Python/perf_jit_trampoline.c - perf_jit_map_state - Python/pyhash.c - PyHash_Func - Python/pylifecycle.c - _C_LOCALE_WARNING - Python/pylifecycle.c - _PyOS_mystrnicmp_hack - From 5dd36732c850084ce262b7869ed90d73a281296a Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sat, 4 May 2024 20:06:42 -0700 Subject: [PATCH 117/220] gh-74929: Remove undesirable DECREF in PEP 667 implementation (#118583) With tests. --- Lib/test/test_frame.py | 15 +++++++++++++++ Objects/frameobject.c | 1 - 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_frame.py b/Lib/test/test_frame.py index 93d0ea839d16eb1..212255374bddd1c 100644 --- a/Lib/test/test_frame.py +++ b/Lib/test/test_frame.py @@ -364,6 +364,21 @@ def f(): c = 0 f() + def test_local_objects(self): + o = object() + k = '.'.join(['a', 'b', 'c']) + f_locals = sys._getframe().f_locals + f_locals['o'] = f_locals['k'] + self.assertEqual(o, 'a.b.c') + + def test_update_with_self(self): + def f(): + f_locals = sys._getframe().f_locals + f_locals.update(f_locals) + f_locals.update(f_locals) + f_locals.update(f_locals) + f() + def test_repr(self): x = 1 # Introduce a reference cycle diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 8030ecb68536747..1cb00e318d9163b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -171,7 +171,6 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) } else if (value != oldvalue) { Py_XSETREF(fast[i], Py_NewRef(value)); } - Py_XDECREF(value); return 0; } } From 711c80bfca5dd17cb7c6ec26f0e44848b33aec04 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 5 May 2024 08:20:06 +0300 Subject: [PATCH 118/220] gh-118164: Break a loop between _pydecimal and _pylong and optimize int to str conversion (GH-118483) For converting large ints to strings, CPython invokes a function in _pylong.py, which uses the decimal module to implement an asymptotically waaaaay sub-quadratic algorithm. But if the C decimal module isn't available, CPython uses _pydecimal.py instead. Which in turn frequently does str(int). If the int is very large, _pylong ends up doing the work, which in turn asks decimal to do "big" arithmetic, which in turn calls str(big_int), which in turn ... it can become infinite mutual recursion. This change introduces a different int->str function that doesn't use decimal. It's asymptotically worse, "Karatsuba time" instead of quadratic time, so still a huge improvement. _pylong switches to that when the C decimal isn't available. It is also used for not too large integers (less than 450_000 bits), where it is faster (up to 2 times for 30_000 bits) than the asymptotically better implementation that uses the C decimal. Co-authored-by: Tim Peters --- Lib/_pylong.py | 46 ++++++++++++++++++- Lib/test/test_int.py | 31 +++++++++---- ...-05-02-15-57-07.gh-issue-118164.AF6kwI.rst | 4 ++ 3 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-02-15-57-07.gh-issue-118164.AF6kwI.rst diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 936346e187ff699..30bee6fc9ef54ca 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -14,6 +14,10 @@ import re import decimal +try: + import _decimal +except ImportError: + _decimal = None def int_to_decimal(n): @@ -82,7 +86,47 @@ def inner(n, w): def int_to_decimal_string(n): """Asymptotically fast conversion of an 'int' to a decimal string.""" - return str(int_to_decimal(n)) + w = n.bit_length() + if w > 450_000 and _decimal is not None: + # It is only usable with the C decimal implementation. + # _pydecimal.py calls str() on very large integers, which in its + # turn calls int_to_decimal_string(), causing very deep recursion. + return str(int_to_decimal(n)) + + # Fallback algorithm for the case when the C decimal module isn't + # available. This algorithm is asymptotically worse than the algorithm + # using the decimal module, but better than the quadratic time + # implementation in longobject.c. + def inner(n, w): + if w <= 1000: + return str(n) + w2 = w >> 1 + d = pow10_cache.get(w2) + if d is None: + d = pow10_cache[w2] = 5**w2 << w2 # 10**i = (5*2)**i = 5**i * 2**i + hi, lo = divmod(n, d) + return inner(hi, w - w2) + inner(lo, w2).zfill(w2) + + # The estimation of the number of decimal digits. + # There is no harm in small error. If we guess too large, there may + # be leading 0's that need to be stripped. If we guess too small, we + # may need to call str() recursively for the remaining highest digits, + # which can still potentially be a large integer. This is manifested + # only if the number has way more than 10**15 digits, that exceeds + # the 52-bit physical address limit in both Intel64 and AMD64. + w = int(w * 0.3010299956639812 + 1) # log10(2) + pow10_cache = {} + if n < 0: + n = -n + sign = '-' + else: + sign = '' + s = inner(n, w) + if s[0] == '0' and n: + # If our guess of w is too large, there may be leading 0's that + # need to be stripped. + s = s.lstrip('0') + return sign + s def _str_to_int_inner(s): diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 47fc50a0e203497..c8626398b35b896 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -829,17 +829,28 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_limit) super().tearDown() - def test_pylong_int_to_decimal(self): - n = (1 << 100_000) - 1 - suffix = '9883109375' + def _test_pylong_int_to_decimal(self, n, suffix): s = str(n) - assert s[-10:] == suffix - s = str(-n) - assert s[-10:] == suffix - s = '%d' % n - assert s[-10:] == suffix - s = b'%d' % n - assert s[-10:] == suffix.encode('ascii') + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') def test_pylong_int_divmod(self): n = (1 << 100_000) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-15-57-07.gh-issue-118164.AF6kwI.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-15-57-07.gh-issue-118164.AF6kwI.rst new file mode 100644 index 000000000000000..5eb3b6f5009bc43 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-15-57-07.gh-issue-118164.AF6kwI.rst @@ -0,0 +1,4 @@ +Break a loop between the Python implementation of the :mod:`decimal` module +and the Python code for integer to string conversion. Also optimize integer +to string conversion for values in the range from 9_000 to 135_000 decimal +digits. From fed8d73fde779fca41026398376cb3038e9b2b5f Mon Sep 17 00:00:00 2001 From: wim glenn Date: Sun, 5 May 2024 01:18:04 -0500 Subject: [PATCH 119/220] gh-118455: Fix mangle_from_ default value in email.policy.Policy.__doc__ (#118456) * Fix mangle_from_ default value in email.policy.Policy.__doc__ The docstring says it defaults to True, but it actually defaults to False. Only the Compat32 subclass overrides that. --------- Co-authored-by: Nikita Sobolev --- Lib/email/_policybase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/email/_policybase.py b/Lib/email/_policybase.py index c9cbadd2a80c48e..2ec54fbabae83cf 100644 --- a/Lib/email/_policybase.py +++ b/Lib/email/_policybase.py @@ -152,7 +152,7 @@ class Policy(_PolicyBase, metaclass=abc.ABCMeta): mangle_from_ -- a flag that, when True escapes From_ lines in the body of the message by putting a `>' in front of them. This is used when the message is being - serialized by a generator. Default: True. + serialized by a generator. Default: False. message_factory -- the class to use to create new message objects. If the value is None, the default is Message. From 3b32575ed6b0905f434f9395d26293c0ae928032 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sun, 5 May 2024 08:30:03 +0200 Subject: [PATCH 120/220] gh-118131: Command-line interface for the `random` module (#118132) --- Doc/library/cmdline.rst | 1 + Doc/library/random.rst | 80 +++++++++++++++++++ Doc/whatsnew/3.13.rst | 6 ++ Lib/random.py | 72 ++++++++++++++++- Lib/test/test_random.py | 43 ++++++++++ ...-04-21-18-55-42.gh-issue-118131.eAT0is.rst | 2 + 6 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-21-18-55-42.gh-issue-118131.eAT0is.rst diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index b2379befeffcbab..5174515ffc23edd 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -36,6 +36,7 @@ The following modules have a command-line interface. * :mod:`pyclbr` * :mod:`pydoc` * :mod:`quopri` +* :ref:`random ` * :mod:`runpy` * :ref:`site ` * :ref:`sqlite3 ` diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 61798263d61195c..4584bbc40aa07bc 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -706,3 +706,83 @@ positive unnormalized float and is equal to ``math.ulp(0.0)``.) `_ a paper by Allen B. Downey describing ways to generate more fine-grained floats than normally generated by :func:`.random`. + +.. _random-cli: + +Command-line usage +------------------ + +.. versionadded:: 3.13 + +The :mod:`!random` module can be executed from the command line. + +.. code-block:: sh + + python -m random [-h] [-c CHOICE [CHOICE ...] | -i N | -f N] [input ...] + +The following options are accepted: + +.. program:: random + +.. option:: -h, --help + + Show the help message and exit. + +.. option:: -c CHOICE [CHOICE ...] + --choice CHOICE [CHOICE ...] + + Print a random choice, using :meth:`choice`. + +.. option:: -i + --integer + + Print a random integer between 1 and N inclusive, using :meth:`randint`. + +.. option:: -f + --float + + Print a random floating point number between 1 and N inclusive, + using :meth:`uniform`. + +If no options are given, the output depends on the input: + +* String or multiple: same as :option:`--choice`. +* Integer: same as :option:`--integer`. +* Float: same as :option:`--float`. + +.. _random-cli-example: + +Command-line example +-------------------- + +Here are some examples of the :mod:`!random` command-line interface: + +.. code-block:: console + + $ # Choose one at random + $ python -m random egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + Lobster Thermidor aux crevettes with a Mornay sauce + + $ # Random integer + $ python -m random 6 + 6 + + $ # Random floating-point number + $ python -m random 1.8 + 1.7080016272295635 + + $ # With explicit arguments + $ python -m random --choice egg bacon sausage spam "Lobster Thermidor aux crevettes with a Mornay sauce" + egg + + $ python -m random --integer 6 + 3 + + $ python -m random --float 1.8 + 1.5666339105010318 + + $ python -m random --integer 6 + 5 + + $ python -m random --float 6 + 3.1942323316565915 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index a5f5ba41ebf66eb..27a1aeaab018dbb 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -722,6 +722,12 @@ queue termination. (Contributed by Laurie Opperman and Yves Duprat in :gh:`104750`.) +random +------ + +* Add a :ref:`command-line interface `. + (Contributed by Hugo van Kemenade in :gh:`54321`.) + re -- * Rename :exc:`!re.error` to :exc:`re.PatternError` for improved clarity. diff --git a/Lib/random.py b/Lib/random.py index 875beb2f8cf41c7..bcc11c7cd3c2083 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -996,5 +996,75 @@ def _test(N=10_000): _os.register_at_fork(after_in_child=_inst.seed) +# ------------------------------------------------------ +# -------------- command-line interface ---------------- + + +def _parse_args(arg_list: list[str] | None): + import argparse + parser = argparse.ArgumentParser( + formatter_class=argparse.RawTextHelpFormatter) + group = parser.add_mutually_exclusive_group() + group.add_argument( + "-c", "--choice", nargs="+", + help="print a random choice") + group.add_argument( + "-i", "--integer", type=int, metavar="N", + help="print a random integer between 1 and N inclusive") + group.add_argument( + "-f", "--float", type=float, metavar="N", + help="print a random floating point number between 1 and N inclusive") + group.add_argument( + "--test", type=int, const=10_000, nargs="?", + help=argparse.SUPPRESS) + parser.add_argument("input", nargs="*", + help="""\ +if no options given, output depends on the input + string or multiple: same as --choice + integer: same as --integer + float: same as --float""") + args = parser.parse_args(arg_list) + return args, parser.format_help() + + +def main(arg_list: list[str] | None = None) -> int | str: + args, help_text = _parse_args(arg_list) + + # Explicit arguments + if args.choice: + return choice(args.choice) + + if args.integer is not None: + return randint(1, args.integer) + + if args.float is not None: + return uniform(1, args.float) + + if args.test: + _test(args.test) + return "" + + # No explicit argument, select based on input + if len(args.input) == 1: + val = args.input[0] + try: + # Is it an integer? + val = int(val) + return randint(1, val) + except ValueError: + try: + # Is it a float? + val = float(val) + return uniform(1, val) + except ValueError: + # Split in case of space-separated string: "a b c" + return choice(val.split()) + + if len(args.input) >= 2: + return choice(args.input) + + return help_text + + if __name__ == '__main__': - _test() + print(main()) diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index b1e4ef4197d1308..9a44ab1768656aa 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -4,6 +4,7 @@ import os import time import pickle +import shlex import warnings import test.support @@ -1397,5 +1398,47 @@ def test_after_fork(self): support.wait_process(pid, exitcode=0) +class CommandLineTest(unittest.TestCase): + def test_parse_args(self): + args, help_text = random._parse_args(shlex.split("--choice a b c")) + self.assertEqual(args.choice, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--integer 5")) + self.assertEqual(args.integer, 5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("--float 2.5")) + self.assertEqual(args.float, 2.5) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("a b c")) + self.assertEqual(args.input, ["a", "b", "c"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("5")) + self.assertEqual(args.input, ["5"]) + self.assertTrue(help_text.startswith("usage: ")) + + args, help_text = random._parse_args(shlex.split("2.5")) + self.assertEqual(args.input, ["2.5"]) + self.assertTrue(help_text.startswith("usage: ")) + + def test_main(self): + for command, expected in [ + ("--choice a b c", "b"), + ('"a b c"', "b"), + ("a b c", "b"), + ("--choice 'a a' 'b b' 'c c'", "b b"), + ("'a a' 'b b' 'c c'", "b b"), + ("--integer 5", 4), + ("5", 4), + ("--float 2.5", 2.266632777287572), + ("2.5", 2.266632777287572), + ]: + random.seed(0) + self.assertEqual(random.main(shlex.split(command)), expected) + + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-04-21-18-55-42.gh-issue-118131.eAT0is.rst b/Misc/NEWS.d/next/Library/2024-04-21-18-55-42.gh-issue-118131.eAT0is.rst new file mode 100644 index 000000000000000..83ed66cf82fc209 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-21-18-55-42.gh-issue-118131.eAT0is.rst @@ -0,0 +1,2 @@ +Add command-line interface for the :mod:`random` module. Patch by Hugo van +Kemenade. From fd0ea63f82bf9b8f766ea40cfa5befa653461e8a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 May 2024 01:35:06 -0500 Subject: [PATCH 121/220] Minor edit: Simplify and tighten the distribution test (gh-118585) Simplify and tighten the distribution test --- Lib/test/test_statistics.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index fe6c59c30dae283..a60791e9b6e1f5b 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2482,29 +2482,30 @@ def test_kde_random(self): # Approximate distribution test: Compare a random sample to the expected distribution data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] n = 1_000_000 h = 1.75 dx = 0.1 - def p_expected(x): - return F_hat(x + dx) - F_hat(x - dx) - def p_observed(x): - # P(x-dx <= X < x+dx) / (2*dx) - i = bisect.bisect_left(big_sample, x - dx) - j = bisect.bisect_right(big_sample, x + dx) + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) return (j - i) / len(big_sample) + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + for kernel in kernels: with self.subTest(kernel=kernel): - F_hat = statistics.kde(data, h, kernel, cumulative=True) rand = kde_random(data, h, kernel, seed=8675309**2) big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) - for x in range(-40, 190): - x /= 10 - self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.001)) + for x in xarr: + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) class TestQuantiles(unittest.TestCase): From c7c9b913c01afb8d2ff4048f82155969f7ef75b1 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 May 2024 01:42:30 -0500 Subject: [PATCH 122/220] gh-118476: Fix corner cases in islice() rough equivalent. (Gh-118559) --- Doc/library/itertools.rst | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 820906085151a32..fb3f33370e3faa2 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -504,24 +504,17 @@ loops that truncate the stream. # islice('ABCDEFG', 2, None) → C D E F G # islice('ABCDEFG', 0, None, 2) → A C E G s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass + start = 0 if s.start is None else s.start + stop = s.stop + step = 1 if s.step is None else s.step + if start < 0 or (stop is not None and stop < 0) or step <= 0: + raise ValueError + indices = count() if stop is None else range(max(stop, start)) + next_i = start + for i, element in zip(indices, iterable): + if i == next_i: + yield element + next_i += step .. function:: pairwise(iterable) From aa61f8bfcf2584dd8345f1f9a07e240100b79192 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 5 May 2024 12:15:19 +0200 Subject: [PATCH 123/220] gh-110850: Remove _PyTime_TimeUnchecked() function (#118552) Use the new public Raw functions: * _PyTime_PerfCounterUnchecked() with PyTime_PerfCounterRaw() * _PyTime_TimeUnchecked() with PyTime_TimeRaw() * _PyTime_MonotonicUnchecked() with PyTime_MonotonicRaw() Remove internal functions: * _PyTime_PerfCounterUnchecked() * _PyTime_TimeUnchecked() * _PyTime_MonotonicUnchecked() --- Include/internal/pycore_time.h | 28 ++-------------- Modules/_lsprof.c | 4 ++- Modules/_testinternalcapi/test_lock.c | 13 +++++--- Python/gc.c | 1 - Python/gc_free_threading.c | 8 +++-- Python/import.c | 9 +++-- Python/lock.c | 14 +++++--- Python/parking_lot.c | 17 +++++++--- Python/pytime.c | 47 ++++----------------------- Python/thread_pthread.h | 18 +++++++--- 10 files changed, 65 insertions(+), 94 deletions(-) diff --git a/Include/internal/pycore_time.h b/Include/internal/pycore_time.h index 138d60fdb698069..15806552e0a3848 100644 --- a/Include/internal/pycore_time.h +++ b/Include/internal/pycore_time.h @@ -249,14 +249,6 @@ typedef struct { double resolution; } _Py_clock_info_t; -// Similar to PyTime_Time() but silently ignore the error and return 0 if the -// internal clock fails. -// -// Use _PyTime_TimeWithInfo() or the public PyTime_Time() to check -// for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_TimeUnchecked(void); - // Get the current time from the system clock. // On success, set *t and *info (if not NULL), and return 0. // On error, raise an exception and return -1. @@ -264,14 +256,6 @@ extern int _PyTime_TimeWithInfo( PyTime_t *t, _Py_clock_info_t *info); -// Similar to PyTime_Monotonic() but silently ignore the error and return 0 if -// the internal clock fails. -// -// Use _PyTime_MonotonicWithInfo() or the public PyTime_Monotonic() -// to check for failure. -// Export for '_random' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_MonotonicUnchecked(void); - // Get the time of a monotonic clock, i.e. a clock that cannot go backwards. // The clock is not affected by system clock updates. The reference point of // the returned value is undefined, so that only the difference between the @@ -296,14 +280,6 @@ PyAPI_FUNC(int) _PyTime_localtime(time_t t, struct tm *tm); // Export for '_datetime' shared extension. PyAPI_FUNC(int) _PyTime_gmtime(time_t t, struct tm *tm); -// Similar to PyTime_PerfCounter() but silently ignore the error and return 0 -// if the internal clock fails. -// -// Use _PyTime_PerfCounterWithInfo() or the public PyTime_PerfCounter() to -// check for failure. -// Export for '_lsprof' shared extension. -PyAPI_FUNC(PyTime_t) _PyTime_PerfCounterUnchecked(void); - // Get the performance counter: clock with the highest available resolution to // measure a short duration. @@ -319,12 +295,12 @@ extern int _PyTime_PerfCounterWithInfo( // --- _PyDeadline ----------------------------------------------------------- // Create a deadline. -// Pseudo code: _PyTime_MonotonicUnchecked() + timeout. +// Pseudo code: return PyTime_MonotonicRaw() + timeout // Export for '_ssl' shared extension. PyAPI_FUNC(PyTime_t) _PyDeadline_Init(PyTime_t timeout); // Get remaining time from a deadline. -// Pseudo code: deadline - _PyTime_MonotonicUnchecked(). +// Pseudo code: return deadline - PyTime_MonotonicRaw() // Export for '_ssl' shared extension. PyAPI_FUNC(PyTime_t) _PyDeadline_Get(PyTime_t deadline); diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 18be01df5c13777..7a9041ac079245e 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -121,7 +121,9 @@ call_timer(ProfilerObject *pObj) return CallExternalTimer(pObj); } else { - return _PyTime_PerfCounterUnchecked(); + PyTime_t t; + (void)PyTime_PerfCounterRaw(&t); + return t; } } diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 1c5048170e9f2ea..4900459c6892793 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -2,7 +2,6 @@ #include "parts.h" #include "pycore_lock.h" -#include "pycore_time.h" // _PyTime_MonotonicUnchecked() #include "clinic/test_lock.c.h" @@ -290,7 +289,10 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, goto exit; } - PyTime_t start = _PyTime_MonotonicUnchecked(); + PyTime_t start, end; + if (PyTime_PerfCounter(&start) < 0) { + goto exit; + } for (Py_ssize_t i = 0; i < num_threads; i++) { thread_data[i].bench_data = &bench_data; @@ -307,7 +309,9 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, } Py_ssize_t total_iters = bench_data.total_iters; - PyTime_t end = _PyTime_MonotonicUnchecked(); + if (PyTime_PerfCounter(&end) < 0) { + goto exit; + } // Return the total number of acquisitions and the number of acquisitions // for each thread. @@ -319,7 +323,8 @@ _testinternalcapi_benchmark_locks_impl(PyObject *module, PyList_SET_ITEM(thread_iters, i, iter); } - double rate = total_iters * 1000000000.0 / (end - start); + assert(end != start); + double rate = total_iters * 1e9 / (end - start); res = Py_BuildValue("(dO)", rate, thread_iters); exit: diff --git a/Python/gc.c b/Python/gc.c index a48738835fface7..aa8b216124c36a8 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -12,7 +12,6 @@ #include "pycore_object_alloc.h" // _PyObject_MallocWithType() #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index ef6aaad12523116..ee006bb4aa12b78 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -11,7 +11,6 @@ #include "pycore_object_stack.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_time.h" // _PyTime_GetPerfCounter() #include "pycore_tstate.h" // _PyThreadStateImpl #include "pycore_weakref.h" // _PyWeakref_ClearRef() #include "pydtrace.h" @@ -1164,7 +1163,8 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) if (gcstate->debug & _PyGC_DEBUG_STATS) { PySys_WriteStderr("gc: collecting generation %d...\n", generation); show_stats_each_generations(gcstate); - t1 = _PyTime_PerfCounterUnchecked(); + // ignore error: don't interrupt the GC if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); } if (PyDTrace_GC_START_ENABLED()) { @@ -1184,7 +1184,9 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) n = state.uncollectable; if (gcstate->debug & _PyGC_DEBUG_STATS) { - double d = PyTime_AsSecondsDouble(_PyTime_PerfCounterUnchecked() - t1); + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + double d = PyTime_AsSecondsDouble(t2 - t1); PySys_WriteStderr( "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", n+m, n, d); diff --git a/Python/import.c b/Python/import.c index fa0e548c9bf4c41..33277073ac543db 100644 --- a/Python/import.c +++ b/Python/import.c @@ -13,7 +13,7 @@ #include "pycore_pymem.h" // _PyMem_SetDefaultAllocator() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_Audit() -#include "pycore_time.h" // _PyTime_PerfCounterUnchecked() +#include "pycore_time.h" // _PyTime_AsMicroseconds() #include "pycore_weakref.h" // _PyWeakref_GET_REF() #include "marshal.h" // PyMarshal_ReadObjectFromString() @@ -3468,7 +3468,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) #undef header import_level++; - t1 = _PyTime_PerfCounterUnchecked(); + // ignore error: don't block import if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); accumulated = 0; } @@ -3483,7 +3484,9 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name) mod != NULL); if (import_time) { - PyTime_t cum = _PyTime_PerfCounterUnchecked() - t1; + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + PyTime_t cum = t2 - t1; import_level--; fprintf(stderr, "import time: %9ld | %10ld | %*s%s\n", diff --git a/Python/lock.c b/Python/lock.c index 5ed95fcaf4188c3..239e56ad929ea3f 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -5,13 +5,13 @@ #include "pycore_lock.h" #include "pycore_parking_lot.h" #include "pycore_semaphore.h" -#include "pycore_time.h" // _PyTime_MonotonicUnchecked() +#include "pycore_time.h" // _PyTime_Add() #ifdef MS_WINDOWS # define WIN32_LEAN_AND_MEAN -# include // SwitchToThread() +# include // SwitchToThread() #elif defined(HAVE_SCHED_H) -# include // sched_yield() +# include // sched_yield() #endif // If a thread waits on a lock for longer than TIME_TO_BE_FAIR_NS (1 ms), then @@ -66,7 +66,9 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_FAILURE; } - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); PyTime_t endtime = 0; if (timeout > 0) { endtime = _PyTime_Add(now, timeout); @@ -143,7 +145,9 @@ mutex_unpark(PyMutex *m, struct mutex_entry *entry, int has_more_waiters) { uint8_t v = 0; if (entry) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); int should_be_fair = now > entry->time_to_be_fair; entry->handed_off = should_be_fair; diff --git a/Python/parking_lot.c b/Python/parking_lot.c index b368b500ccdfdb1..e2def9e249cd569 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -6,7 +6,7 @@ #include "pycore_pyerrors.h" // _Py_FatalErrorFormat #include "pycore_pystate.h" // _PyThreadState_GET #include "pycore_semaphore.h" // _PySemaphore -#include "pycore_time.h" //_PyTime_MonotonicUnchecked() +#include "pycore_time.h" // _PyTime_Add() #include @@ -120,13 +120,18 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) struct timespec ts; #if defined(CLOCK_MONOTONIC) && defined(HAVE_SEM_CLOCKWAIT) - PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); - + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = sem_clockwait(&sema->platform_sem, CLOCK_MONOTONIC, &ts); #else - PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); @@ -163,7 +168,9 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) _PyTime_AsTimespec_clamp(timeout, &ts); err = pthread_cond_timedwait_relative_np(&sema->cond, &sema->mutex, &ts); #else - PyTime_t deadline = _PyTime_Add(_PyTime_TimeUnchecked(), timeout); + PyTime_t now; + (void)PyTime_TimeRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &ts); err = pthread_cond_timedwait(&sema->cond, &sema->mutex, &ts); diff --git a/Python/pytime.c b/Python/pytime.c index 12b36bbc881f9ad..560aea33f201a07 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -1030,22 +1030,6 @@ PyTime_TimeRaw(PyTime_t *result) } -PyTime_t -_PyTime_TimeUnchecked(void) -{ - PyTime_t t; -#ifdef Py_DEBUG - int result = PyTime_TimeRaw(&t); - if (result != 0) { - Py_FatalError("unable to read the system clock"); - } -#else - (void)PyTime_TimeRaw(&t); -#endif - return t; -} - - int _PyTime_TimeWithInfo(PyTime_t *t, _Py_clock_info_t *info) { @@ -1270,22 +1254,6 @@ PyTime_MonotonicRaw(PyTime_t *result) } -PyTime_t -_PyTime_MonotonicUnchecked(void) -{ - PyTime_t t; -#ifdef Py_DEBUG - int result = PyTime_MonotonicRaw(&t); - if (result != 0) { - Py_FatalError("unable to read the monotonic clock"); - } -#else - (void)PyTime_MonotonicRaw(&t); -#endif - return t; -} - - int _PyTime_MonotonicWithInfo(PyTime_t *tp, _Py_clock_info_t *info) { @@ -1314,13 +1282,6 @@ PyTime_PerfCounterRaw(PyTime_t *result) } -PyTime_t -_PyTime_PerfCounterUnchecked(void) -{ - return _PyTime_MonotonicUnchecked(); -} - - int _PyTime_localtime(time_t t, struct tm *tm) { @@ -1391,7 +1352,9 @@ _PyTime_gmtime(time_t t, struct tm *tm) PyTime_t _PyDeadline_Init(PyTime_t timeout) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return _PyTime_Add(now, timeout); } @@ -1399,6 +1362,8 @@ _PyDeadline_Init(PyTime_t timeout) PyTime_t _PyDeadline_Get(PyTime_t deadline) { - PyTime_t now = _PyTime_MonotonicUnchecked(); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); return deadline - now; } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index 65d366e91c322a4..f588b4620da0d3d 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -158,12 +158,14 @@ _PyThread_cond_after(long long us, struct timespec *abs) PyTime_t t; #ifdef CONDATTR_MONOTONIC if (condattr_monotonic) { - t = _PyTime_MonotonicUnchecked(); + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&t); } else #endif { - t = _PyTime_TimeUnchecked(); + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&t); } t = _PyTime_Add(t, timeout); _PyTime_AsTimespec_clamp(t, abs); @@ -506,7 +508,10 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, struct timespec abs_timeout; // Local scope for deadline { - PyTime_t deadline = _PyTime_Add(_PyTime_MonotonicUnchecked(), timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_MonotonicRaw(&now); + PyTime_t deadline = _PyTime_Add(now, timeout); _PyTime_AsTimespec_clamp(deadline, &abs_timeout); } #else @@ -522,8 +527,11 @@ PyThread_acquire_lock_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds, status = fix_status(sem_clockwait(thelock, CLOCK_MONOTONIC, &abs_timeout)); #else - PyTime_t abs_time = _PyTime_Add(_PyTime_TimeUnchecked(), - timeout); + PyTime_t now; + // silently ignore error: cannot report error to the caller + (void)PyTime_TimeRaw(&now); + PyTime_t abs_time = _PyTime_Add(now, timeout); + struct timespec ts; _PyTime_AsTimespec_clamp(abs_time, &ts); status = fix_status(sem_timedwait(thelock, &ts)); From aeb36c5bb9f0e0813ac7c244c094abdcd967d622 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 5 May 2024 14:37:32 +0200 Subject: [PATCH 124/220] gh-118518: Use the raw syscall directly for gettid (#118592) --- Python/perf_jit_trampoline.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index fdce0da3eded720..23478c6d469c2b3 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -13,6 +13,7 @@ #include #include // sysconf() #include // gettimeofday() +#include // ---------------------------------- // Perf jitdump API @@ -575,7 +576,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev.base.size = sizeof(ev) + (name_length+1) + size; ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); - ev.thread_id = gettid(); + ev.thread_id = syscall(SYS_gettid); ev.vma = base; ev.code_address = base; ev.code_size = size; From b6f0ab5b1cb6d779efe4867d83a60e8d66c48dee Mon Sep 17 00:00:00 2001 From: Ryan Batchelder Date: Sun, 5 May 2024 09:09:59 -0400 Subject: [PATCH 125/220] gh-83505: Add markdown mimetype mapping (#17995) --- Lib/mimetypes.py | 2 ++ .../next/Library/2020-01-14-09-46-51.bpo-39324.qUcDrM.rst | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-01-14-09-46-51.bpo-39324.qUcDrM.rst diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index dad3813e39dbaee..021ae985b40eb0a 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -552,6 +552,8 @@ def _default_mime_types(): '.csv' : 'text/csv', '.html' : 'text/html', '.htm' : 'text/html', + '.md' : 'text/markdown', + '.markdown': 'text/markdown', '.n3' : 'text/n3', '.txt' : 'text/plain', '.bat' : 'text/plain', diff --git a/Misc/NEWS.d/next/Library/2020-01-14-09-46-51.bpo-39324.qUcDrM.rst b/Misc/NEWS.d/next/Library/2020-01-14-09-46-51.bpo-39324.qUcDrM.rst new file mode 100644 index 000000000000000..357d5a26fee743a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-01-14-09-46-51.bpo-39324.qUcDrM.rst @@ -0,0 +1 @@ +Add mime type mapping for .md <-> text/markdown From 1511bc95c4bc95bd35599dc9c88111c9aac44c0d Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 5 May 2024 16:39:50 +0300 Subject: [PATCH 126/220] gh-101137: Add `text/x-rst` to `mimetypes` (#118593) --- Lib/mimetypes.py | 1 + Lib/test/test_mimetypes.py | 1 + .../next/Library/2024-05-05-16-08-03.gh-issue-101137.71ECXu.rst | 1 + 3 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-05-05-16-08-03.gh-issue-101137.71ECXu.rst diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 021ae985b40eb0a..65a049ae9945f66 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -567,6 +567,7 @@ def _default_mime_types(): '.tsv' : 'text/tab-separated-values', '.vtt' : 'text/vtt', '.py' : 'text/x-python', + '.rst' : 'text/x-rst', '.etx' : 'text/x-setext', '.sgm' : 'text/x-sgml', '.sgml' : 'text/x-sgml', diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 30e1c56bf0bc52a..bf6eae7d0ac9c67 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -214,6 +214,7 @@ def check_extensions(): self.assertEqual(mimetypes.guess_extension('text/html'), '.html') self.assertEqual(mimetypes.guess_extension('text/plain'), '.txt') self.assertEqual(mimetypes.guess_extension('text/rtf'), '.rtf') + self.assertEqual(mimetypes.guess_extension('text/x-rst'), '.rst') self.assertEqual(mimetypes.guess_extension('video/mpeg'), '.mpeg') self.assertEqual(mimetypes.guess_extension('video/quicktime'), '.mov') diff --git a/Misc/NEWS.d/next/Library/2024-05-05-16-08-03.gh-issue-101137.71ECXu.rst b/Misc/NEWS.d/next/Library/2024-05-05-16-08-03.gh-issue-101137.71ECXu.rst new file mode 100644 index 000000000000000..3df689b2f03866c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-05-16-08-03.gh-issue-101137.71ECXu.rst @@ -0,0 +1 @@ +Mime type ``text/x-rst`` is now supported by :mod:`mimetypes`. From 5a0022a1d70e4f7f781c4e8d7b43e9f5c9e2f0b4 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 5 May 2024 07:05:01 -0700 Subject: [PATCH 127/220] GH-111744: Make breakpoint() enter the debugger immediately (GH-118579) --- Doc/library/bdb.rst | 4 + Doc/library/pdb.rst | 7 +- Doc/whatsnew/3.13.rst | 6 + Lib/bdb.py | 2 +- Lib/pdb.py | 2 + Lib/test/test_doctest/test_doctest.py | 14 +- Lib/test/test_pdb.py | 194 ++++++++++++------ ...-05-04-18-40-43.gh-issue-111744.nuCtwN.rst | 1 + 8 files changed, 162 insertions(+), 68 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-04-18-40-43.gh-issue-111744.nuCtwN.rst diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index b050560cd9038cb..504fbd0464515ba 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -289,6 +289,10 @@ The :mod:`bdb` module also defines two classes: Start debugging from *frame*. If *frame* is not specified, debugging starts from caller's frame. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. + .. method:: set_continue() Stop only at breakpoints or when finished. If there are no breakpoints, diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index a640976493caba8..7a47a7d5d6754f3 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -62,8 +62,8 @@ running without the debugger using the :pdbcmd:`continue` command. The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: - > ...(3)double() - -> return x * 2 + > ...(2)double() + -> breakpoint() (Pdb) p x 3 (Pdb) continue @@ -164,6 +164,9 @@ slightly different way: .. versionchanged:: 3.7 The keyword-only argument *header*. + .. versionchanged:: 3.13 + :func:`set_trace` will enter the debugger immediately, rather than + on the next line of code to be executed. .. function:: post_mortem(traceback=None) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 27a1aeaab018dbb..c76d0087f9e2ff7 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -715,6 +715,12 @@ pdb * :mod:`zipapp` is supported as a debugging target. (Contributed by Tian Gao in :gh:`118501`.) +* ``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately + rather than on the next line of code to be executed. This change prevents the + debugger from breaking outside of the context when ``breakpoint()`` is positioned + at the end of the context. + (Contributed by Tian Gao in :gh:`118579`.) + queue ----- diff --git a/Lib/bdb.py b/Lib/bdb.py index 675c8ae51df4c3d..7d63fce6ca63f18 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -378,7 +378,7 @@ def set_trace(self, frame=None): # We need f_trace_lines == True for the debugger to work frame.f_trace_lines = True frame = frame.f_back - self.set_step() + self.set_stepinstr() sys.settrace(self.trace_dispatch) def set_continue(self): diff --git a/Lib/pdb.py b/Lib/pdb.py index e6450861328b280..b2bd2d79384e3b0 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -431,6 +431,8 @@ def user_line(self, frame): if self.bp_commands(frame): self.interaction(frame, None) + user_opcode = user_line + def bp_commands(self, frame): """Call every command that was set for the current active breakpoint (if there is one). diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index 3a173e823dd9a3f..286c3ecfbc9239f 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -2056,8 +2056,7 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin - --Return-- - > (1)()->None + > (1)() -> import pdb; pdb.set_trace() (Pdb) print(x) 42 @@ -2087,8 +2086,7 @@ def test_pdb_set_trace(): ... runner.run(test) ... finally: ... sys.stdin = real_stdin - --Return-- - > (3)calls_set_trace()->None + > (3)calls_set_trace() -> import pdb; pdb.set_trace() (Pdb) print(y) 2 @@ -2114,6 +2112,7 @@ def test_pdb_set_trace(): >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', # return event of g ... 'list', # list source from example 2 ... 'next', # return from g() ... 'list', # list source from example 1 @@ -2124,6 +2123,9 @@ def test_pdb_set_trace(): >>> try: runner.run(test) ... finally: sys.stdin = real_stdin ... # doctest: +NORMALIZE_WHITESPACE + > (3)g() + -> import pdb; pdb.set_trace() + (Pdb) step --Return-- > (3)g()->None -> import pdb; pdb.set_trace() @@ -2188,6 +2190,7 @@ def test_pdb_set_trace_nested(): >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) >>> real_stdin = sys.stdin >>> sys.stdin = FakeInput([ + ... 'step', ... 'print(y)', # print data defined in the function ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', ... 'up', 'print(x)', @@ -2201,6 +2204,9 @@ def test_pdb_set_trace_nested(): ... finally: ... sys.stdin = real_stdin ... # doctest: +REPORT_NDIFF + > (4)calls_set_trace() + -> import pdb; pdb.set_trace() + (Pdb) step > (5)calls_set_trace() -> self.f1() (Pdb) print(y) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 001562acae74586..82f3fbc26b90d95 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -46,7 +46,6 @@ def test_pdb_displayhook(): >>> def test_function(foo, bar): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'foo', @@ -55,8 +54,8 @@ def test_pdb_displayhook(): ... 'continue', ... ]): ... test_function(1, None) - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) foo 1 (Pdb) bar @@ -98,6 +97,7 @@ def test_pdb_basic_commands(): ... print(ret) >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to line ret = test_function_2('baz') ... 'step', # entering the function call ... 'args', # display function args ... 'list', # list function source @@ -122,6 +122,9 @@ def test_pdb_basic_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) step @@ -145,7 +148,7 @@ def test_pdb_basic_commands(): [EOF] (Pdb) bt ... - (25)() + (26)() -> test_function() (3)test_function() -> ret = test_function_2('baz') @@ -275,8 +278,8 @@ def test_pdb_breakpoint_commands(): ... 'continue', ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) break 4, + @@ -380,8 +383,8 @@ def test_pdb_breakpoint_with_filename(): ... 'continue', ... ]): ... test_function() - > (5)test_function() - -> mod2.func88() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test.test_inspect.inspect_fodder2:90 Breakpoint 1 at ...inspect_fodder2.py:90 (Pdb) continue @@ -479,8 +482,7 @@ def test_pdb_pp_repr_exc(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p obj *** Exception: repr_exc @@ -520,6 +522,7 @@ def test_list_commands(): ... ret = test_function_2('baz') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # go to the test function line ... 'list', # list first function ... 'step', # step into second function ... 'list', # list second function @@ -535,6 +538,9 @@ def test_list_commands(): ... 'continue', ... ]): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> ret = test_function_2('baz') (Pdb) list @@ -615,8 +621,7 @@ def test_pdb_whatis_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)test_function()->None + > (2)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) whatis myvar @@ -645,6 +650,7 @@ def test_pdb_display_command(): ... a = 4 >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'display +', ... 'display', ... 'display a', @@ -660,6 +666,9 @@ def test_pdb_display_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> a = 1 (Pdb) display + @@ -708,6 +717,7 @@ def test_pdb_alias_command(): ... o.method() >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 's', ... 'alias pi', ... 'alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}")', ... 'alias ps pi self', @@ -726,6 +736,9 @@ def test_pdb_alias_command(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)test_function() -> o.method() (Pdb) alias pi @@ -781,8 +794,7 @@ def test_pdb_where_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) w ... @@ -792,7 +804,7 @@ def test_pdb_where_command(): -> f() (2)f() -> g(); - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) where ... @@ -802,7 +814,7 @@ def test_pdb_where_command(): -> f() (2)f() -> g(); - > (2)g()->None + > (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) u > (2)f() @@ -815,7 +827,7 @@ def test_pdb_where_command(): -> f() > (2)f() -> g(); - (2)g()->None + (2)g() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -851,8 +863,7 @@ def test_pdb_interact_command(): ... 'continue', ... ]): ... test_function() - --Return-- - > (4)test_function()->None + > (4)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) interact *pdb interact start* @@ -892,6 +903,7 @@ def test_convenience_variables(): ... util_function() >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # Step to try statement ... '$_frame.f_lineno', # Check frame convenience variable ... '$ _frame', # This should be a syntax error ... '$a = 10', # Set a convenience variable @@ -914,6 +926,9 @@ def test_convenience_variables(): ... 'continue', ... ]): ... test_function() + > (2)util_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)util_function() -> try: (Pdb) $_frame.f_lineno @@ -1387,6 +1402,7 @@ def test_post_mortem(): ... print('Not reached.') >>> with PdbTestInput([ # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE + ... 'step', # step to test_function_2() line ... 'next', # step over exception-raising call ... 'bt', # get a backtrace ... 'list', # list code of test_function() @@ -1398,6 +1414,9 @@ def test_post_mortem(): ... test_function() ... except ZeroDivisionError: ... print('Correctly reraised.') + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> test_function_2() (Pdb) next @@ -1407,7 +1426,7 @@ def test_post_mortem(): -> test_function_2() (Pdb) bt ... - (10)() + (11)() -> test_function() > (3)test_function() -> test_function_2() @@ -1444,9 +1463,13 @@ def test_pdb_skip_modules(): >>> with PdbTestInput([ ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() + > (3)skip_module() + -> import pdb; pdb.Pdb(skip=['stri*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (4)skip_module() -> string.capwords('FOO') (Pdb) step @@ -1461,7 +1484,6 @@ def test_pdb_invalid_arg(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ ... 'a = 3', @@ -1470,8 +1492,8 @@ def test_pdb_invalid_arg(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) a = 3 *** Invalid argument: = 3 Usage: a(rgs) @@ -1505,10 +1527,14 @@ def test_pdb_skip_modules_with_callback(): ... 'step', ... 'step', ... 'step', + ... 'step', ... 'continue', ... ]): ... skip_module() ... pass # provides something to "step" to + > (4)skip_module() + -> import pdb; pdb.Pdb(skip=['module_to_skip*'], nosigint=True, readrc=False).set_trace() + (Pdb) step > (5)skip_module() -> mod.foo_pony(callback) (Pdb) step @@ -1527,7 +1553,7 @@ def test_pdb_skip_modules_with_callback(): > (5)skip_module()->None -> mod.foo_pony(callback) (Pdb) step - > (10)() + > (11)() -> pass # provides something to "step" to (Pdb) continue """ @@ -1546,6 +1572,7 @@ def test_pdb_continue_in_bottomframe(): ... print(4) >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'step', ... 'next', ... 'break 7', ... 'continue', @@ -1554,6 +1581,9 @@ def test_pdb_continue_in_bottomframe(): ... 'continue', ... ]): ... test_function() + > (3)test_function() + -> inst.set_trace() + (Pdb) step > (4)test_function() -> inst.botframe = sys._getframe() # hackery to get the right botframe (Pdb) next @@ -1645,8 +1675,8 @@ def test_next_until_return_at_return_event(): ... 'return', ... 'continue']): ... test_function() - > (3)test_function() - -> test_function_2() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_function_2 Breakpoint 1 at :2 (Pdb) continue @@ -1705,12 +1735,16 @@ def test_pdb_next_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1765,12 +1799,16 @@ def test_pdb_next_command_for_coroutine(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'step', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1825,12 +1863,16 @@ def test_pdb_next_command_for_asyncgen(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1883,11 +1925,15 @@ def test_pdb_return_command_for_generator(): >>> with PdbTestInput(['step', ... 'step', ... 'step', + ... 'step', ... 'return', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> it = test_gen() (Pdb) step @@ -1938,9 +1984,13 @@ def test_pdb_return_command_for_coroutine(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -1973,11 +2023,15 @@ def test_pdb_until_command_for_generator(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 4', ... 'step', ... 'step', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -2029,9 +2083,13 @@ def test_pdb_until_command_for_coroutine(): ... print("finished") >>> with PdbTestInput(['step', + ... 'step', ... 'until 8', ... 'continue']): ... test_function() + > (2)test_main() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_main() -> await test_coro() (Pdb) step @@ -2070,8 +2128,8 @@ def test_pdb_next_command_in_generator_for_loop(): ... 'next', ... 'continue']): ... test_function() - > (3)test_function() - -> for i in test_gen(): + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break test_gen Breakpoint 1 at :2 (Pdb) continue @@ -2110,11 +2168,15 @@ def test_pdb_next_command_subiterator(): >>> with PdbTestInput(['step', ... 'step', + ... 'step', ... 'next', ... 'next', ... 'next', ... 'continue']): ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (3)test_function() -> for i in test_gen(): (Pdb) step @@ -2143,7 +2205,6 @@ def test_pdb_multiline_statement(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'def f(x):', @@ -2153,8 +2214,8 @@ def test_pdb_multiline_statement(): ... 'c' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) def f(x): ... return x * 2 ... @@ -2170,7 +2231,6 @@ def test_pdb_show_attribute_and_item(): ... n = lambda x: x ... c = {"a": 1} ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'c["a"]', @@ -2184,8 +2244,8 @@ def test_pdb_show_attribute_and_item(): ... 'c' ... ]): ... test_function() - > (5)test_function() - -> pass + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) c["a"] 1 (Pdb) c.get("a") @@ -2219,12 +2279,12 @@ def test_pdb_issue_20766(): >>> with PdbTestInput(['continue', ... 'continue']): ... test_function() - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 1: - > (6)test_function() - -> print('pdb %d: %s' % (i, sess._previous_sigint_handler)) + > (5)test_function() + -> sess.set_trace(sys._getframe()) (Pdb) continue pdb 2: """ @@ -2245,8 +2305,8 @@ def test_pdb_issue_43318(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> print(1) + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break 3 Breakpoint 1 at :3 (Pdb) clear :3 @@ -2278,12 +2338,16 @@ def test_pdb_issue_gh_91742(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 5', ... 'continue' ... ]): ... test_function() + > (11)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (12)test_function() -> about() (Pdb) step @@ -2321,6 +2385,7 @@ def test_pdb_issue_gh_94215(): >>> reset_Breakpoint() >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE ... 'step', + ... 'step', ... 'next', ... 'next', ... 'jump 3', @@ -2333,6 +2398,9 @@ def test_pdb_issue_gh_94215(): ... 'continue' ... ]): ... test_function() + > (8)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) step > (9)test_function() -> func() (Pdb) step @@ -2385,8 +2453,7 @@ def test_pdb_issue_gh_101673(): ... 'continue' ... ]): ... test_function() - --Return-- - > (3)test_function()->None + > (3)test_function() -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) !a = 2 (Pdb) ll @@ -2410,16 +2477,16 @@ def test_pdb_issue_gh_103225(): ... a = 1 ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... b = 2 - > (7)() - -> b = 2 + > (6)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) longlist 1 with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE 2 'longlist', 3 'continue' 4 ]): 5 a = 1 - 6 import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - 7 -> b = 2 + 6 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + 7 b = 2 (Pdb) continue """ @@ -2438,9 +2505,8 @@ def test_pdb_issue_gh_101517(): ... 'continue' ... ]): ... test_function() - --Return-- - > (None)test_function()->None - -> Warning: lineno is None + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2456,8 +2522,8 @@ def test_pdb_issue_gh_108976(): ... 'continue' ... ]): ... test_function() - > (5)test_function() - -> a = 1 + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2476,9 +2542,8 @@ def test_pdb_issue_gh_80731(): ... raise ValueError('Correct') ... except ValueError: ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass - > (10)() - -> pass + > (9)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) import sys (Pdb) sys.exc_info() (, ValueError('Correct'), ) @@ -2492,6 +2557,7 @@ def test_pdb_ambiguous_statements(): Make sure that ambiguous statements prefixed by '!' are properly disambiguated >>> with PdbTestInput([ + ... 's', # step to the print line ... '! n = 42', # disambiguated statement: reassign the name n ... 'n', # advance the debugger into the print() ... 'continue' @@ -2500,6 +2566,9 @@ def test_pdb_ambiguous_statements(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() ... print(f"The value of n is {n}") > (8)() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s + > (9)() -> print(f"The value of n is {n}") (Pdb) ! n = 42 (Pdb) n @@ -2528,8 +2597,8 @@ def test_pdb_f_trace_lines(): ... 'continue' ... ]): ... test_function() - > (6)test_function() - -> if frame.f_trace_lines != False: + > (5)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) continue """ @@ -2552,7 +2621,6 @@ def test_pdb_function_break(): >>> def test_function(): ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() - ... pass >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 'break foo', @@ -2562,8 +2630,8 @@ def test_pdb_function_break(): ... 'continue' ... ]): ... test_function() - > (3)test_function() - -> pass + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) break foo Breakpoint ... at :1 (Pdb) break bar @@ -2593,6 +2661,7 @@ def test_pdb_issue_gh_65052(): ... A() >>> with PdbTestInput([ # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE ... 's', + ... 's', ... 'retval', ... 'continue', ... 'args', @@ -2601,6 +2670,9 @@ def test_pdb_issue_gh_65052(): ... 'continue', ... ]): ... test_function() + > (3)__new__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) s > (4)__new__() -> return object.__new__(cls) (Pdb) s @@ -2610,8 +2682,8 @@ def test_pdb_issue_gh_65052(): (Pdb) retval *** repr(retval) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) continue - > (7)__init__() - -> self.a = 1 + > (6)__init__() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) args self = *** repr(self) failed: AttributeError: 'A' object has no attribute 'a' *** (Pdb) display self diff --git a/Misc/NEWS.d/next/Library/2024-05-04-18-40-43.gh-issue-111744.nuCtwN.rst b/Misc/NEWS.d/next/Library/2024-05-04-18-40-43.gh-issue-111744.nuCtwN.rst new file mode 100644 index 000000000000000..6986aaaaf284e23 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-04-18-40-43.gh-issue-111744.nuCtwN.rst @@ -0,0 +1 @@ +``breakpoint()`` and ``pdb.set_trace()`` now enter the debugger immediately after the call rather than before the next line is executed. From 9c13d9e37a194f574b8591da634bf98419786448 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 May 2024 08:31:26 -0700 Subject: [PATCH 128/220] gh-74929: Rudimentary docs for PEP 667 (#118581) This is *not* sufficient for the final 3.13 release, but it will do for beta 1: - What's new entry - Updated changelog entry (news blurb) - Mention the proxy for f_globals in the datamodel and Python frame object docs This doesn't have any C API details (what's new refers to the PEP). --- Doc/c-api/frame.rst | 9 ++++++++- Doc/reference/datamodel.rst | 7 ++++++- Doc/whatsnew/3.13.rst | 5 +++++ .../2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst | 4 +++- 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 6bb1e9b5803b580..82e0980ad753d00 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -120,12 +120,19 @@ See also :ref:`Reflection `. .. c:function:: PyObject* PyFrame_GetLocals(PyFrameObject *frame) - Get the *frame*'s :attr:`~frame.f_locals` attribute (:class:`dict`). + Get the *frame*'s :attr:`~frame.f_locals` attribute. + If the frame refers to a function or comprehension, this returns + a write-through proxy object that allows modifying the locals. + In all other cases (classes, modules) it returns the :class:`dict` + representing the frame locals directly. Return a :term:`strong reference`. .. versionadded:: 3.11 + .. versionchanged:: 3.13 + Return a proxy object for functions and comprehensions. + .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 5e1558362ffaa03..f9438a141657be1 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -1341,7 +1341,12 @@ Special read-only attributes * - .. attribute:: frame.f_locals - The dictionary used by the frame to look up - :ref:`local variables ` + :ref:`local variables `. + If the frame refers to a function or comprehension, + this may return a write-through proxy object. + + .. versionchanged:: 3.13 + Return a proxy for functions and comprehensions. * - .. attribute:: frame.f_globals - The dictionary used by the frame to look up diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c76d0087f9e2ff7..152c8707dfe259f 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -87,6 +87,11 @@ Interpreter improvements: Performance improvements are modest -- we expect to be improving this over the next few releases. +* :pep:`667`: :attr:`FrameType.f_locals ` when used in + a function now returns a write-through proxy to the frame's locals, + rather than a ``dict``. See the PEP for corresponding C API changes + and deprecations. + New typing features: * :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst index 46e628f7fa7dbe3..29c7975fa66bc04 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-21-44-40.gh-issue-74929.C2nESp.rst @@ -1 +1,3 @@ -Implement PEP 667 - converted ``frame.f_locals`` to a write through proxy +Implement PEP 667: converted :attr:`FrameType.f_locals ` +and :c:func:`PyFrame_GetLocals` to return a write-through proxy object +when the frame refers to a function or comprehension. From 5092ea238e28c7d099c662d416b2a96fdbea4790 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 5 May 2024 12:29:23 -0500 Subject: [PATCH 129/220] Fix negative bandwidth test and add online code path test. (gh-118600) --- Lib/statistics.py | 5 ++--- Lib/test/test_statistics.py | 28 ++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/Lib/statistics.py b/Lib/statistics.py index f3ce2d8b6b442a5..c2f4fe8e054d3dd 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -1791,9 +1791,8 @@ def kde_random(data, h, kernel='normal', *, seed=None): if h <= 0.0: raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') - try: - kernel_invcdf = _kernel_invcdfs[kernel] - except KeyError: + kernel_invcdf = _kernel_invcdfs.get(kernel) + if kernel_invcdf is None: raise StatisticsError(f'Unknown kernel name: {kernel!r}') prng = _random.Random(seed) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index a60791e9b6e1f5b..40680759d456ac7 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2402,7 +2402,7 @@ def integrate(func, low, high, steps=10_000): with self.assertRaises(StatisticsError): kde(sample, h=0.0) # Zero bandwidth with self.assertRaises(StatisticsError): - kde(sample, h=0.0) # Negative bandwidth + kde(sample, h=-1.0) # Negative bandwidth with self.assertRaises(TypeError): kde(sample, h='str') # Wrong bandwidth type with self.assertRaises(StatisticsError): @@ -2426,6 +2426,14 @@ def integrate(func, low, high, steps=10_000): self.assertEqual(f_hat(-1.0), 1/2) self.assertEqual(f_hat(1.0), 1/2) + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + def test_kde_kernel_invcdfs(self): kernel_invcdfs = statistics._kernel_invcdfs kde = statistics.kde @@ -2462,7 +2470,7 @@ def test_kde_random(self): with self.assertRaises(TypeError): kde_random(iter(sample), 1.5) # Data is not a sequence with self.assertRaises(StatisticsError): - kde_random(sample, h=0.0) # Zero bandwidth + kde_random(sample, h=-1.0) # Zero bandwidth with self.assertRaises(StatisticsError): kde_random(sample, h=0.0) # Negative bandwidth with self.assertRaises(TypeError): @@ -2474,10 +2482,10 @@ def test_kde_random(self): h = 1.5 kernel = 'cosine' - prng = kde_random(sample, h, kernel) - self.assertEqual(prng.__name__, 'rand') - self.assertIn(kernel, prng.__doc__) - self.assertIn(repr(h), prng.__doc__) + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) # Approximate distribution test: Compare a random sample to the expected distribution @@ -2507,6 +2515,14 @@ def p_expected(x): for x in xarr: self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.0005)) + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + class TestQuantiles(unittest.TestCase): From 44f67916dafd3583f482e6d001766581a1a734fc Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sun, 5 May 2024 21:46:37 +0300 Subject: [PATCH 130/220] gh-117389: Fix `test_compileall.EncodingTest` (#117390) --- Lib/test/test_compileall.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 14c2af19e3eb28b..bf0fd051672db46 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -502,19 +502,25 @@ def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') with open(self.source_path, 'w', encoding='utf-8') as file: - file.write('# -*- coding: utf-8 -*-\n') - file.write('print u"\u20ac"\n') + # Intentional syntax error: bytes can only contain + # ASCII literal characters. + file.write('b"\u20ac"') def tearDown(self): shutil.rmtree(self.directory) def test_error(self): - try: - orig_stdout = sys.stdout - sys.stdout = io.TextIOWrapper(io.BytesIO(),encoding='ascii') - compileall.compile_dir(self.directory) - finally: - sys.stdout = orig_stdout + buffer = io.TextIOWrapper(io.BytesIO(), encoding='ascii') + with contextlib.redirect_stdout(buffer): + compiled = compileall.compile_dir(self.directory) + self.assertFalse(compiled) # should not be successful + buffer.seek(0) + res = buffer.read() + self.assertIn( + 'SyntaxError: bytes can only contain ASCII literal characters', + res, + ) + self.assertNotIn('UnicodeEncodeError', res) class CommandLineTestsBase: From 40cc809902304f60c6e1c933191dd4d64e570e28 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Sun, 5 May 2024 12:28:55 -0700 Subject: [PATCH 131/220] gh-117549: Don't use designated initializers in headers (#118580) The designated initializer syntax in static inline functions in pycore_backoff.h causes problems for C++ or MSVC users who aren't yet using C++20. While internal, pycore_backoff.h is included (indirectly, via pycore_code.h) by some key 3rd party software that does so for speed. --- Include/internal/pycore_backoff.h | 9 +++++++-- .../2024-05-05-12-04-02.gh-issue-117549.kITawD.rst | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-05-12-04-02.gh-issue-117549.kITawD.rst diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index decf92bc419c044..90735b202c7a93d 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -44,13 +44,18 @@ make_backoff_counter(uint16_t value, uint16_t backoff) { assert(backoff <= 15); assert(value <= 0xFFF); - return (_Py_BackoffCounter){.backoff = backoff, .value = value}; + _Py_BackoffCounter result; + result.value = value; + result.backoff = backoff; + return result; } static inline _Py_BackoffCounter forge_backoff_counter(uint16_t counter) { - return (_Py_BackoffCounter){.as_counter = counter}; + _Py_BackoffCounter result; + result.as_counter = counter; + return result; } static inline _Py_BackoffCounter diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-05-12-04-02.gh-issue-117549.kITawD.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-05-12-04-02.gh-issue-117549.kITawD.rst new file mode 100644 index 000000000000000..48ca1693a6a3659 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-05-12-04-02.gh-issue-117549.kITawD.rst @@ -0,0 +1,5 @@ +Don't use designated initializer syntax in inline functions in internal +headers. They cause problems for C++ or MSVC users who aren't yet using the +latest C++ standard (C++20). While internal, pycore_backoff.h, is included +(indirectly, via pycore_code.h) by some key 3rd party software that does so +for speed. From f27f8c790af1233d499b795af1c0d1b36aaecaf5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 5 May 2024 21:32:23 +0200 Subject: [PATCH 132/220] gh-111201: A new Python REPL (GH-111567) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa Co-authored-by: Marta Gómez Macías Co-authored-by: Lysandros Nikolaou Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Jelle Zijlstra --- .github/workflows/mypy.yml | 4 +- .github/workflows/reusable-macos.yml | 1 + .github/workflows/reusable-ubuntu.yml | 1 + Doc/glossary.rst | 14 +- Doc/tutorial/appendix.rst | 22 + Doc/using/cmdline.rst | 10 + Doc/whatsnew/3.13.rst | 28 + Lib/_pyrepl/__init__.py | 19 + Lib/_pyrepl/__main__.py | 43 + Lib/_pyrepl/_minimal_curses.py | 68 ++ Lib/_pyrepl/commands.py | 464 +++++++++ Lib/_pyrepl/completing_reader.py | 287 ++++++ Lib/_pyrepl/console.py | 112 +++ Lib/_pyrepl/curses.py | 33 + Lib/_pyrepl/fancy_termios.py | 74 ++ Lib/_pyrepl/historical_reader.py | 345 +++++++ Lib/_pyrepl/input.py | 114 +++ Lib/_pyrepl/keymap.py | 215 ++++ Lib/_pyrepl/mypy.ini | 27 + Lib/_pyrepl/pager.py | 169 ++++ Lib/_pyrepl/reader.py | 660 +++++++++++++ Lib/_pyrepl/readline.py | 501 ++++++++++ Lib/_pyrepl/simple_interact.py | 157 +++ Lib/_pyrepl/trace.py | 21 + Lib/_pyrepl/types.py | 8 + Lib/_pyrepl/unix_console.py | 743 ++++++++++++++ Lib/_pyrepl/unix_eventqueue.py | 152 +++ Lib/_pyrepl/utils.py | 18 + Lib/code.py | 13 +- Lib/pydoc.py | 158 +-- Lib/site.py | 14 +- Lib/test/test_pyrepl.py | 929 ++++++++++++++++++ Lib/test/test_traceback.py | 4 +- Lib/traceback.py | 5 +- Makefile.pre.in | 1 + ...-04-28-00-41-17.gh-issue-111201.cQsh5U.rst | 4 + Modules/main.c | 21 +- Python/clinic/sysmodule.c.h | 20 +- Python/pythonrun.c | 2 - Python/stdlib_module_names.h | 1 + Python/sysmodule.c | 16 + 41 files changed, 5328 insertions(+), 170 deletions(-) create mode 100644 Lib/_pyrepl/__init__.py create mode 100644 Lib/_pyrepl/__main__.py create mode 100644 Lib/_pyrepl/_minimal_curses.py create mode 100644 Lib/_pyrepl/commands.py create mode 100644 Lib/_pyrepl/completing_reader.py create mode 100644 Lib/_pyrepl/console.py create mode 100644 Lib/_pyrepl/curses.py create mode 100644 Lib/_pyrepl/fancy_termios.py create mode 100644 Lib/_pyrepl/historical_reader.py create mode 100644 Lib/_pyrepl/input.py create mode 100644 Lib/_pyrepl/keymap.py create mode 100644 Lib/_pyrepl/mypy.ini create mode 100644 Lib/_pyrepl/pager.py create mode 100644 Lib/_pyrepl/reader.py create mode 100644 Lib/_pyrepl/readline.py create mode 100644 Lib/_pyrepl/simple_interact.py create mode 100644 Lib/_pyrepl/trace.py create mode 100644 Lib/_pyrepl/types.py create mode 100644 Lib/_pyrepl/unix_console.py create mode 100644 Lib/_pyrepl/unix_eventqueue.py create mode 100644 Lib/_pyrepl/utils.py create mode 100644 Lib/test/test_pyrepl.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index b766785de405d2e..35996f237814baf 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -8,6 +8,7 @@ on: pull_request: paths: - ".github/workflows/mypy.yml" + - "Lib/_pyrepl/**" - "Lib/test/libregrtest/**" - "Tools/build/generate_sbom.py" - "Tools/cases_generator/**" @@ -35,8 +36,9 @@ jobs: strategy: matrix: target: [ + "Lib/_pyrepl", "Lib/test/libregrtest", - "Tools/build/", + "Tools/build", "Tools/cases_generator", "Tools/clinic", "Tools/jit", diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index dabeca8c81ece1c..e3319f141bd7f5b 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -22,6 +22,7 @@ jobs: HOMEBREW_NO_INSTALL_CLEANUP: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux strategy: fail-fast: false matrix: diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index e6fbaaf74c5a4bd..021859179af7ccc 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -17,6 +17,7 @@ jobs: FORCE_COLOR: 1 OPENSSL_VER: 3.0.13 PYTHONSTRICTEXTENSIONBUILD: 1 + TERM: linux steps: - uses: actions/checkout@v4 - name: Register gcc problem matcher diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 05ac3edb63b65d4..2846f77feb112dc 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -9,13 +9,14 @@ Glossary .. glossary:: ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. + The default Python prompt of the :term:`interactive` shell. Often + seen for code examples which can be executed interactively in the + interpreter. ``...`` Can refer to: - * The default Python prompt of the interactive shell when entering the + * The default Python prompt of the :term:`interactive` shell when entering the code for an indented code block, when within a pair of matching left and right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. @@ -620,7 +621,8 @@ Glossary execute them and see their results. Just launch ``python`` with no arguments (possibly by selecting it from your computer's main menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). + modules and packages (remember ``help(x)``). For more on interactive + mode, see :ref:`tut-interac`. interpreted Python is an interpreted language, as opposed to a compiled one, @@ -1084,6 +1086,10 @@ Glossary See also :term:`namespace package`. + REPL + An acronym for the "read–eval–print loop", another name for the + :term:`interactive` interpreter shell. + __slots__ A declaration inside a class that saves memory by pre-declaring space for instance attributes and eliminating instance dictionaries. Though diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 4bea0d8a49ce209..10eb1432175a84c 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -10,6 +10,28 @@ Appendix Interactive Mode ================ +There are two variants of the interactive :term:`REPL`. The classic +basic interpreter is supported on all platforms with minimal line +control capabilities. + +On Unix-like systems (e.g. Linux or macOS) with :mod:`curses` and +:mod:`readline` support, a new interactive shell is used by default. +This one supports color, multiline editing, history browsing, and +paste mode. To disable color, see :ref:`using-on-controlling-color` for +details. Function keys provide some additional functionality. +:kbd:`F1` enters the interactive help browser :mod:`pydoc`. +:kbd:`F2` allows for browsing command-line history without output nor the +:term:`>>>` and :term:`...` prompts. :kbd:`F3` enters "paste mode", which +makes pasting larger blocks of code easier. Press :kbd:`F3` to return to +the regular prompt. + +When using the new interactive shell, exit the shell by typing :kbd:`exit` +or :kbd:`quit`. Adding call parentheses after those commands is not +required. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + .. _tut-error: Error Handling diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 051dbf94a953666..522e6e5ccd290d9 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -42,6 +42,7 @@ additional methods of invocation: * When called with standard input connected to a tty device, it prompts for commands and executes them until an EOF (an end-of-file character, you can produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. + For more on interactive mode, see :ref:`tut-interac`. * When called with a file name argument or with a file as standard input, it reads and executes a script from that file. * When called with a directory name argument, it reads and executes an @@ -1182,6 +1183,15 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_BASIC_REPL + + If this variable is set to ``1``, the interpreter will not attempt to + load the Python-based :term:`REPL` that requires :mod:`curses` and + :mod:`readline`, and will instead use the traditional parser-based + :term:`REPL`. + + .. versionadded:: 3.13 + .. envvar:: PYTHON_HISTORY This environment variable can be used to set the location of a diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 152c8707dfe259f..11c3f931909a347 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -102,6 +102,34 @@ New typing features: New Features ============ +A Better Interactive Interpreter +-------------------------------- + +On Unix-like systems like Linux or macOS, Python now uses a new +:term:`interactive` shell. When the user starts the :term:`REPL` +from a tty, and both :mod:`curses` and :mod:`readline` are available, +the interactive shell now supports the following new features: + +* colorized prompts; +* multiline editing with history preservation; +* interactive help browsing using :kbd:`F1` with a separate command + history; +* history browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts; +* "paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt); +* ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit` without the need to use call parentheses after the + command name. + +If the new interactive shell is not desired, it can be disabled via +the :envvar:`PYTHON_BASIC_REPL` environment variable. + +For more on interactive mode, see :ref:`tut-interac`. + +(Contributed by Pablo Galindo Salgado, Łukasz Langa, and +Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project.) + Improved Error Messages ----------------------- diff --git a/Lib/_pyrepl/__init__.py b/Lib/_pyrepl/__init__.py new file mode 100644 index 000000000000000..1693cbd0b98b74c --- /dev/null +++ b/Lib/_pyrepl/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py new file mode 100644 index 000000000000000..417ee17adc83d3b --- /dev/null +++ b/Lib/_pyrepl/__main__.py @@ -0,0 +1,43 @@ +import os +import sys + +CAN_USE_PYREPL = True + +def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): + global CAN_USE_PYREPL + if not CAN_USE_PYREPL: + return sys._baserepl() + + startup_path = os.getenv("PYTHONSTARTUP") + if pythonstartup and startup_path: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") + exec(startup_code) + + # set sys.{ps1,ps2} just before invoking the interactive interpreter. This + # mimics what CPython does in pythonrun.c + if not hasattr(sys, "ps1"): + sys.ps1 = ">>> " + if not hasattr(sys, "ps2"): + sys.ps2 = "... " + # + run_interactive = None + try: + import errno + if not os.isatty(sys.stdin.fileno()): + raise OSError(errno.ENOTTY, "tty required", "stdin") + from .simple_interact import check + if err := check(): + raise RuntimeError(err) + from .simple_interact import run_multiline_interactive_console + run_interactive = run_multiline_interactive_console + except Exception as e: + print(f"warning: can't use pyrepl: {e}", file=sys.stderr) + CAN_USE_PYREPL = False + if run_interactive is None: + return sys._baserepl() + return run_interactive(mainmodule) + +if __name__ == "__main__": + interactive_console() diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py new file mode 100644 index 000000000000000..0757fb2c664addf --- /dev/null +++ b/Lib/_pyrepl/_minimal_curses.py @@ -0,0 +1,68 @@ +"""Minimal '_curses' module, the low-level interface for curses module +which is not meant to be used directly. + +Based on ctypes. It's too incomplete to be really called '_curses', so +to use it, you have to import it and stick it in sys.modules['_curses'] +manually. + +Note that there is also a built-in module _minimal_curses which will +hide this one if compiled in. +""" + +import ctypes +import ctypes.util + + +class error(Exception): + pass + + +def _find_clib(): + trylibs = ["ncursesw", "ncurses", "curses"] + + for lib in trylibs: + path = ctypes.util.find_library(lib) + if path: + return path + raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") + + +_clibpath = _find_clib() +clib = ctypes.cdll.LoadLibrary(_clibpath) + +clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] +clib.setupterm.restype = ctypes.c_int + +clib.tigetstr.argtypes = [ctypes.c_char_p] +clib.tigetstr.restype = ctypes.POINTER(ctypes.c_char) + +clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] +clib.tparm.restype = ctypes.c_char_p + +OK = 0 +ERR = -1 + +# ____________________________________________________________ + + +def setupterm(termstr, fd): + err = ctypes.c_int(0) + result = clib.setupterm(termstr, fd, ctypes.byref(err)) + if result == ERR: + raise error("setupterm() failed (err=%d)" % err.value) + + +def tigetstr(cap): + if not isinstance(cap, bytes): + cap = cap.encode("ascii") + result = clib.tigetstr(cap) + if ctypes.cast(result, ctypes.c_void_p).value == ERR: + return None + return ctypes.cast(result, ctypes.c_char_p).value + + +def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): + result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) + if result is None: + raise error("tparm() returned NULL") + return result diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py new file mode 100644 index 000000000000000..60ceb30d2cd77dc --- /dev/null +++ b/Lib/_pyrepl/commands.py @@ -0,0 +1,464 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations +import os + +# Categories of actions: +# killing +# yanking +# motion +# editing +# history +# finishing +# [completion] + + +# types +if False: + from .reader import Reader + from .historical_reader import HistoricalReader + from .console import Event + + +class Command: + finish: bool = False + kills_digit_arg: bool = True + + def __init__( + self, reader: HistoricalReader, event_name: str, event: list[str] + ) -> None: + # Reader should really be "any reader" but there's too much usage of + # HistoricalReader methods and fields in the code below for us to + # refactor at the moment. + + self.reader = reader + self.event = event + self.event_name = event_name + + def do(self) -> None: + pass + + +class KillCommand(Command): + def kill_range(self, start: int, end: int) -> None: + if start == end: + return + r = self.reader + b = r.buffer + text = b[start:end] + del b[start:end] + if is_kill(r.last_command): + if start < r.pos: + r.kill_ring[-1] = text + r.kill_ring[-1] + else: + r.kill_ring[-1] = r.kill_ring[-1] + text + else: + r.kill_ring.append(text) + r.pos = start + r.dirty = True + + +class YankCommand(Command): + pass + + +class MotionCommand(Command): + pass + + +class EditCommand(Command): + pass + + +class FinishCommand(Command): + finish = True + pass + + +def is_kill(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, KillCommand) + + +def is_yank(command: type[Command] | None) -> bool: + return command is not None and issubclass(command, YankCommand) + + +# etc + + +class digit_arg(Command): + kills_digit_arg = False + + def do(self) -> None: + r = self.reader + c = self.event[-1] + if c == "-": + if r.arg is not None: + r.arg = -r.arg + else: + r.arg = -1 + else: + d = int(c) + if r.arg is None: + r.arg = d + else: + if r.arg < 0: + r.arg = 10 * r.arg - d + else: + r.arg = 10 * r.arg + d + r.dirty = True + + +class clear_screen(Command): + def do(self) -> None: + r = self.reader + r.console.clear() + r.dirty = True + + +class refresh(Command): + def do(self) -> None: + self.reader.dirty = True + + +class repaint(Command): + def do(self) -> None: + self.reader.dirty = True + self.reader.console.repaint() + + +class kill_line(KillCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + eol = r.eol() + for c in b[r.pos : eol]: + if not c.isspace(): + self.kill_range(r.pos, eol) + return + else: + self.kill_range(r.pos, eol + 1) + + +class unix_line_discard(KillCommand): + def do(self) -> None: + r = self.reader + self.kill_range(r.bol(), r.pos) + + +class unix_word_rubout(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.pos, r.eow()) + + +class backward_kill_word(KillCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + self.kill_range(r.bow(), r.pos) + + +class yank(YankCommand): + def do(self) -> None: + r = self.reader + if not r.kill_ring: + r.error("nothing to yank") + return + r.insert(r.kill_ring[-1]) + + +class yank_pop(YankCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if not r.kill_ring: + r.error("nothing to yank") + return + if not is_yank(r.last_command): + r.error("previous command was not a yank") + return + repl = len(r.kill_ring[-1]) + r.kill_ring.insert(0, r.kill_ring.pop()) + t = r.kill_ring[-1] + b[r.pos - repl : r.pos] = t + r.pos = r.pos - repl + len(t) + r.dirty = True + + +class interrupt(FinishCommand): + def do(self) -> None: + import signal + + self.reader.console.finish() + os.kill(os.getpid(), signal.SIGINT) + + +class suspend(Command): + def do(self) -> None: + import signal + + r = self.reader + p = r.pos + r.console.finish() + os.kill(os.getpid(), signal.SIGSTOP) + ## this should probably be done + ## in a handler for SIGCONT? + r.console.prepare() + r.pos = p + # r.posxy = 0, 0 # XXX this is invalid + r.dirty = True + r.console.screen = [] + + +class up(MotionCommand): + def do(self) -> None: + r = self.reader + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y - 1 + + if new_y < 0: + if r.historyi > 0: + r.select_item(r.historyi - 1) + return + r.pos = 0 + r.error("start of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class down(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for _ in range(r.get_arg()): + x, y = r.pos2xy() + new_y = y + 1 + + if new_y > r.max_row(): + if r.historyi < len(r.history): + r.select_item(r.historyi + 1) + r.pos = r.eol(0) + return + r.pos = len(b) + r.error("end of buffer") + return + + if ( + x + > ( + new_x := r.max_column(new_y) + ) # we're past the end of the previous line + or x == r.max_column(y) + and any( + not i.isspace() for i in r.buffer[r.bol() :] + ) # move between eols + ): + x = new_x + + r.setpos_from_xy(x, new_y) + + +class left(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + p = r.pos - 1 + if p >= 0: + r.pos = p + else: + self.reader.error("start of buffer") + + +class right(MotionCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + p = r.pos + 1 + if p <= len(b): + r.pos = p + else: + self.reader.error("end of buffer") + + +class beginning_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.bol() + + +class end_of_line(MotionCommand): + def do(self) -> None: + self.reader.pos = self.reader.eol() + + +class home(MotionCommand): + def do(self) -> None: + self.reader.pos = 0 + + +class end(MotionCommand): + def do(self) -> None: + self.reader.pos = len(self.reader.buffer) + + +class forward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.eow() + + +class backward_word(MotionCommand): + def do(self) -> None: + r = self.reader + for i in range(r.get_arg()): + r.pos = r.bow() + + +class self_insert(EditCommand): + def do(self) -> None: + r = self.reader + r.insert(self.event * r.get_arg()) + + +class insert_nl(EditCommand): + def do(self) -> None: + r = self.reader + r.insert("\n" * r.get_arg()) + + +class transpose_characters(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + s = r.pos - 1 + if s < 0: + r.error("cannot transpose at start of buffer") + else: + if s == len(b): + s -= 1 + t = min(s + r.get_arg(), len(b) - 1) + c = b[s] + del b[s] + b.insert(t, c) + r.pos = t + r.dirty = True + + +class backspace(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + for i in range(r.get_arg()): + if r.pos > 0: + r.pos -= 1 + del b[r.pos] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +class delete(EditCommand): + def do(self) -> None: + r = self.reader + b = r.buffer + if ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + and self.event[-1] == "\004" + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): + if r.pos != len(b): + del b[r.pos] + r.dirty = True + else: + self.reader.error("end of buffer") + + +class accept(FinishCommand): + def do(self) -> None: + pass + + +class help(Command): + def do(self) -> None: + import _sitebuiltins + + with self.reader.suspend(): + self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment, call-arg] + + +class invalid_key(Command): + def do(self) -> None: + pending = self.reader.console.getpending() + s = "".join(self.event) + pending.data + self.reader.error("`%r' not bound" % s) + + +class invalid_command(Command): + def do(self) -> None: + s = self.event_name + self.reader.error("command `%s' not known" % s) + + +class show_history(Command): + def do(self) -> None: + from .pager import get_pager + from site import gethistoryfile # type: ignore[attr-defined] + + history = os.linesep.join(self.reader.history[:]) + with self.reader.suspend(): + pager = get_pager() + pager(history, gethistoryfile()) + + +class paste_mode(Command): + + def do(self) -> None: + self.reader.paste_mode = not self.reader.paste_mode + self.reader.dirty = True diff --git a/Lib/_pyrepl/completing_reader.py b/Lib/_pyrepl/completing_reader.py new file mode 100644 index 000000000000000..19fc06feaf3ced7 --- /dev/null +++ b/Lib/_pyrepl/completing_reader.py @@ -0,0 +1,287 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from dataclasses import dataclass, field + +import re +from . import commands, console, reader +from .reader import Reader + + +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def prefix(wordlist: list[str], j: int = 0) -> str: + d = {} + i = j + try: + while 1: + for word in wordlist: + d[word[i]] = 1 + if len(d) > 1: + return wordlist[0][j:i] + i += 1 + d = {} + except IndexError: + return wordlist[0][j:i] + return "" + + +STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]") + +def stripcolor(s: str) -> str: + return STRIPCOLOR_REGEX.sub('', s) + + +def real_len(s: str) -> int: + return len(stripcolor(s)) + + +def left_align(s: str, maxlen: int) -> str: + stripped = stripcolor(s) + if len(stripped) > maxlen: + # too bad, we remove the color + return stripped[:maxlen] + padding = maxlen - len(stripped) + return s + ' '*padding + + +def build_menu( + cons: console.Console, + wordlist: list[str], + start: int, + use_brackets: bool, + sort_in_column: bool, +) -> tuple[list[str], int]: + if use_brackets: + item = "[ %s ]" + padding = 4 + else: + item = "%s " + padding = 2 + maxlen = min(max(map(real_len, wordlist)), cons.width - padding) + cols = int(cons.width / (maxlen + padding)) + rows = int((len(wordlist) - 1)/cols + 1) + + if sort_in_column: + # sort_in_column=False (default) sort_in_column=True + # A B C A D G + # D E F B E + # G C F + # + # "fill" the table with empty words, so we always have the same amout + # of rows for each column + missing = cols*rows - len(wordlist) + wordlist = wordlist + ['']*missing + indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))] + wordlist = [wordlist[i] for i in indexes] + menu = [] + i = start + for r in range(rows): + row = [] + for col in range(cols): + row.append(item % left_align(wordlist[i], maxlen)) + i += 1 + if i >= len(wordlist): + break + menu.append(''.join(row)) + if i >= len(wordlist): + i = 0 + break + if r + 5 > cons.height: + menu.append(" %d more... " % (len(wordlist) - i)) + break + return menu, i + +# this gets somewhat user interface-y, and as a result the logic gets +# very convoluted. +# +# To summarise the summary of the summary:- people are a problem. +# -- The Hitch-Hikers Guide to the Galaxy, Episode 12 + +#### Desired behaviour of the completions commands. +# the considerations are: +# (1) how many completions are possible +# (2) whether the last command was a completion +# (3) if we can assume that the completer is going to return the same set of +# completions: this is controlled by the ``assume_immutable_completions`` +# variable on the reader, which is True by default to match the historical +# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match +# more closely readline's semantics (this is needed e.g. by +# fancycompleter) +# +# if there's no possible completion, beep at the user and point this out. +# this is easy. +# +# if there's only one possible completion, stick it in. if the last thing +# user did was a completion, point out that he isn't getting anywhere, but +# only if the ``assume_immutable_completions`` is True. +# +# now it gets complicated. +# +# for the first press of a completion key: +# if there's a common prefix, stick it in. + +# irrespective of whether anything got stuck in, if the word is now +# complete, show the "complete but not unique" message + +# if there's no common prefix and if the word is not now complete, +# beep. + +# common prefix -> yes no +# word complete \/ +# yes "cbnu" "cbnu" +# no - beep + +# for the second bang on the completion key +# there will necessarily be no common prefix +# show a menu of the choices. + +# for subsequent bangs, rotate the menu around (if there are sufficient +# choices). + + +class complete(commands.Command): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + last_is_completer = r.last_command_is(self.__class__) + immutable_completions = r.assume_immutable_completions + completions_unchangable = last_is_completer and immutable_completions + stem = r.get_stem() + if not completions_unchangable: + r.cmpltn_menu_choices = r.get_completions(stem) + + completions = r.cmpltn_menu_choices + if not completions: + r.error("no matches") + elif len(completions) == 1: + if completions_unchangable and len(completions[0]) == len(stem): + r.msg = "[ sole completion ]" + r.dirty = True + r.insert(completions[0][len(stem):]) + else: + p = prefix(completions, len(stem)) + if p: + r.insert(p) + if last_is_completer: + if not r.cmpltn_menu_vis: + r.cmpltn_menu_vis = 1 + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, r.cmpltn_menu_end, + r.use_brackets, r.sort_in_column) + r.dirty = True + elif stem + p in completions: + r.msg = "[ complete but not unique ]" + r.dirty = True + else: + r.msg = "[ not unique ]" + r.dirty = True + + +class self_insert(commands.self_insert): + def do(self) -> None: + r: CompletingReader + r = self.reader # type: ignore[assignment] + + commands.self_insert.do(self) + + if r.cmpltn_menu_vis: + stem = r.get_stem() + if len(stem) < 1: + r.cmpltn_reset() + else: + completions = [w for w in r.cmpltn_menu_choices + if w.startswith(stem)] + if completions: + r.cmpltn_menu, r.cmpltn_menu_end = build_menu( + r.console, completions, 0, + r.use_brackets, r.sort_in_column) + else: + r.cmpltn_reset() + + +@dataclass +class CompletingReader(Reader): + """Adds completion support""" + + ### Class variables + # see the comment for the complete command + assume_immutable_completions = True + use_brackets = True # display completions inside [] + sort_in_column = False + + ### Instance variables + cmpltn_menu: list[str] = field(init=False) + cmpltn_menu_vis: int = field(init=False) + cmpltn_menu_end: int = field(init=False) + cmpltn_menu_choices: list[str] = field(init=False) + + def __post_init__(self) -> None: + super().__post_init__() + self.cmpltn_reset() + for c in (complete, self_insert): + self.commands[c.__name__] = c + self.commands[c.__name__.replace('_', '-')] = c + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r'\t', 'complete'),) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if not isinstance(cmd, (complete, self_insert)): + self.cmpltn_reset() + + def calc_screen(self) -> list[str]: + screen = super().calc_screen() + if self.cmpltn_menu_vis: + ly = self.lxy[1] + screen[ly:ly] = self.cmpltn_menu + self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu) + self.cxy = self.cxy[0], self.cxy[1] + len(self.cmpltn_menu) + return screen + + def finish(self) -> None: + super().finish() + self.cmpltn_reset() + + def cmpltn_reset(self) -> None: + self.cmpltn_menu = [] + self.cmpltn_menu_vis = 0 + self.cmpltn_menu_end = 0 + self.cmpltn_menu_choices = [] + + def get_stem(self) -> str: + st = self.syntax_table + SW = reader.SYNTAX_WORD + b = self.buffer + p = self.pos - 1 + while p >= 0 and st.get(b[p], SW) == SW: + p -= 1 + return ''.join(b[p+1:self.pos]) + + def get_completions(self, stem: str) -> list[str]: + return [] diff --git a/Lib/_pyrepl/console.py b/Lib/_pyrepl/console.py new file mode 100644 index 000000000000000..d7e86e768671dc1 --- /dev/null +++ b/Lib/_pyrepl/console.py @@ -0,0 +1,112 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field + + +@dataclass +class Event: + evt: str + data: str + raw: bytes = b"" + + +@dataclass +class Console(ABC): + screen: list[str] = field(default_factory=list) + height: int = 25 + width: int = 80 + + @abstractmethod + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ... + + @abstractmethod + def prepare(self) -> None: ... + + @abstractmethod + def restore(self) -> None: ... + + @abstractmethod + def move_cursor(self, x: int, y: int) -> None: ... + + @abstractmethod + def set_cursor_vis(self, visible: bool) -> None: ... + + @abstractmethod + def getheightwidth(self) -> tuple[int, int]: + """Return (height, width) where height and width are the height + and width of the terminal window in characters.""" + ... + + @abstractmethod + def get_event(self, block: bool = True) -> Event | None: + """Return an Event instance. Returns None if |block| is false + and there is no event pending, otherwise waits for the + completion of an event.""" + ... + + @abstractmethod + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + ... + + @abstractmethod + def beep(self) -> None: ... + + @abstractmethod + def clear(self) -> None: + """Wipe the screen""" + ... + + @abstractmethod + def finish(self) -> None: + """Move the cursor to the end of the display and otherwise get + ready for end. XXX could be merged with restore? Hmm.""" + ... + + @abstractmethod + def flushoutput(self) -> None: + """Flush all output to the screen (assuming there's some + buffering going on somewhere).""" + ... + + @abstractmethod + def forgetinput(self) -> None: + """Forget all pending, but not yet processed input.""" + ... + + @abstractmethod + def getpending(self) -> Event: + """Return the characters that have been typed but not yet + processed.""" + ... + + @abstractmethod + def wait(self) -> None: + """Wait for an event.""" + ... + + @abstractmethod + def repaint(self) -> None: + ... diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py new file mode 100644 index 000000000000000..3a624d9f6835d13 --- /dev/null +++ b/Lib/_pyrepl/curses.py @@ -0,0 +1,33 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +try: + import _curses +except ImportError: + try: + import curses as _curses # type: ignore[no-redef] + except ImportError: + from . import _minimal_curses as _curses # type: ignore[no-redef] + +setupterm = _curses.setupterm +tigetstr = _curses.tigetstr +tparm = _curses.tparm +error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py new file mode 100644 index 000000000000000..5b85cb0f52521f5 --- /dev/null +++ b/Lib/_pyrepl/fancy_termios.py @@ -0,0 +1,74 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +import termios + + +class TermState: + def __init__(self, tuples): + ( + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ) = tuples + + def as_list(self): + return [ + self.iflag, + self.oflag, + self.cflag, + self.lflag, + self.ispeed, + self.ospeed, + self.cc, + ] + + def copy(self): + return self.__class__(self.as_list()) + + +def tcgetattr(fd): + return TermState(termios.tcgetattr(fd)) + + +def tcsetattr(fd, when, attrs): + termios.tcsetattr(fd, when, attrs.as_list()) + + +class Term(TermState): + TS__init__ = TermState.__init__ + + def __init__(self, fd=0): + self.TS__init__(termios.tcgetattr(fd)) + self.fd = fd + self.stack = [] + + def save(self): + self.stack.append(self.as_list()) + + def set(self, when=termios.TCSANOW): + termios.tcsetattr(self.fd, when, self.as_list()) + + def restore(self): + self.TS__init__(self.stack.pop()) + self.set() diff --git a/Lib/_pyrepl/historical_reader.py b/Lib/_pyrepl/historical_reader.py new file mode 100644 index 000000000000000..eef7d901b083ef0 --- /dev/null +++ b/Lib/_pyrepl/historical_reader.py @@ -0,0 +1,345 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field + +from . import commands, input +from .reader import Reader + + +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"] + + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"] + + [ + ("\\%03o" % c, "isearch-add-character") + for c in range(256) + if chr(c).isalpha() and chr(c) != "\\" + ] + + [ + ("\\\\", "self-insert"), + (r"\C-r", "isearch-backwards"), + (r"\C-s", "isearch-forwards"), + (r"\C-c", "isearch-cancel"), + (r"\C-g", "isearch-cancel"), + (r"\", "isearch-backspace"), + ] +) + +ISEARCH_DIRECTION_NONE = "" +ISEARCH_DIRECTION_BACKWARDS = "r" +ISEARCH_DIRECTION_FORWARDS = "f" + + +class next_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == len(r.history): + r.error("end of history list") + return + r.select_item(r.historyi + 1) + + +class previous_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi == 0: + r.error("start of history list") + return + r.select_item(r.historyi - 1) + + +class restore_history(commands.Command): + def do(self) -> None: + r = self.reader + if r.historyi != len(r.history): + if r.get_unicode() != r.history[r.historyi]: + r.buffer = list(r.history[r.historyi]) + r.pos = len(r.buffer) + r.dirty = True + + +class first_history(commands.Command): + def do(self) -> None: + self.reader.select_item(0) + + +class last_history(commands.Command): + def do(self) -> None: + self.reader.select_item(len(self.reader.history)) + + +class operate_and_get_next(commands.FinishCommand): + def do(self) -> None: + self.reader.next_history = self.reader.historyi + 1 + + +class yank_arg(commands.Command): + def do(self) -> None: + r = self.reader + if r.last_command is self.__class__: + r.yank_arg_i += 1 + else: + r.yank_arg_i = 0 + if r.historyi < r.yank_arg_i: + r.error("beginning of history list") + return + a = r.get_arg(-1) + # XXX how to split? + words = r.get_item(r.historyi - r.yank_arg_i - 1).split() + if a < -len(words) or a >= len(words): + r.error("no such arg") + return + w = words[a] + b = r.buffer + if r.yank_arg_i > 0: + o = len(r.yank_arg_yanked) + else: + o = 0 + b[r.pos - o : r.pos] = list(w) + r.yank_arg_yanked = w + r.pos += len(w) - o + r.dirty = True + + +class forward_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_start = r.historyi, r.pos + r.isearch_term = "" + r.dirty = True + r.push_input_trans(r.isearch_trans) + + +class reverse_history_isearch(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.dirty = True + r.isearch_term = "" + r.push_input_trans(r.isearch_trans) + r.isearch_start = r.historyi, r.pos + + +class isearch_cancel(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.pop_input_trans() + r.select_item(r.isearch_start[0]) + r.pos = r.isearch_start[1] + r.dirty = True + + +class isearch_add_character(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + r.isearch_term += self.event[-1] + r.dirty = True + p = r.pos + len(r.isearch_term) - 1 + if b[p : p + 1] != [r.isearch_term[-1]]: + r.isearch_next() + + +class isearch_backspace(commands.Command): + def do(self) -> None: + r = self.reader + if len(r.isearch_term) > 0: + r.isearch_term = r.isearch_term[:-1] + r.dirty = True + else: + r.error("nothing to rubout") + + +class isearch_forwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_FORWARDS + r.isearch_next() + + +class isearch_backwards(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS + r.isearch_next() + + +class isearch_end(commands.Command): + def do(self) -> None: + r = self.reader + r.isearch_direction = ISEARCH_DIRECTION_NONE + r.console.forgetinput() + r.pop_input_trans() + r.dirty = True + + +@dataclass +class HistoricalReader(Reader): + """Adds history support (with incremental history searching) to the + Reader class. + """ + + history: list[str] = field(default_factory=list) + historyi: int = 0 + next_history: int | None = None + transient_history: dict[int, str] = field(default_factory=dict) + isearch_term: str = "" + isearch_direction: str = ISEARCH_DIRECTION_NONE + isearch_start: tuple[int, int] = field(init=False) + isearch_trans: input.KeymapTranslator = field(init=False) + yank_arg_i: int = 0 + yank_arg_yanked: str = "" + + def __post_init__(self) -> None: + super().__post_init__() + for c in [ + next_history, + previous_history, + restore_history, + first_history, + last_history, + yank_arg, + forward_history_isearch, + reverse_history_isearch, + isearch_end, + isearch_add_character, + isearch_cancel, + isearch_add_character, + isearch_backspace, + isearch_forwards, + isearch_backwards, + operate_and_get_next, + ]: + self.commands[c.__name__] = c + self.commands[c.__name__.replace("_", "-")] = c + self.isearch_start = self.historyi, self.pos + self.isearch_trans = input.KeymapTranslator( + isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character + ) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\C-n", "next-history"), + (r"\C-p", "previous-history"), + (r"\C-o", "operate-and-get-next"), + (r"\C-r", "reverse-history-isearch"), + (r"\C-s", "forward-history-isearch"), + (r"\M-r", "restore-history"), + (r"\M-.", "yank-arg"), + (r"\", "last-history"), + (r"\", "first-history"), + ) + + def select_item(self, i: int) -> None: + self.transient_history[self.historyi] = self.get_unicode() + buf = self.transient_history.get(i) + if buf is None: + buf = self.history[i] + self.buffer = list(buf) + self.historyi = i + self.pos = len(self.buffer) + self.dirty = True + + def get_item(self, i: int) -> str: + if i != len(self.history): + return self.transient_history.get(i, self.history[i]) + else: + return self.transient_history.get(i, self.get_unicode()) + + @contextmanager + def suspend(self) -> SimpleContextManager: + with super().suspend(): + try: + old_history = self.history[:] + del self.history[:] + yield + finally: + self.history[:] = old_history + + def prepare(self) -> None: + super().prepare() + try: + self.transient_history = {} + if self.next_history is not None and self.next_history < len(self.history): + self.historyi = self.next_history + self.buffer[:] = list(self.history[self.next_history]) + self.pos = len(self.buffer) + self.transient_history[len(self.history)] = "" + else: + self.historyi = len(self.history) + self.next_history = None + except: + self.restore() + raise + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE: + d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS] + return "(%s-search `%s') " % (d, self.isearch_term) + else: + return super().get_prompt(lineno, cursor_on_line) + + def isearch_next(self) -> None: + st = self.isearch_term + p = self.pos + i = self.historyi + s = self.get_unicode() + forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS + while 1: + if forwards: + p = s.find(st, p + 1) + else: + p = s.rfind(st, 0, p + len(st) - 1) + if p != -1: + self.select_item(i) + self.pos = p + return + elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0): + self.error("not found") + return + else: + if forwards: + i += 1 + s = self.get_item(i) + p = -1 + else: + i -= 1 + s = self.get_item(i) + p = len(s) + + def finish(self) -> None: + super().finish() + ret = self.get_unicode() + for i, t in self.transient_history.items(): + if i < len(self.history) and i != self.historyi: + self.history[i] = t + if ret and should_auto_add_history: + self.history.append(ret) + + +should_auto_add_history = True diff --git a/Lib/_pyrepl/input.py b/Lib/_pyrepl/input.py new file mode 100644 index 000000000000000..300e16d1d254414 --- /dev/null +++ b/Lib/_pyrepl/input.py @@ -0,0 +1,114 @@ +# Copyright 2000-2004 Michael Hudson-Doyle +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +# (naming modules after builtin functions is not such a hot idea...) + +# an KeyTrans instance translates Event objects into Command objects + +# hmm, at what level do we want [C-i] and [tab] to be equivalent? +# [meta-a] and [esc a]? obviously, these are going to be equivalent +# for the UnixConsole, but should they be for PygameConsole? + +# it would in any situation seem to be a bad idea to bind, say, [tab] +# and [C-i] to *different* things... but should binding one bind the +# other? + +# executive, temporary decision: [tab] and [C-i] are distinct, but +# [meta-key] is identified with [esc key]. We demand that any console +# class does quite a lot towards emulating a unix terminal. + +from __future__ import annotations + +from abc import ABC, abstractmethod +import unicodedata +from collections import deque + + +# types +if False: + from .types import EventTuple + + +class InputTranslator(ABC): + @abstractmethod + def push(self, evt: EventTuple) -> None: + pass + + @abstractmethod + def get(self) -> EventTuple | None: + return None + + @abstractmethod + def empty(self) -> bool: + return True + + +class KeymapTranslator(InputTranslator): + def __init__(self, keymap, verbose=0, invalid_cls=None, character_cls=None): + self.verbose = verbose + from .keymap import compile_keymap, parse_keys + + self.keymap = keymap + self.invalid_cls = invalid_cls + self.character_cls = character_cls + d = {} + for keyspec, command in keymap: + keyseq = tuple(parse_keys(keyspec)) + d[keyseq] = command + if self.verbose: + print(d) + self.k = self.ck = compile_keymap(d, ()) + self.results = deque() + self.stack = [] + + def push(self, evt): + if self.verbose: + print("pushed", evt.data, end="") + key = evt.data + d = self.k.get(key) + if isinstance(d, dict): + if self.verbose: + print("transition") + self.stack.append(key) + self.k = d + else: + if d is None: + if self.verbose: + print("invalid") + if self.stack or len(key) > 1 or unicodedata.category(key) == "C": + self.results.append((self.invalid_cls, self.stack + [key])) + else: + # small optimization: + self.k[key] = self.character_cls + self.results.append((self.character_cls, [key])) + else: + if self.verbose: + print("matched", d) + self.results.append((d, self.stack + [key])) + self.stack = [] + self.k = self.ck + + def get(self): + if self.results: + return self.results.popleft() + else: + return None + + def empty(self): + return not self.results diff --git a/Lib/_pyrepl/keymap.py b/Lib/_pyrepl/keymap.py new file mode 100644 index 000000000000000..31a02642ce8ceb6 --- /dev/null +++ b/Lib/_pyrepl/keymap.py @@ -0,0 +1,215 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" +functions for parsing keyspecs + +Support for turning keyspecs into appropriate sequences. + +pyrepl uses it's own bastardized keyspec format, which is meant to be +a strict superset of readline's \"KEYSEQ\" format (which is to say +that if you can come up with a spec readline accepts that this +doesn't, you've found a bug and should tell me about it). + +Note that this is the `\\C-o' style of readline keyspec, not the +`Control-o' sort. + +A keyspec is a string representing a sequence of keypresses that can +be bound to a command. + +All characters other than the backslash represent themselves. In the +traditional manner, a backslash introduces a escape sequence. + +The extension to readline is that the sequence \\ denotes the +sequence of charaters produced by hitting KEY. + +Examples: + +`a' - what you get when you hit the `a' key +`\\EOA' - Escape - O - A (up, on my terminal) +`\\' - the up arrow key +`\\' - ditto (keynames are case insensitive) +`\\C-o', `\\c-o' - control-o +`\\M-.' - meta-period +`\\E.' - ditto (that's how meta works for pyrepl) +`\\', `\\', `\\t', `\\011', '\\x09', '\\X09', '\\C-i', '\\C-I' + - all of these are the tab character. Can you think of any more? +""" + +_escapes = { + "\\": "\\", + "'": "'", + '"': '"', + "a": "\a", + "b": "\b", + "e": "\033", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", +} + +_keynames = { + "backspace": "backspace", + "delete": "delete", + "down": "down", + "end": "end", + "enter": "\r", + "escape": "\033", + "f1": "f1", + "f2": "f2", + "f3": "f3", + "f4": "f4", + "f5": "f5", + "f6": "f6", + "f7": "f7", + "f8": "f8", + "f9": "f9", + "f10": "f10", + "f11": "f11", + "f12": "f12", + "f13": "f13", + "f14": "f14", + "f15": "f15", + "f16": "f16", + "f17": "f17", + "f18": "f18", + "f19": "f19", + "f20": "f20", + "home": "home", + "insert": "insert", + "left": "left", + "page down": "page down", + "page up": "page up", + "return": "\r", + "right": "right", + "space": " ", + "tab": "\t", + "up": "up", +} + + +class KeySpecError(Exception): + pass + + +def _parse_key1(key, s): + ctrl = 0 + meta = 0 + ret = "" + while not ret and s < len(key): + if key[s] == "\\": + c = key[s + 1].lower() + if c in _escapes: + ret = _escapes[c] + s += 2 + elif c == "c": + if key[s + 2] != "-": + raise KeySpecError( + "\\C must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if ctrl: + raise KeySpecError( + "doubled \\C- (char %d of %s)" % (s + 1, repr(key)) + ) + ctrl = 1 + s += 3 + elif c == "m": + if key[s + 2] != "-": + raise KeySpecError( + "\\M must be followed by `-' (char %d of %s)" + % (s + 2, repr(key)) + ) + if meta: + raise KeySpecError( + "doubled \\M- (char %d of %s)" % (s + 1, repr(key)) + ) + meta = 1 + s += 3 + elif c.isdigit(): + n = key[s + 1 : s + 4] + ret = chr(int(n, 8)) + s += 4 + elif c == "x": + n = key[s + 2 : s + 4] + ret = chr(int(n, 16)) + s += 4 + elif c == "<": + t = key.find(">", s) + if t == -1: + raise KeySpecError( + "unterminated \\< starting at char %d of %s" + % (s + 1, repr(key)) + ) + ret = key[s + 2 : t].lower() + if ret not in _keynames: + raise KeySpecError( + "unrecognised keyname `%s' at char %d of %s" + % (ret, s + 2, repr(key)) + ) + ret = _keynames[ret] + s = t + 1 + else: + raise KeySpecError( + "unknown backslash escape %s at char %d of %s" + % (repr(c), s + 2, repr(key)) + ) + else: + ret = key[s] + s += 1 + if ctrl: + if len(ret) > 1: + raise KeySpecError("\\C- must be followed by a character") + ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl() + if meta: + ret = ["\033", ret] + else: + ret = [ret] + return ret, s + + +def parse_keys(key): + s = 0 + r = [] + while s < len(key): + k, s = _parse_key1(key, s) + r.extend(k) + return r + + +def compile_keymap(keymap, empty=b""): + r = {} + for key, value in keymap.items(): + if isinstance(key, bytes): + first = key[:1] + else: + first = key[0] + r.setdefault(first, {})[key[1:]] = value + for key, value in r.items(): + if empty in value: + if len(value) != 1: + raise KeySpecError("key definitions for %s clash" % (value.values(),)) + else: + r[key] = value[empty] + else: + r[key] = compile_keymap(value, empty) + return r diff --git a/Lib/_pyrepl/mypy.ini b/Lib/_pyrepl/mypy.ini new file mode 100644 index 000000000000000..ecd03094dbf538d --- /dev/null +++ b/Lib/_pyrepl/mypy.ini @@ -0,0 +1,27 @@ +# Config file for running mypy on _pyrepl. +# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini` +# on the command-line from the repo root + +[mypy] +files = Lib/_pyrepl +explicit_package_bases = True +python_version = 3.12 +platform = linux +pretty = True + +# Enable most stricter settings +enable_error_code = ignore-without-code +strict = True + +# Various stricter settings that we can't yet enable +# Try to enable these in the following order: +disallow_any_generics = False +disallow_untyped_calls = False +disallow_untyped_defs = False +check_untyped_defs = False + +disable_error_code = return + +# Various internal modules that typeshed deliberately doesn't have stubs for: +[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*] +ignore_missing_imports = True diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py new file mode 100644 index 000000000000000..ecf5ddc79a5efa3 --- /dev/null +++ b/Lib/_pyrepl/pager.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import io +import os +import re +import sys + + +# types +if False: + from typing import Protocol, Any + class Pager(Protocol): + def __call__(self, text: str, title: str = "") -> None: + ... + + +def get_pager() -> Pager: + """Decide what method to use for paging through text.""" + if not hasattr(sys.stdin, "isatty"): + return plain_pager + if not hasattr(sys.stdout, "isatty"): + return plain_pager + if not sys.stdin.isatty() or not sys.stdout.isatty(): + return plain_pager + if sys.platform == "emscripten": + return plainpager + use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') + if use_pager: + if sys.platform == 'win32': # pipes completely broken in Windows + return lambda text, title='': tempfile_pager(plain(text), use_pager) + elif os.environ.get('TERM') in ('dumb', 'emacs'): + return lambda text, title='': pipe_pager(plain(text), use_pager, title) + else: + return lambda text, title='': pipe_pager(text, use_pager, title) + if os.environ.get('TERM') in ('dumb', 'emacs'): + return plain_pager + if sys.platform == 'win32': + return lambda text, title='': tempfilepager(plain(text), 'more <') + if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: + return lambda text, title='': pipe_pager(text, 'less', title) + + import tempfile + (fd, filename) = tempfile.mkstemp() + os.close(fd) + try: + if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: + return lambda text, title='': pipe_pager(text, 'more', title) + else: + return tty_pager + finally: + os.unlink(filename) + + +def escape_stdout(text: str) -> str: + # Escape non-encodable characters to avoid encoding errors later + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + return text.encode(encoding, 'backslashreplace').decode(encoding) + + +def escape_less(s: str) -> str: + return re.sub(r'([?:.%\\])', r'\\\1', s) + + +def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + +def tty_pager(text: str, title: str = '') -> None: + """Page through text on a text terminal.""" + lines = plain(escape_stdout(text)).split('\n') + has_tty = False + try: + import tty + import termios + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + tty.setcbreak(fd) + getchar = lambda: sys.stdin.read(1) + has_tty = True + except (ImportError, AttributeError, io.UnsupportedOperation): + getchar = lambda: sys.stdin.readline()[:-1][:1] + + try: + try: + h = int(os.environ.get('LINES', 0)) + except ValueError: + h = 0 + if h <= 1: + h = 25 + r = inc = h - 1 + sys.stdout.write('\n'.join(lines[:inc]) + '\n') + while lines[r:]: + sys.stdout.write('-- more --') + sys.stdout.flush() + c = getchar() + + if c in ('q', 'Q'): + sys.stdout.write('\r \r') + break + elif c in ('\r', '\n'): + sys.stdout.write('\r \r' + lines[r] + '\n') + r = r + 1 + continue + if c in ('b', 'B', '\x1b'): + r = r - inc - inc + if r < 0: r = 0 + sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') + r = r + inc + + finally: + if has_tty: + termios.tcsetattr(fd, termios.TCSAFLUSH, old) + + +def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + sys.stdout.write(plain(escape_stdout(text))) + + +def pipe_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by feeding it to another program.""" + import subprocess + env = os.environ.copy() + if title: + title += ' ' + esc_title = escape_less(title) + prompt_string = ( + f' {esc_title}' + + '?ltline %lt?L/%L.' + ':byte %bB?s/%s.' + '.' + '?e (END):?pB %pB\\%..' + ' (press h for help or q to quit)') + env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) + proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + errors='backslashreplace', env=env) + assert proc.stdin is not None + try: + with proc.stdin as pipe: + try: + pipe.write(text) + except KeyboardInterrupt: + # We've hereby abandoned whatever text hasn't been written, + # but the pager is still in control of the terminal. + pass + except OSError: + pass # Ignore broken pipes caused by quitting the pager program. + while True: + try: + proc.wait() + break + except KeyboardInterrupt: + # Ignore ctl-c like the pager itself does. Otherwise the pager is + # left running and the terminal is in raw mode and unusable. + pass + + +def tempfile_pager(text: str, cmd: str, title: str = '') -> None: + """Page through text by invoking a program on a temporary file.""" + import tempfile + with tempfile.TemporaryDirectory() as tempdir: + filename = os.path.join(tempdir, 'pydoc.out') + with open(filename, 'w', errors='backslashreplace', + encoding=os.device_encoding(0) if + sys.platform == 'win32' else None + ) as file: + file.write(text) + os.system(cmd + ' "' + filename + '"') diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py new file mode 100644 index 000000000000000..a7ef988da12a6a2 --- /dev/null +++ b/Lib/_pyrepl/reader.py @@ -0,0 +1,660 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +from contextlib import contextmanager +from dataclasses import dataclass, field, fields +import re +import unicodedata +from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] + + +from . import commands, console, input +from .utils import ANSI_ESCAPE_SEQUENCE, wlen +from .trace import trace + + +# types +Command = commands.Command +if False: + from .types import Callback, SimpleContextManager, KeySpec, CommandName + + +def disp_str(buffer: str) -> tuple[str, list[int]]: + """disp_str(buffer:string) -> (string, [int]) + + Return the string that should be the printed represenation of + |buffer| and a list detailing where the characters of |buffer| + get used up. E.g.: + + >>> disp_str(chr(3)) + ('^C', [1, 0]) + + """ + b: list[int] = [] + s: list[str] = [] + for c in buffer: + if unicodedata.category(c).startswith("C"): + c = r"\u%04x" % ord(c) + s.append(c) + b.append(wlen(c)) + b.extend([0] * (len(c) - 1)) + return "".join(s), b + + +# syntax classes: + +SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3) + + +def make_default_syntax_table() -> dict[str, int]: + # XXX perhaps should use some unicodedata here? + st: dict[str, int] = {} + for c in map(chr, range(256)): + st[c] = SYNTAX_SYMBOL + for c in [a for a in map(chr, range(256)) if a.isalnum()]: + st[c] = SYNTAX_WORD + st["\n"] = st[" "] = SYNTAX_WHITESPACE + return st + + +def make_default_commands() -> dict[CommandName, type[Command]]: + result: dict[CommandName, type[Command]] = {} + for v in vars(commands).values(): + if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower(): + result[v.__name__] = v + result[v.__name__.replace("_", "-")] = v + return result + + +default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple( + [ + (r"\C-a", "beginning-of-line"), + (r"\C-b", "left"), + (r"\C-c", "interrupt"), + (r"\C-d", "delete"), + (r"\C-e", "end-of-line"), + (r"\C-f", "right"), + (r"\C-g", "cancel"), + (r"\C-h", "backspace"), + (r"\C-j", "accept"), + (r"\", "accept"), + (r"\C-k", "kill-line"), + (r"\C-l", "clear-screen"), + (r"\C-m", "accept"), + (r"\C-t", "transpose-characters"), + (r"\C-u", "unix-line-discard"), + (r"\C-w", "unix-word-rubout"), + (r"\C-x\C-u", "upcase-region"), + (r"\C-y", "yank"), + (r"\C-z", "suspend"), + (r"\M-b", "backward-word"), + (r"\M-c", "capitalize-word"), + (r"\M-d", "kill-word"), + (r"\M-f", "forward-word"), + (r"\M-l", "downcase-word"), + (r"\M-t", "transpose-words"), + (r"\M-u", "upcase-word"), + (r"\M-y", "yank-pop"), + (r"\M--", "digit-arg"), + (r"\M-0", "digit-arg"), + (r"\M-1", "digit-arg"), + (r"\M-2", "digit-arg"), + (r"\M-3", "digit-arg"), + (r"\M-4", "digit-arg"), + (r"\M-5", "digit-arg"), + (r"\M-6", "digit-arg"), + (r"\M-7", "digit-arg"), + (r"\M-8", "digit-arg"), + (r"\M-9", "digit-arg"), + # (r'\M-\n', 'insert-nl'), + ("\\\\", "self-insert"), + ] + + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] + + [ + (r"\", "up"), + (r"\", "down"), + (r"\", "left"), + (r"\", "right"), + (r"\", "delete"), + (r"\", "backspace"), + (r"\M-\", "backward-kill-word"), + (r"\", "end-of-line"), # was 'end' + (r"\", "beginning-of-line"), # was 'home' + (r"\", "help"), + (r"\", "show-history"), + (r"\", "paste-mode"), + (r"\EOF", "end"), # the entries in the terminfo database for xterms + (r"\EOH", "home"), # seem to be wrong. this is a less than ideal + # workaround + ] +) + + +@dataclass(slots=True) +class Reader: + """The Reader class implements the bare bones of a command reader, + handling such details as editing and cursor motion. What it does + not support are such things as completion or history support - + these are implemented elsewhere. + + Instance variables of note include: + + * buffer: + A *list* (*not* a string at the moment :-) containing all the + characters that have been entered. + * console: + Hopefully encapsulates the OS dependent stuff. + * pos: + A 0-based index into `buffer' for where the insertion point + is. + * screeninfo: + Ahem. This list contains some info needed to move the + insertion point around reasonably efficiently. + * cxy, lxy: + the position of the insertion point in screen ... + * syntax_table: + Dictionary mapping characters to `syntax class'; read the + emacs docs to see what this means :-) + * commands: + Dictionary mapping command names to command classes. + * arg: + The emacs-style prefix argument. It will be None if no such + argument has been provided. + * dirty: + True if we need to refresh the display. + * kill_ring: + The emacs-style kill-ring; manipulated with yank & yank-pop + * ps1, ps2, ps3, ps4: + prompts. ps1 is the prompt for a one-line input; for a + multiline input it looks like: + ps2> first line of input goes here + ps3> second and further + ps3> lines get ps3 + ... + ps4> and the last one gets ps4 + As with the usual top-level, you can set these to instances if + you like; str() will be called on them (once) at the beginning + of each command. Don't put really long or newline containing + strings here, please! + This is just the default policy; you can change it freely by + overriding get_prompt() (and indeed some standard subclasses + do). + * finished: + handle1 will set this to a true value if a command signals + that we're done. + """ + + console: console.Console + + ## state + buffer: list[str] = field(default_factory=list) + pos: int = 0 + ps1: str = "->> " + ps2: str = "/>> " + ps3: str = "|.. " + ps4: str = R"\__ " + kill_ring: list[list[str]] = field(default_factory=list) + msg: str = "" + arg: int | None = None + dirty: bool = False + finished: bool = False + paste_mode: bool = False + commands: dict[str, type[Command]] = field(default_factory=make_default_commands) + last_command: type[Command] | None = None + syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) + msg_at_bottom: bool = True + keymap: tuple[tuple[str, str], ...] = () + input_trans: input.KeymapTranslator = field(init=False) + input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list) + screeninfo: list[tuple[int, list[int]]] = field(init=False) + cxy: tuple[int, int] = field(init=False) + lxy: tuple[int, int] = field(init=False) + + def __post_init__(self) -> None: + # Enable the use of `insert` without a `prepare` call - necessary to + # facilitate the tab completion hack implemented for + # . + self.keymap = self.collect_keymap() + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + self.screeninfo = [(0, [0])] + self.cxy = self.pos2xy() + self.lxy = (self.pos, 0) + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return default_keymap + + def calc_screen(self) -> list[str]: + """The purpose of this method is to translate changes in + self.buffer into changes in self.screen. Currently it rips + everything down and starts from scratch, which whilst not + especially efficient is certainly simple(r). + """ + lines = self.get_unicode().split("\n") + screen: list[str] = [] + screeninfo: list[tuple[int, list[int]]] = [] + pos = self.pos + for ln, line in enumerate(lines): + ll = len(line) + if 0 <= pos <= ll: + if self.msg and not self.msg_at_bottom: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + self.lxy = pos, ln + prompt = self.get_prompt(ln, ll >= pos >= 0) + while "\n" in prompt: + pre_prompt, _, prompt = prompt.partition("\n") + screen.append(pre_prompt) + screeninfo.append((0, [])) + pos -= ll + 1 + prompt, lp = self.process_prompt(prompt) + l, l2 = disp_str(line) + wrapcount = (wlen(l) + lp) // self.console.width + if wrapcount == 0: + screen.append(prompt + l) + screeninfo.append((lp, l2)) + else: + for i in range(wrapcount + 1): + prelen = lp if i == 0 else 0 + index_to_wrap_before = 0 + column = 0 + for character_width in l2: + if column + character_width >= self.console.width - prelen: + break + index_to_wrap_before += 1 + column += character_width + pre = prompt if i == 0 else "" + post = "\\" if i != wrapcount else "" + after = [1] if i != wrapcount else [] + screen.append(pre + l[:index_to_wrap_before] + post) + screeninfo.append((prelen, l2[:index_to_wrap_before] + after)) + l = l[index_to_wrap_before:] + l2 = l2[index_to_wrap_before:] + self.screeninfo = screeninfo + self.cxy = self.pos2xy() + if self.msg and self.msg_at_bottom: + for mline in self.msg.split("\n"): + screen.append(mline) + screeninfo.append((0, [])) + return screen + + def process_prompt(self, prompt: str) -> tuple[str, int]: + """Process the prompt. + + This means calculate the length of the prompt. The character \x01 + and \x02 are used to bracket ANSI control sequences and need to be + excluded from the length calculation. So also a copy of the prompt + is returned with these control characters removed.""" + + # The logic below also ignores the length of common escape + # sequences if they were not explicitly within \x01...\x02. + # They are CSI (or ANSI) sequences ( ESC [ ... LETTER ) + + out_prompt = "" + l = wlen(prompt) + pos = 0 + while True: + s = prompt.find("\x01", pos) + if s == -1: + break + e = prompt.find("\x02", s) + if e == -1: + break + # Found start and end brackets, subtract from string length + l = l - (e - s + 1) + keep = prompt[pos:s] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + prompt[s + 1 : e] + pos = e + 1 + keep = prompt[pos:] + l -= sum(map(wlen, ANSI_ESCAPE_SEQUENCE.findall(keep))) + out_prompt += keep + return out_prompt, l + + def bow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break preceding p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p -= 1 + while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p -= 1 + return p + 1 + + def eow(self, p: int | None = None) -> int: + """Return the 0-based index of the word break following p most + immediately. + + p defaults to self.pos; word boundaries are determined using + self.syntax_table.""" + if p is None: + p = self.pos + st = self.syntax_table + b = self.buffer + while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD: + p += 1 + while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD: + p += 1 + return p + + def bol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break preceding p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + p -= 1 + while p >= 0 and b[p] != "\n": + p -= 1 + return p + 1 + + def eol(self, p: int | None = None) -> int: + """Return the 0-based index of the line break following p most + immediately. + + p defaults to self.pos.""" + if p is None: + p = self.pos + b = self.buffer + while p < len(b) and b[p] != "\n": + p += 1 + return p + + def max_column(self, y: int) -> int: + """Return the last x-offset for line y""" + return self.screeninfo[y][0] + sum(self.screeninfo[y][1]) + + def max_row(self) -> int: + return len(self.screeninfo) - 1 + + def get_arg(self, default: int = 1) -> int: + """Return any prefix argument that the user has supplied, + returning `default' if there is None. Defaults to 1. + """ + if self.arg is None: + return default + else: + return self.arg + + def get_prompt(self, lineno: int, cursor_on_line: bool) -> str: + """Return what should be in the left-hand margin for line + `lineno'.""" + if self.arg is not None and cursor_on_line: + prompt = "(arg: %s) " % self.arg + elif self.paste_mode: + prompt = "(paste) " + elif "\n" in self.buffer: + if lineno == 0: + prompt = self.ps2 + elif lineno == self.buffer.count("\n"): + prompt = self.ps4 + else: + prompt = self.ps3 + else: + prompt = self.ps1 + + if can_colorize(): + prompt = f"{ANSIColors.BOLD_MAGENTA}{prompt}{ANSIColors.RESET}" + return prompt + + def push_input_trans(self, itrans: input.KeymapTranslator) -> None: + self.input_trans_stack.append(self.input_trans) + self.input_trans = itrans + + def pop_input_trans(self) -> None: + self.input_trans = self.input_trans_stack.pop() + + def setpos_from_xy(self, x: int, y: int) -> None: + """Set pos according to coordinates x, y""" + pos = 0 + i = 0 + while i < y: + prompt_len, character_widths = self.screeninfo[i] + offset = len(character_widths) - character_widths.count(0) + in_wrapped_line = prompt_len + sum(character_widths) >= self.console.width + if in_wrapped_line: + pos += offset - 1 # -1 cause backslash is not in buffer + else: + pos += offset + 1 # +1 cause newline is in buffer + i += 1 + + j = 0 + cur_x = self.screeninfo[i][0] + while cur_x < x: + if self.screeninfo[i][1][j] == 0: + continue + cur_x += self.screeninfo[i][1][j] + j += 1 + pos += 1 + + self.pos = pos + + def pos2xy(self) -> tuple[int, int]: + """Return the x, y coordinates of position 'pos'.""" + # this *is* incomprehensible, yes. + y = 0 + pos = self.pos + assert 0 <= pos <= len(self.buffer) + if pos == len(self.buffer): + y = len(self.screeninfo) - 1 + p, l2 = self.screeninfo[y] + return p + sum(l2) + l2.count(0), y + + for p, l2 in self.screeninfo: + l = len(l2) - l2.count(0) + in_wrapped_line = p + sum(l2) >= self.console.width + offset = l - 1 if in_wrapped_line else l # need to remove backslash + if offset >= pos: + break + else: + if p + sum(l2) >= self.console.width: + pos -= l - 1 # -1 cause backslash is not in buffer + else: + pos -= l + 1 # +1 cause newline is in buffer + y += 1 + return p + sum(l2[:pos]), y + + def insert(self, text: str | list[str]) -> None: + """Insert 'text' at the insertion point.""" + self.buffer[self.pos : self.pos] = list(text) + self.pos += len(text) + self.dirty = True + + def update_cursor(self) -> None: + """Move the cursor to reflect changes in self.pos""" + self.cxy = self.pos2xy() + self.console.move_cursor(*self.cxy) + + def after_command(self, cmd: Command) -> None: + """This function is called to allow post command cleanup.""" + if getattr(cmd, "kills_digit_arg", True): + if self.arg is not None: + self.dirty = True + self.arg = None + + def prepare(self) -> None: + """Get ready to run. Call restore when finished. You must not + write to the console in between the calls to prepare and + restore.""" + try: + self.console.prepare() + self.arg = None + self.finished = False + del self.buffer[:] + self.pos = 0 + self.dirty = True + self.last_command = None + self.calc_screen() + except BaseException: + self.restore() + raise + + def last_command_is(self, cls: type) -> bool: + if not self.last_command: + return False + return issubclass(cls, self.last_command) + + def restore(self) -> None: + """Clean up after a run.""" + self.console.restore() + + @contextmanager + def suspend(self) -> SimpleContextManager: + """A context manager to delegate to another reader.""" + prev_state = {f.name: getattr(self, f.name) for f in fields(self)} + try: + self.restore() + yield + finally: + for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"): + setattr(self, arg, prev_state[arg]) + self.prepare() + pass + + def finish(self) -> None: + """Called when a command signals that we're finished.""" + pass + + def error(self, msg: str = "none") -> None: + self.msg = "! " + msg + " " + self.dirty = True + self.console.beep() + + def update_screen(self) -> None: + if self.dirty: + self.refresh() + + def refresh(self) -> None: + """Recalculate and refresh the screen.""" + # this call sets up self.cxy, so call it first. + screen = self.calc_screen() + self.console.refresh(screen, self.cxy) + self.dirty = False + + def do_cmd(self, cmd: tuple[str, list[str]]) -> None: + """`cmd` is a tuple of "event_name" and "event", which in the current + implementation is always just the "buffer" which happens to be a list + of single-character strings.""" + assert isinstance(cmd[0], str) + + trace("received command {cmd}", cmd=cmd) + command_type = self.commands.get(cmd[0], commands.invalid_command) + command = command_type(self, *cmd) # type: ignore[arg-type] + + command.do() + + self.after_command(command) + + if self.dirty: + self.refresh() + else: + self.update_cursor() + + if not isinstance(cmd, commands.digit_arg): + self.last_command = command_type + + self.finished = bool(command.finish) + if self.finished: + self.console.finish() + self.finish() + + def handle1(self, block: bool = True) -> bool: + """Handle a single event. Wait as long as it takes if block + is true (the default), otherwise return False if no event is + pending.""" + + if self.msg: + self.msg = "" + self.dirty = True + + while True: + event = self.console.get_event(block) + if not event: # can only happen if we're not blocking + return False + + translate = True + + if event.evt == "key": + self.input_trans.push(event) + elif event.evt == "scroll": + self.refresh() + elif event.evt == "resize": + self.refresh() + else: + translate = False + + if translate: + cmd = self.input_trans.get() + else: + cmd = [event.evt, event.data] + + if cmd is None: + if block: + continue + else: + return False + + self.do_cmd(cmd) + return True + + def push_char(self, char: int | bytes) -> None: + self.console.push_char(char) + self.handle1(block=False) + + def readline(self, startup_hook: Callback | None = None) -> str: + """Read a line. The implementation of this method also shows + how to drive Reader if you want more control over the event + loop.""" + self.prepare() + try: + if startup_hook is not None: + startup_hook() + self.refresh() + while not self.finished: + self.handle1() + return self.get_unicode() + + finally: + self.restore() + + def bind(self, spec: KeySpec, command: CommandName) -> None: + self.keymap = self.keymap + ((spec, command),) + self.input_trans = input.KeymapTranslator( + self.keymap, invalid_cls="invalid-key", character_cls="self-insert" + ) + + def get_unicode(self) -> str: + """Return the current buffer as a unicode string.""" + return "".join(self.buffer) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py new file mode 100644 index 000000000000000..37ba98d4c8c87a5 --- /dev/null +++ b/Lib/_pyrepl/readline.py @@ -0,0 +1,501 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Alex Gaynor +# Antonio Cuni +# Armin Rigo +# Holger Krekel +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""A compatibility wrapper reimplementing the 'readline' standard module +on top of pyrepl. Not all functionalities are supported. Contains +extensions for multiline input. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import os +import readline +from site import gethistoryfile # type: ignore[attr-defined] +import sys + +from . import commands, historical_reader +from .completing_reader import CompletingReader +from .unix_console import UnixConsole, _error + +ENCODING = sys.getdefaultencoding() or "latin1" + + +# types +Command = commands.Command +from collections.abc import Callable, Collection +from .types import Callback, Completer, KeySpec, CommandName + + +__all__ = [ + "add_history", + "clear_history", + "get_begidx", + "get_completer", + "get_completer_delims", + "get_current_history_length", + "get_endidx", + "get_history_item", + "get_history_length", + "get_line_buffer", + "insert_text", + "parse_and_bind", + "read_history_file", + # "read_init_file", + # "redisplay", + "remove_history_item", + "replace_history_item", + "set_auto_history", + "set_completer", + "set_completer_delims", + "set_history_length", + # "set_pre_input_hook", + "set_startup_hook", + "write_history_file", + # ---- multiline extensions ---- + "multiline_input", +] + +# ____________________________________________________________ + +@dataclass +class ReadlineConfig: + readline_completer: Completer | None = readline.get_completer() + completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?") + + +@dataclass(kw_only=True) +class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader): + # Class fields + assume_immutable_completions = False + use_brackets = False + sort_in_column = True + + # Instance fields + config: ReadlineConfig + more_lines: Callable[[str], bool] | None = None + + def __post_init__(self) -> None: + super().__post_init__() + self.commands["maybe_accept"] = maybe_accept + self.commands["maybe-accept"] = maybe_accept + self.commands["backspace_dedent"] = backspace_dedent + self.commands["backspace-dedent"] = backspace_dedent + + def error(self, msg: str = "none") -> None: + pass # don't show error messages by default + + def get_stem(self) -> str: + b = self.buffer + p = self.pos - 1 + completer_delims = self.config.completer_delims + while p >= 0 and b[p] not in completer_delims: + p -= 1 + return "".join(b[p + 1 : self.pos]) + + def get_completions(self, stem: str) -> list[str]: + if len(stem) == 0 and self.more_lines is not None: + b = self.buffer + p = self.pos + while p > 0 and b[p - 1] != "\n": + p -= 1 + num_spaces = 4 - ((self.pos - p) % 4) + return [" " * num_spaces] + result = [] + function = self.config.readline_completer + if function is not None: + try: + stem = str(stem) # rlcompleter.py seems to not like unicode + except UnicodeEncodeError: + pass # but feed unicode anyway if we have no choice + state = 0 + while True: + try: + next = function(stem, state) + except Exception: + break + if not isinstance(next, str): + break + result.append(next) + state += 1 + # emulate the behavior of the standard readline that sorts + # the completions before displaying them. + result.sort() + return result + + def get_trimmed_history(self, maxlength: int) -> list[str]: + if maxlength >= 0: + cut = len(self.history) - maxlength + if cut < 0: + cut = 0 + else: + cut = 0 + return self.history[cut:] + + # --- simplified support for reading multiline Python statements --- + + def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]: + return super().collect_keymap() + ( + (r"\n", "maybe-accept"), + (r"\", "backspace-dedent"), + ) + + def after_command(self, cmd: Command) -> None: + super().after_command(cmd) + if self.more_lines is None: + # Force single-line input if we are in raw_input() mode. + # Although there is no direct way to add a \n in this mode, + # multiline buffers can still show up using various + # commands, e.g. navigating the history. + try: + index = self.buffer.index("\n") + except ValueError: + pass + else: + self.buffer = self.buffer[:index] + if self.pos > len(self.buffer): + self.pos = len(self.buffer) + + +def set_auto_history(_should_auto_add_history: bool) -> None: + """Enable or disable automatic history""" + historical_reader.should_auto_add_history = bool(_should_auto_add_history) + + +def _get_this_line_indent(buffer: list[str], pos: int) -> int: + indent = 0 + while pos > 0 and buffer[pos - 1] in " \t": + indent += 1 + pos -= 1 + if pos > 0 and buffer[pos - 1] == "\n": + return indent + return 0 + + +def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]: + prevlinestart = pos + while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n": + prevlinestart -= 1 + prevlinetext = prevlinestart + while prevlinetext < pos and buffer[prevlinetext] in " \t": + prevlinetext += 1 + if prevlinetext == pos: + indent = None + else: + indent = prevlinetext - prevlinestart + return prevlinestart, indent + + +class maybe_accept(commands.Command): + def do(self) -> None: + r: ReadlineAlikeReader + r = self.reader # type: ignore[assignment] + r.dirty = True # this is needed to hide the completion menu, if visible + # + # if there are already several lines and the cursor + # is not on the last one, always insert a new \n. + text = r.get_unicode() + if "\n" in r.buffer[r.pos :] or ( + r.more_lines is not None and r.more_lines(text) + ): + # + # auto-indent the next line like the previous line + prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos) + r.insert("\n") + if not self.reader.paste_mode and indent: + for i in range(prevlinestart, prevlinestart + indent): + r.insert(r.buffer[i]) + elif not self.reader.paste_mode: + self.finish = True + else: + r.insert("\n") + + +class backspace_dedent(commands.Command): + def do(self) -> None: + r = self.reader + b = r.buffer + if r.pos > 0: + repeat = 1 + if b[r.pos - 1] != "\n": + indent = _get_this_line_indent(b, r.pos) + if indent > 0: + ls = r.pos - indent + while ls > 0: + ls, pi = _get_previous_line_indent(b, ls - 1) + if pi is not None and pi < indent: + repeat = indent - pi + break + r.pos -= repeat + del b[r.pos : r.pos + repeat] + r.dirty = True + else: + self.reader.error("can't backspace at start") + + +# ____________________________________________________________ + + +@dataclass(slots=True) +class _ReadlineWrapper: + f_in: int = -1 + f_out: int = -1 + reader: ReadlineAlikeReader | None = None + saved_history_length: int = -1 + startup_hook: Callback | None = None + config: ReadlineConfig = field(default_factory=ReadlineConfig) + + def __post_init__(self) -> None: + if self.f_in == -1: + self.f_in = os.dup(0) + if self.f_out == -1: + self.f_out = os.dup(1) + + def get_reader(self) -> ReadlineAlikeReader: + if self.reader is None: + console = UnixConsole(self.f_in, self.f_out, encoding=ENCODING) + self.reader = ReadlineAlikeReader(console=console, config=self.config) + return self.reader + + def input(self, prompt: object = "") -> str: + try: + reader = self.get_reader() + except _error: + assert raw_input is not None + return raw_input(prompt) + reader.ps1 = str(prompt) + return reader.readline(startup_hook=self.startup_hook) + + def multiline_input(self, more_lines, ps1, ps2): + """Read an input on possibly multiple lines, asking for more + lines as long as 'more_lines(unicodetext)' returns an object whose + boolean value is true. + """ + reader = self.get_reader() + saved = reader.more_lines + try: + reader.more_lines = more_lines + reader.ps1 = reader.ps2 = ps1 + reader.ps3 = reader.ps4 = ps2 + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + def parse_and_bind(self, string: str) -> None: + pass # XXX we don't support parsing GNU-readline-style init files + + def set_completer(self, function: Completer | None = None) -> None: + self.config.readline_completer = function + + def get_completer(self) -> Completer | None: + return self.config.readline_completer + + def set_completer_delims(self, delimiters: Collection[str]) -> None: + self.config.completer_delims = frozenset(delimiters) + + def get_completer_delims(self) -> str: + return "".join(sorted(self.config.completer_delims)) + + def _histline(self, line: str) -> str: + line = line.rstrip("\n") + return line + + def get_history_length(self) -> int: + return self.saved_history_length + + def set_history_length(self, length: int) -> None: + self.saved_history_length = length + + def get_current_history_length(self) -> int: + return len(self.get_reader().history) + + def read_history_file(self, filename: str = gethistoryfile()) -> None: + # multiline extension (really a hack) for the end of lines that + # are actually continuations inside a single multiline_input() + # history item: we use \r\n instead of just \n. If the history + # file is passed to GNU readline, the extra \r are just ignored. + history = self.get_reader().history + + with open(os.path.expanduser(filename), 'rb') as f: + lines = [line.decode('utf-8', errors='replace') for line in f.read().split(b'\n')] + buffer = [] + for line in lines: + # Ignore readline history file header + if line.startswith("_HiStOrY_V2_"): + continue + if line.endswith("\r"): + buffer.append(line+'\n') + else: + line = self._histline(line) + if buffer: + line = "".join(buffer).replace("\r", "") + line + del buffer[:] + if line: + history.append(line) + + def write_history_file(self, filename: str = gethistoryfile()) -> None: + maxlength = self.saved_history_length + history = self.get_reader().get_trimmed_history(maxlength) + with open(os.path.expanduser(filename), "w", encoding="utf-8") as f: + for entry in history: + entry = entry.replace("\n", "\r\n") # multiline history support + f.write(entry + "\n") + + def clear_history(self) -> None: + del self.get_reader().history[:] + + def get_history_item(self, index: int) -> str | None: + history = self.get_reader().history + if 1 <= index <= len(history): + return history[index - 1] + else: + return None # like readline.c + + def remove_history_item(self, index: int) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + del history[index] + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def replace_history_item(self, index: int, line: str) -> None: + history = self.get_reader().history + if 0 <= index < len(history): + history[index] = self._histline(line) + else: + raise ValueError("No history item at position %d" % index) + # like readline.c + + def add_history(self, line: str) -> None: + self.get_reader().history.append(self._histline(line)) + + def set_startup_hook(self, function: Callback | None = None) -> None: + self.startup_hook = function + + def get_line_buffer(self) -> bytes: + buf_str = self.get_reader().get_unicode() + return buf_str.encode(ENCODING) + + def _get_idxs(self) -> tuple[int, int]: + start = cursor = self.get_reader().pos + buf = self.get_line_buffer() + for i in range(cursor - 1, -1, -1): + if str(buf[i]) in self.get_completer_delims(): + break + start = i + return start, cursor + + def get_begidx(self) -> int: + return self._get_idxs()[0] + + def get_endidx(self) -> int: + return self._get_idxs()[1] + + def insert_text(self, text: str) -> None: + self.get_reader().insert(text) + + +_wrapper = _ReadlineWrapper() + +# ____________________________________________________________ +# Public API + +parse_and_bind = _wrapper.parse_and_bind +set_completer = _wrapper.set_completer +get_completer = _wrapper.get_completer +set_completer_delims = _wrapper.set_completer_delims +get_completer_delims = _wrapper.get_completer_delims +get_history_length = _wrapper.get_history_length +set_history_length = _wrapper.set_history_length +get_current_history_length = _wrapper.get_current_history_length +read_history_file = _wrapper.read_history_file +write_history_file = _wrapper.write_history_file +clear_history = _wrapper.clear_history +get_history_item = _wrapper.get_history_item +remove_history_item = _wrapper.remove_history_item +replace_history_item = _wrapper.replace_history_item +add_history = _wrapper.add_history +set_startup_hook = _wrapper.set_startup_hook +get_line_buffer = _wrapper.get_line_buffer +get_begidx = _wrapper.get_begidx +get_endidx = _wrapper.get_endidx +insert_text = _wrapper.insert_text + +# Extension +multiline_input = _wrapper.multiline_input + +# Internal hook +_get_reader = _wrapper.get_reader + +# ____________________________________________________________ +# Stubs + + +def _make_stub(_name: str, _ret: object) -> None: + def stub(*args: object, **kwds: object) -> None: + import warnings + + warnings.warn("readline.%s() not implemented" % _name, stacklevel=2) + + stub.__name__ = _name + globals()[_name] = stub + + +for _name, _ret in [ + ("read_init_file", None), + ("redisplay", None), + ("set_pre_input_hook", None), +]: + assert _name not in globals(), _name + _make_stub(_name, _ret) + +# ____________________________________________________________ + + +def _setup() -> None: + global raw_input + if raw_input is not None: + return # don't run _setup twice + + try: + f_in = sys.stdin.fileno() + f_out = sys.stdout.fileno() + except (AttributeError, ValueError): + return + if not os.isatty(f_in) or not os.isatty(f_out): + return + + _wrapper.f_in = f_in + _wrapper.f_out = f_out + + # this is not really what readline.c does. Better than nothing I guess + import builtins + + raw_input = builtins.input + builtins.input = _wrapper.input + + +raw_input: Callable[[object], str] | None = None diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py new file mode 100644 index 000000000000000..ce79d0d547ebcea --- /dev/null +++ b/Lib/_pyrepl/simple_interact.py @@ -0,0 +1,157 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +"""This is an alternative to python_reader which tries to emulate +the CPython prompt as closely as possible, with the exception of +allowing multiline input and multiline history entries. +""" + +from __future__ import annotations + +import _colorize # type: ignore[import-not-found] +import _sitebuiltins +import linecache +import sys +import code +from types import ModuleType + +from .console import Event +from .readline import _get_reader, multiline_input +from .unix_console import _error + + +def check() -> str: + """Returns the error message if there is a problem initializing the state.""" + try: + _get_reader() + except _error as e: + return str(e) or repr(e) or "unknown error" + return "" + + +def _strip_final_indent(text: str) -> str: + # kill spaces and tabs at the end, but only if they follow '\n'. + # meant to remove the auto-indentation only (although it would of + # course also remove explicitly-added indentation). + short = text.rstrip(" \t") + n = len(short) + if n > 0 and text[n - 1] == "\n": + return short + return text + + +REPL_COMMANDS = { + "exit": _sitebuiltins.Quitter('exit', ''), + "quit": _sitebuiltins.Quitter('quit' ,''), + "copyright": _sitebuiltins._Printer('copyright', sys.copyright), + "help": "help", +} + +class InteractiveColoredConsole(code.InteractiveConsole): + def __init__( + self, + locals: dict[str, object] | None = None, + filename: str = "", + *, + local_exit: bool = False, + ) -> None: + super().__init__(locals=locals, filename=filename, local_exit=local_exit) # type: ignore[call-arg] + self.can_colorize = _colorize.can_colorize() + + def showtraceback(self): + super().showtraceback(colorize=self.can_colorize) + + +def run_multiline_interactive_console( + mainmodule: ModuleType | None= None, future_flags: int = 0 +) -> None: + import code + import __main__ + from .readline import _setup + _setup() + + mainmodule = mainmodule or __main__ + console = InteractiveColoredConsole(mainmodule.__dict__, filename="") + if future_flags: + console.compile.compiler.flags |= future_flags + + input_n = 0 + + def maybe_run_command(statement: str) -> bool: + statement = statement.strip() + if statement in console.locals or statement not in REPL_COMMANDS: + return False + + reader = _get_reader() + reader.history.pop() # skip internal commands in history + command = REPL_COMMANDS[statement] + if callable(command): + command() + return True + + if isinstance(command, str): + # Internal readline commands require a prepared reader like + # inside multiline_input. + reader.prepare() + reader.refresh() + reader.do_cmd((command, [statement])) + reader.restore() + return True + + return False + + def more_lines(unicodetext: str) -> bool: + # ooh, look at the hack: + src = _strip_final_indent(unicodetext) + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + while 1: + try: + try: + sys.stdout.flush() + except Exception: + pass + + ps1 = getattr(sys, "ps1", ">>> ") + ps2 = getattr(sys, "ps2", "... ") + try: + statement = multiline_input(more_lines, ps1, ps2) + except EOFError: + break + + if maybe_run_command(statement): + continue + + input_name = f"" + linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] + more = console.push(_strip_final_indent(statement), filename=input_name) # type: ignore[call-arg] + assert not more + input_n += 1 + except KeyboardInterrupt: + console.write("\nKeyboardInterrupt\n") + console.resetbuffer() + except MemoryError: + console.write("\nMemoryError\n") + console.resetbuffer() diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py new file mode 100644 index 000000000000000..a8eb2433cd3ccee --- /dev/null +++ b/Lib/_pyrepl/trace.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +import os + +# types +if False: + from typing import IO + + +trace_file: IO[str] | None = None +if trace_filename := os.environ.get("PYREPL_TRACE"): + trace_file = open(trace_filename, "a") + + +def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/types.py b/Lib/_pyrepl/types.py new file mode 100644 index 000000000000000..f9d48b828c720b7 --- /dev/null +++ b/Lib/_pyrepl/types.py @@ -0,0 +1,8 @@ +from collections.abc import Callable, Iterator + +Callback = Callable[[], object] +SimpleContextManager = Iterator[None] +KeySpec = str # like r"\C-c" +CommandName = str # like "interrupt" +EventTuple = tuple[CommandName, str] +Completer = Callable[[str, int], str | None] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py new file mode 100644 index 000000000000000..c22b1d5b5bc2904 --- /dev/null +++ b/Lib/_pyrepl/unix_console.py @@ -0,0 +1,743 @@ +# Copyright 2000-2010 Michael Hudson-Doyle +# Antonio Cuni +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from __future__ import annotations + +import errno +import os +import re +import select +import signal +import struct +import sys +import termios +import time +from fcntl import ioctl + +from . import curses +from .console import Console, Event +from .fancy_termios import tcgetattr, tcsetattr +from .trace import trace +from .unix_eventqueue import EventQueue +from .utils import wlen + + +# types +if False: + from typing import IO + + +class InvalidTerminal(RuntimeError): + pass + + +_error = (termios.error, curses.error, InvalidTerminal) + +SIGWINCH_EVENT = "repaint" + +FIONREAD = getattr(termios, "FIONREAD", None) +TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None) + +# ------------ start of baudrate definitions ------------ + +# Add (possibly) missing baudrates (check termios man page) to termios + + +def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None: + baudrate_name = "B%d" % rate + if hasattr(termios, baudrate_name): + dictionary[getattr(termios, baudrate_name)] = rate + + +# Check the termios man page (Line speed) to know where these +# values come from. +potential_baudrates = [ + 0, + 110, + 115200, + 1200, + 134, + 150, + 1800, + 19200, + 200, + 230400, + 2400, + 300, + 38400, + 460800, + 4800, + 50, + 57600, + 600, + 75, + 9600, +] + +ratedict: dict[int, int] = {} +for rate in potential_baudrates: + add_baudrate_if_supported(ratedict, rate) + +# Clean up variables to avoid unintended usage +del rate, add_baudrate_if_supported + +# ------------ end of baudrate definitions ------------ + +delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>") + +try: + poll: type[select.poll] = select.poll +except AttributeError: + # this is exactly the minumum necessary to support what we + # do with poll objects + class MinimalPoll: + def __init__(self): + pass + + def register(self, fd, flag): + self.fd = fd + + def poll(self): # note: a 'timeout' argument would be *milliseconds* + r, w, e = select.select([self.fd], [], []) + return r + + poll = MinimalPoll # type: ignore[assignment] + + +class UnixConsole(Console): + def __init__( + self, + f_in: IO[bytes] | int = 0, + f_out: IO[bytes] | int = 1, + term: str = "", + encoding: str = "", + ): + """ + Initialize the UnixConsole. + + Parameters: + - f_in (int or file-like object): Input file descriptor or object. + - f_out (int or file-like object): Output file descriptor or object. + - term (str): Terminal name. + - encoding (str): Encoding to use for I/O operations. + """ + + self.encoding = encoding or sys.getdefaultencoding() + + if isinstance(f_in, int): + self.input_fd = f_in + else: + self.input_fd = f_in.fileno() + + if isinstance(f_out, int): + self.output_fd = f_out + else: + self.output_fd = f_out.fileno() + + self.pollob = poll() + self.pollob.register(self.input_fd, select.POLLIN) + curses.setupterm(term or None, self.output_fd) + self.term = term + + def _my_getstr(cap, optional=0): + r = curses.tigetstr(cap) + if not optional and r is None: + raise InvalidTerminal( + f"terminal doesn't have the required {cap} capability" + ) + return r + + self._bel = _my_getstr("bel") + self._civis = _my_getstr("civis", optional=True) + self._clear = _my_getstr("clear") + self._cnorm = _my_getstr("cnorm", optional=True) + self._cub = _my_getstr("cub", optional=True) + self._cub1 = _my_getstr("cub1", optional=True) + self._cud = _my_getstr("cud", optional=True) + self._cud1 = _my_getstr("cud1", optional=True) + self._cuf = _my_getstr("cuf", optional=True) + self._cuf1 = _my_getstr("cuf1", optional=True) + self._cup = _my_getstr("cup") + self._cuu = _my_getstr("cuu", optional=True) + self._cuu1 = _my_getstr("cuu1", optional=True) + self._dch1 = _my_getstr("dch1", optional=True) + self._dch = _my_getstr("dch", optional=True) + self._el = _my_getstr("el") + self._hpa = _my_getstr("hpa", optional=True) + self._ich = _my_getstr("ich", optional=True) + self._ich1 = _my_getstr("ich1", optional=True) + self._ind = _my_getstr("ind", optional=True) + self._pad = _my_getstr("pad", optional=True) + self._ri = _my_getstr("ri", optional=True) + self._rmkx = _my_getstr("rmkx", optional=True) + self._smkx = _my_getstr("smkx", optional=True) + + self.__setup_movement() + + self.event_queue = EventQueue(self.input_fd, self.encoding) + self.cursor_visible = 1 + + def change_encoding(self, encoding: str) -> None: + """ + Change the encoding used for I/O operations. + + Parameters: + - encoding (str): New encoding to use. + """ + self.encoding = encoding + + def refresh(self, screen, c_xy): + """ + Refresh the console screen. + + Parameters: + - screen (list): List of strings representing the screen contents. + - c_xy (tuple): Cursor position (x, y) on the screen. + """ + cx, cy = c_xy + if not self.__gone_tall: + while len(self.screen) < min(len(screen), self.height): + self.__hide_cursor() + self.__move(0, len(self.screen) - 1) + self.__write("\n") + self.__posxy = 0, len(self.screen) + self.screen.append("") + else: + while len(self.screen) < len(screen): + self.screen.append("") + + if len(screen) > self.height: + self.__gone_tall = 1 + self.__move = self.__move_tall + + px, py = self.__posxy + old_offset = offset = self.__offset + height = self.height + + # we make sure the cursor is on the screen, and that we're + # using all of the screen if we can + if cy < offset: + offset = cy + elif cy >= offset + height: + offset = cy - height + 1 + elif offset > 0 and len(screen) < offset + height: + offset = max(len(screen) - height, 0) + screen.append("") + + oldscr = self.screen[old_offset : old_offset + height] + newscr = screen[offset : offset + height] + + # use hardware scrolling if we have it. + if old_offset > offset and self._ri: + self.__hide_cursor() + self.__write_code(self._cup, 0, 0) + self.__posxy = 0, old_offset + for i in range(old_offset - offset): + self.__write_code(self._ri) + oldscr.pop(-1) + oldscr.insert(0, "") + elif old_offset < offset and self._ind: + self.__hide_cursor() + self.__write_code(self._cup, self.height - 1, 0) + self.__posxy = 0, old_offset + self.height - 1 + for i in range(offset - old_offset): + self.__write_code(self._ind) + oldscr.pop(0) + oldscr.append("") + + self.__offset = offset + + for ( + y, + oldline, + newline, + ) in zip(range(offset, offset + height), oldscr, newscr): + if oldline != newline: + self.__write_changed_line(y, oldline, newline, px) + + y = len(newscr) + while y < len(oldscr): + self.__hide_cursor() + self.__move(0, y) + self.__posxy = 0, y + self.__write_code(self._el) + y += 1 + + self.__show_cursor() + + self.screen = screen + self.move_cursor(cx, cy) + self.flushoutput() + + def move_cursor(self, x, y): + """ + Move the cursor to the specified position on the screen. + + Parameters: + - x (int): X coordinate. + - y (int): Y coordinate. + """ + if y < self.__offset or y >= self.__offset + self.height: + self.event_queue.insert(Event("scroll", None)) + else: + self.__move(x, y) + self.__posxy = x, y + self.flushoutput() + + def prepare(self): + """ + Prepare the console for input/output operations. + """ + self.__svtermstate = tcgetattr(self.input_fd) + raw = self.__svtermstate.copy() + raw.iflag &= ~(termios.BRKINT | termios.INPCK | termios.ISTRIP | termios.IXON) + raw.oflag &= ~(termios.OPOST) + raw.cflag &= ~(termios.CSIZE | termios.PARENB) + raw.cflag |= termios.CS8 + raw.lflag &= ~( + termios.ICANON | termios.ECHO | termios.IEXTEN | (termios.ISIG * 1) + ) + raw.cc[termios.VMIN] = 1 + raw.cc[termios.VTIME] = 0 + tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + + self.screen = [] + self.height, self.width = self.getheightwidth() + + self.__buffer = [] + + self.__posxy = 0, 0 + self.__gone_tall = 0 + self.__move = self.__move_short + self.__offset = 0 + + self.__maybe_write_code(self._smkx) + + try: + self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch) + except ValueError: + pass + + def restore(self): + """ + Restore the console to the default state + """ + self.__maybe_write_code(self._rmkx) + self.flushoutput() + tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + + if hasattr(self, "old_sigwinch"): + signal.signal(signal.SIGWINCH, self.old_sigwinch) + del self.old_sigwinch + + def push_char(self, char: int | bytes) -> None: + """ + Push a character to the console event queue. + """ + trace("push char {char!r}", char=char) + self.event_queue.push(char) + + def get_event(self, block: bool = True) -> Event | None: + """ + Get an event from the console event queue. + + Parameters: + - block (bool): Whether to block until an event is available. + + Returns: + - Event: Event object from the event queue. + """ + while self.event_queue.empty(): + while True: + try: + self.push_char(os.read(self.input_fd, 1)) + except OSError as err: + if err.errno == errno.EINTR: + if not self.event_queue.empty(): + return self.event_queue.get() + else: + continue + else: + raise + else: + break + if not block: + break + return self.event_queue.get() + + def wait(self): + """ + Wait for events on the console. + """ + self.pollob.poll() + + def set_cursor_vis(self, visible): + """ + Set the visibility of the cursor. + + Parameters: + - visible (bool): Visibility flag. + """ + if visible: + self.__show_cursor() + else: + self.__hide_cursor() + + if TIOCGWINSZ: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + height, width = struct.unpack( + "hhhh", ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8) + )[0:2] + if not height: + return 25, 80 + return height, width + + else: + + def getheightwidth(self): + """ + Get the height and width of the console. + + Returns: + - tuple: Height and width of the console. + """ + try: + return int(os.environ["LINES"]), int(os.environ["COLUMNS"]) + except KeyError: + return 25, 80 + + def forgetinput(self): + """ + Discard any pending input on the console. + """ + termios.tcflush(self.input_fd, termios.TCIFLUSH) + + def flushoutput(self): + """ + Flush the output buffer. + """ + for text, iscode in self.__buffer: + if iscode: + self.__tputs(text) + else: + os.write(self.output_fd, text.encode(self.encoding, "replace")) + del self.__buffer[:] + + def finish(self): + """ + Finish console operations and flush the output buffer. + """ + y = len(self.screen) - 1 + while y >= 0 and not self.screen[y]: + y -= 1 + self.__move(0, min(y, self.height + self.__offset - 1)) + self.__write("\n\r") + self.flushoutput() + + def beep(self): + """ + Emit a beep sound. + """ + self.__maybe_write_code(self._bel) + self.flushoutput() + + if FIONREAD: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0] + raw = os.read(self.input_fd, amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + else: + + def getpending(self): + """ + Get pending events from the console event queue. + + Returns: + - Event: Pending event from the event queue. + """ + e = Event("key", "", b"") + + while not self.event_queue.empty(): + e2 = self.event_queue.get() + e.data += e2.data + e.raw += e.raw + + amount = 10000 + raw = os.read(self.input_fd, amount) + data = str(raw, self.encoding, "replace") + e.data += data + e.raw += raw + return e + + def clear(self): + """ + Clear the console screen. + """ + self.__write_code(self._clear) + self.__gone_tall = 1 + self.__move = self.__move_tall + self.__posxy = 0, 0 + self.screen = [] + + def __setup_movement(self): + """ + Set up the movement functions based on the terminal capabilities. + """ + if 0 and self._hpa: # hpa don't work in windows telnet :-( + self.__move_x = self.__move_x_hpa + elif self._cub and self._cuf: + self.__move_x = self.__move_x_cub_cuf + elif self._cub1 and self._cuf1: + self.__move_x = self.__move_x_cub1_cuf1 + else: + raise RuntimeError("insufficient terminal (horizontal)") + + if self._cuu and self._cud: + self.__move_y = self.__move_y_cuu_cud + elif self._cuu1 and self._cud1: + self.__move_y = self.__move_y_cuu1_cud1 + else: + raise RuntimeError("insufficient terminal (vertical)") + + if self._dch1: + self.dch1 = self._dch1 + elif self._dch: + self.dch1 = curses.tparm(self._dch, 1) + else: + self.dch1 = None + + if self._ich1: + self.ich1 = self._ich1 + elif self._ich: + self.ich1 = curses.tparm(self._ich, 1) + else: + self.ich1 = None + + self.__move = self.__move_short + + def __write_changed_line(self, y, oldline, newline, px_coord): + # this is frustrating; there's no reason to test (say) + # self.dch1 inside the loop -- but alternative ways of + # structuring this function are equally painful (I'm trying to + # avoid writing code generators these days...) + minlen = min(wlen(oldline), wlen(newline)) + x_pos = 0 + x_coord = 0 + + px_pos = 0 + j = 0 + for c in oldline: + if j >= px_coord: break + j += wlen(c) + px_pos += 1 + + # reuse the oldline as much as possible, but stop as soon as we + # encounter an ESCAPE, because it might be the start of an escape + # sequene + while x_coord < minlen and oldline[x_pos] == newline[x_pos] and newline[x_pos] != "\x1b": + x_coord += wlen(newline[x_pos]) + x_pos += 1 + + # if we need to insert a single character right after the first detected change + if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1: + if ( + y == self.__posxy[1] + and x_coord > self.__posxy[0] + and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1] + ): + x_pos = px_pos + x_coord = px_coord + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if it's a single character change in the middle of the line + elif x_coord < minlen and oldline[x_pos + 1 :] == newline[x_pos + 1 :] and wlen(oldline[x_pos]) == wlen(newline[x_pos]): + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write(newline[x_pos]) + self.__posxy = x_coord + character_width, y + + # if this is the last character to fit in the line and we edit in the middle of the line + elif ( + self.dch1 + and self.ich1 + and wlen(newline) == self.width + and x_coord < wlen(newline) - 2 + and newline[x_pos + 1 : -1] == oldline[x_pos:-2] + ): + self.__hide_cursor() + self.__move(self.width - 2, y) + self.__posxy = self.width - 2, y + self.__write_code(self.dch1) + + character_width = wlen(newline[x_pos]) + self.__move(x_coord, y) + self.__write_code(self.ich1) + self.__write(newline[x_pos]) + self.__posxy = character_width + 1, y + + else: + self.__hide_cursor() + self.__move(x_coord, y) + if wlen(oldline) > wlen(newline): + self.__write_code(self._el) + self.__write(newline[x_pos:]) + self.__posxy = wlen(newline), y + + if "\x1b" in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) + + def __write(self, text): + self.__buffer.append((text, 0)) + + def __write_code(self, fmt, *args): + self.__buffer.append((curses.tparm(fmt, *args), 1)) + + def __maybe_write_code(self, fmt, *args): + if fmt: + self.__write_code(fmt, *args) + + def __move_y_cuu1_cud1(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(dy * self._cud1) + elif dy < 0: + self.__write_code((-dy) * self._cuu1) + + def __move_y_cuu_cud(self, y): + dy = y - self.__posxy[1] + if dy > 0: + self.__write_code(self._cud, dy) + elif dy < 0: + self.__write_code(self._cuu, -dy) + + def __move_x_hpa(self, x): + if x != self.__posxy[0]: + self.__write_code(self._hpa, x) + + def __move_x_cub1_cuf1(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf1 * dx) + elif dx < 0: + self.__write_code(self._cub1 * (-dx)) + + def __move_x_cub_cuf(self, x): + dx = x - self.__posxy[0] + if dx > 0: + self.__write_code(self._cuf, dx) + elif dx < 0: + self.__write_code(self._cub, -dx) + + def __move_short(self, x, y): + self.__move_x(x) + self.__move_y(y) + + def __move_tall(self, x, y): + assert 0 <= y - self.__offset < self.height, y - self.__offset + self.__write_code(self._cup, y - self.__offset, x) + + def __sigwinch(self, signum, frame): + self.height, self.width = self.getheightwidth() + self.event_queue.insert(Event("resize", None)) + + def __hide_cursor(self): + if self.cursor_visible: + self.__maybe_write_code(self._civis) + self.cursor_visible = 0 + + def __show_cursor(self): + if not self.cursor_visible: + self.__maybe_write_code(self._cnorm) + self.cursor_visible = 1 + + def repaint(self): + if not self.__gone_tall: + self.__posxy = 0, self.__posxy[1] + self.__write("\r") + ns = len(self.screen) * ["\000" * self.width] + self.screen = ns + else: + self.__posxy = 0, self.__offset + self.__move(0, self.__offset) + ns = self.height * ["\000" * self.width] + self.screen = ns + + def __tputs(self, fmt, prog=delayprog): + """A Python implementation of the curses tputs function; the + curses one can't really be wrapped in a sane manner. + + I have the strong suspicion that this is complexity that + will never do anyone any good.""" + # using .get() means that things will blow up + # only if the bps is actually needed (which I'm + # betting is pretty unlkely) + bps = ratedict.get(self.__svtermstate.ospeed) + while 1: + m = prog.search(fmt) + if not m: + os.write(self.output_fd, fmt) + break + x, y = m.span() + os.write(self.output_fd, fmt[:x]) + fmt = fmt[y:] + delay = int(m.group(1)) + if b"*" in m.group(2): + delay *= self.height + if self._pad and bps is not None: + nchars = (bps * delay) / 1000 + os.write(self.output_fd, self._pad * nchars) + else: + time.sleep(float(delay) / 1000.0) diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py new file mode 100644 index 000000000000000..70cfade26e23b1b --- /dev/null +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -0,0 +1,152 @@ +# Copyright 2000-2008 Michael Hudson-Doyle +# Armin Rigo +# +# All Rights Reserved +# +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose is hereby granted without fee, +# provided that the above copyright notice appear in all copies and +# that both that copyright notice and this permission notice appear in +# supporting documentation. +# +# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, +# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER +# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +from collections import deque + +from . import keymap +from .console import Event +from . import curses +from .trace import trace +from termios import tcgetattr, VERASE +import os + + +# Mapping of human-readable key names to their terminal-specific codes +TERMINAL_KEYNAMES = { + "delete": "kdch1", + "down": "kcud1", + "end": "kend", + "enter": "kent", + "home": "khome", + "insert": "kich1", + "left": "kcub1", + "page down": "knp", + "page up": "kpp", + "right": "kcuf1", + "up": "kcuu1", +} + + +# Function keys F1-F20 mapping +TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21)) + +# Known CTRL-arrow keycodes +CTRL_ARROW_KEYCODES= { + # for xterm, gnome-terminal, xfce terminal, etc. + b'\033[1;5D': 'ctrl left', + b'\033[1;5C': 'ctrl right', + # for rxvt + b'\033Od': 'ctrl left', + b'\033Oc': 'ctrl right', +} + +def get_terminal_keycodes() -> dict[bytes, str]: + """ + Generates a dictionary mapping terminal keycodes to human-readable names. + """ + keycodes = {} + for key, terminal_code in TERMINAL_KEYNAMES.items(): + keycode = curses.tigetstr(terminal_code) + trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) + if keycode: + keycodes[keycode] = key + keycodes.update(CTRL_ARROW_KEYCODES) + return keycodes + +class EventQueue: + def __init__(self, fd: int, encoding: str) -> None: + self.keycodes = get_terminal_keycodes() + if os.isatty(fd): + backspace = tcgetattr(fd)[6][VERASE] + self.keycodes[backspace] = "backspace" + self.compiled_keymap = keymap.compile_keymap(self.keycodes) + self.keymap = self.compiled_keymap + trace("keymap {k!r}", k=self.keymap) + self.encoding = encoding + self.events: deque[Event] = deque() + self.buf = bytearray() + + def get(self) -> Event | None: + """ + Retrieves the next event from the queue. + """ + if self.events: + return self.events.popleft() + else: + return None + + def empty(self) -> bool: + """ + Checks if the queue is empty. + """ + return not self.events + + def flush_buf(self) -> bytearray: + """ + Flushes the buffer and returns its contents. + """ + old = self.buf + self.buf = bytearray() + return old + + def insert(self, event: Event) -> None: + """ + Inserts an event into the queue. + """ + trace('added event {event}', event=event) + self.events.append(event) + + def push(self, char: int | bytes) -> None: + """ + Processes a character by updating the buffer and handling special key mappings. + """ + ord_char = char if isinstance(char, int) else ord(char) + char = bytes(bytearray((ord_char,))) + self.buf.append(ord_char) + if char in self.keymap: + if self.keymap is self.compiled_keymap: + #sanity check, buffer is empty when a special key comes + assert len(self.buf) == 1 + k = self.keymap[char] + trace('found map {k!r}', k=k) + if isinstance(k, dict): + self.keymap = k + else: + self.insert(Event('key', k, self.flush_buf())) + self.keymap = self.compiled_keymap + + elif self.buf and self.buf[0] == 27: # escape + # escape sequence not recognized by our keymap: propagate it + # outside so that i can be recognized as an M-... key (see also + # the docstring in keymap.py + trace('unrecognized escape sequence, propagating...') + self.keymap = self.compiled_keymap + self.insert(Event('key', '\033', bytearray(b'\033'))) + for _c in self.flush_buf()[1:]: + self.push(_c) + + else: + try: + decoded = bytes(self.buf).decode(self.encoding) + except UnicodeError: + return + else: + self.insert(Event('key', decoded, self.flush_buf())) + self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py new file mode 100644 index 000000000000000..cd1df7c49a216dd --- /dev/null +++ b/Lib/_pyrepl/utils.py @@ -0,0 +1,18 @@ +import re +import unicodedata + +ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]") + + +def str_width(c: str) -> int: + w = unicodedata.east_asian_width(c) + if w in ('N', 'Na', 'H', 'A'): + return 1 + return 2 + + +def wlen(s: str) -> int: + length = sum(str_width(i) for i in s) + + # remove lengths of any escape sequences + return length - sum(len(i) for i in ANSI_ESCAPE_SEQUENCE.findall(s)) diff --git a/Lib/code.py b/Lib/code.py index f4aecddeca7813b..1ee1ad62ff4506e 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -130,7 +130,7 @@ def showsyntaxerror(self, filename=None): # over self.write sys.excepthook(type, value, tb) - def showtraceback(self): + def showtraceback(self, **kwargs): """Display the exception that just occurred. We remove the first stack item because it is our own code. @@ -138,11 +138,12 @@ def showtraceback(self): The output is written by self.write(), below. """ + colorize = kwargs.pop('colorize', False) sys.last_type, sys.last_value, last_tb = ei = sys.exc_info() sys.last_traceback = last_tb sys.last_exc = ei[1] try: - lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next) + lines = traceback.format_exception(ei[0], ei[1], last_tb.tb_next, colorize=colorize) if sys.excepthook is sys.__excepthook__: self.write(''.join(lines)) else: @@ -170,7 +171,7 @@ class InteractiveConsole(InteractiveInterpreter): """ - def __init__(self, locals=None, filename="", local_exit=False): + def __init__(self, locals=None, filename="", *, local_exit=False): """Constructor. The optional locals argument will be passed to the @@ -280,7 +281,7 @@ def interact(self, banner=None, exitmsg=None): elif exitmsg != '': self.write('%s\n' % exitmsg) - def push(self, line): + def push(self, line, filename=None): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -296,7 +297,9 @@ def push(self, line): """ self.buffer.append(line) source = "\n".join(self.buffer) - more = self.runsource(source, self.filename) + if filename is None: + filename = self.filename + more = self.runsource(source, filename) if not more: self.resetbuffer() return more diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 02af672a628f5ab..eaaf8249b205bc5 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -76,6 +76,18 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only +from _pyrepl.pager import (get_pager, plain, escape_less, pipe_pager, + plain_pager, tempfile_pager, tty_pager) + + +# --------------------------------------------------------- old names + +getpager = get_pager +pipepager = pipe_pager +plainpager = plain_pager +tempfilepager = tempfile_pager +ttypager = tty_pager + # --------------------------------------------------------- common routines @@ -1640,153 +1652,9 @@ def bold(self, text): def pager(text, title=''): """The first time this is called, determine what kind of pager to use.""" global pager - pager = getpager() + pager = get_pager() pager(text, title) -def getpager(): - """Decide what method to use for paging through text.""" - if not hasattr(sys.stdin, "isatty"): - return plainpager - if not hasattr(sys.stdout, "isatty"): - return plainpager - if not sys.stdin.isatty() or not sys.stdout.isatty(): - return plainpager - if sys.platform == "emscripten": - return plainpager - use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') - if use_pager: - if sys.platform == 'win32': # pipes completely broken in Windows - return lambda text, title='': tempfilepager(plain(text), use_pager) - elif os.environ.get('TERM') in ('dumb', 'emacs'): - return lambda text, title='': pipepager(plain(text), use_pager, title) - else: - return lambda text, title='': pipepager(text, use_pager, title) - if os.environ.get('TERM') in ('dumb', 'emacs'): - return plainpager - if sys.platform == 'win32': - return lambda text, title='': tempfilepager(plain(text), 'more <') - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return lambda text, title='': pipepager(text, 'less', title) - - import tempfile - (fd, filename) = tempfile.mkstemp() - os.close(fd) - try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return lambda text, title='': pipepager(text, 'more', title) - else: - return ttypager - finally: - os.unlink(filename) - -def plain(text): - """Remove boldface formatting from text.""" - return re.sub('.\b', '', text) - -def escape_less(s): - return re.sub(r'([?:.%\\])', r'\\\1', s) - -def pipepager(text, cmd, title=''): - """Page through text by feeding it to another program.""" - import subprocess - env = os.environ.copy() - if title: - title += ' ' - esc_title = escape_less(title) - prompt_string = ( - f' {esc_title}' + - '?ltline %lt?L/%L.' - ':byte %bB?s/%s.' - '.' - '?e (END):?pB %pB\\%..' - ' (press h for help or q to quit)') - env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string) - proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - errors='backslashreplace', env=env) - try: - with proc.stdin as pipe: - try: - pipe.write(text) - except KeyboardInterrupt: - # We've hereby abandoned whatever text hasn't been written, - # but the pager is still in control of the terminal. - pass - except OSError: - pass # Ignore broken pipes caused by quitting the pager program. - while True: - try: - proc.wait() - break - except KeyboardInterrupt: - # Ignore ctl-c like the pager itself does. Otherwise the pager is - # left running and the terminal is in raw mode and unusable. - pass - -def tempfilepager(text, cmd, title=''): - """Page through text by invoking a program on a temporary file.""" - import tempfile - with tempfile.TemporaryDirectory() as tempdir: - filename = os.path.join(tempdir, 'pydoc.out') - with open(filename, 'w', errors='backslashreplace', - encoding=os.device_encoding(0) if - sys.platform == 'win32' else None - ) as file: - file.write(text) - os.system(cmd + ' "' + filename + '"') - -def _escape_stdout(text): - # Escape non-encodable characters to avoid encoding errors later - encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' - return text.encode(encoding, 'backslashreplace').decode(encoding) - -def ttypager(text, title=''): - """Page through text on a text terminal.""" - lines = plain(_escape_stdout(text)).split('\n') - try: - import tty - fd = sys.stdin.fileno() - old = tty.tcgetattr(fd) - tty.setcbreak(fd) - getchar = lambda: sys.stdin.read(1) - except (ImportError, AttributeError, io.UnsupportedOperation): - tty = None - getchar = lambda: sys.stdin.readline()[:-1][:1] - - try: - try: - h = int(os.environ.get('LINES', 0)) - except ValueError: - h = 0 - if h <= 1: - h = 25 - r = inc = h - 1 - sys.stdout.write('\n'.join(lines[:inc]) + '\n') - while lines[r:]: - sys.stdout.write('-- more --') - sys.stdout.flush() - c = getchar() - - if c in ('q', 'Q'): - sys.stdout.write('\r \r') - break - elif c in ('\r', '\n'): - sys.stdout.write('\r \r' + lines[r] + '\n') - r = r + 1 - continue - if c in ('b', 'B', '\x1b'): - r = r - inc - inc - if r < 0: r = 0 - sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n') - r = r + inc - - finally: - if tty: - tty.tcsetattr(fd, tty.TCSAFLUSH, old) - -def plainpager(text, title=''): - """Simply print unformatted text. This is the ultimate fallback.""" - sys.stdout.write(plain(_escape_stdout(text))) - def describe(thing): """Produce a short description of the given thing.""" if inspect.ismodule(thing): diff --git a/Lib/site.py b/Lib/site.py index 93af9c453ac7bb2..b63447d6673f68e 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -485,6 +485,8 @@ def register_readline(): try: import readline import rlcompleter + import _pyrepl.readline + import _pyrepl.unix_console except ImportError: return @@ -513,13 +515,19 @@ def register_readline(): # http://bugs.python.org/issue5845#msg198636 history = gethistoryfile() try: - readline.read_history_file(history) - except OSError: + if os.getenv("PYTHON_BASIC_REPL"): + readline.read_history_file(history) + else: + _pyrepl.readline.read_history_file(history) + except (OSError,* _pyrepl.unix_console._error): pass def write_history(): try: - readline.write_history_file(history) + if os.getenv("PYTHON_BASIC_REPL"): + readline.write_history_file(history) + else: + _pyrepl.readline.write_history_file(history) except (FileNotFoundError, PermissionError): # home directory does not exist or is not writable # https://bugs.python.org/issue19891 diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py new file mode 100644 index 000000000000000..c53bdef0ca9cb19 --- /dev/null +++ b/Lib/test/test_pyrepl.py @@ -0,0 +1,929 @@ +import itertools +import os +import rlcompleter +import sys +import unittest +from code import InteractiveConsole +from functools import partial +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from test.support import requires +from test.support.import_helper import import_module + +# Optionally test pyrepl. This currently requires that the +# 'curses' resource be given on the regrtest command line using the -u +# option. Additionally, we need to attempt to import curses and readline. +requires('curses') +curses = import_module('curses') +readline = import_module('readline') + +from _pyrepl.console import Console, Event +from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig +from _pyrepl.simple_interact import _strip_final_indent +from _pyrepl.unix_eventqueue import EventQueue + + +def more_lines(unicodetext, namespace=None): + if namespace is None: + namespace = {} + src = _strip_final_indent(unicodetext) + console = InteractiveConsole(namespace, filename="") + try: + code = console.compile(src, "", "single") + except (OverflowError, SyntaxError, ValueError): + return False + else: + return code is None + + +def multiline_input(reader, namespace=None): + saved = reader.more_lines + try: + reader.more_lines = partial(more_lines, namespace=namespace) + reader.ps1 = reader.ps2 = ">>>" + reader.ps3 = reader.ps4 = "..." + return reader.readline() + finally: + reader.more_lines = saved + reader.paste_mode = False + + +def code_to_events(code): + for c in code: + yield Event(evt="key", data=c, raw=bytearray(c.encode("utf-8"))) + + +def prepare_mock_console(events, **kwargs): + console = MagicMock() + console.get_event.side_effect = events + console.height = 100 + console.width = 80 + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def prepare_fake_console(**kwargs): + console = FakeConsole() + for key, val in kwargs.items(): + setattr(console, key, val) + return console + + +def prepare_reader(console, **kwargs): + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + reader.more_lines = partial(more_lines, namespace=None) + reader.paste_mode = True # Avoid extra indents + + def get_prompt(lineno, cursor_on_line) -> str: + return "" + + reader.get_prompt = get_prompt # Remove prompt for easier calculations of (x, y) + + for key, val in kwargs.items(): + setattr(reader, key, val) + + return reader + + +def handle_all_events( + events, prepare_console=prepare_mock_console, prepare_reader=prepare_reader +): + console = prepare_console(events) + reader = prepare_reader(console) + try: + while True: + reader.handle1() + except StopIteration: + pass + return reader, console + + +handle_events_narrow_console = partial( + handle_all_events, prepare_console=partial(prepare_mock_console, width=10) +) + + +class FakeConsole(Console): + def __init__(self, events, encoding="utf-8"): + self.events = iter(events) + self.encoding = encoding + self.screen = [] + self.height = 100 + self.width = 80 + + def get_event(self, block: bool = True) -> Event | None: + return next(self.events) + + def getpending(self) -> Event: + return self.get_event(block=False) + + def getheightwidth(self) -> tuple[int, int]: + return self.height, self.width + + def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: + pass + + def prepare(self) -> None: + pass + + def restore(self) -> None: + pass + + def move_cursor(self, x: int, y: int) -> None: + pass + + def set_cursor_vis(self, visible: bool) -> None: + pass + + def push_char(self, char: int | bytes) -> None: + pass + + def beep(self) -> None: + pass + + def clear(self) -> None: + pass + + def finish(self) -> None: + pass + + def flushoutput(self) -> None: + pass + + def forgetinput(self) -> None: + pass + + def wait(self) -> None: + pass + + def repaint(self) -> None: + pass + + +class TestCursorPosition(TestCase): + def test_up_arrow_simple(self): + # fmt: off + code = ( + 'def f():\n' + ' ...\n' + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 1)) + console.move_cursor.assert_called_once_with(0, 1) + + def test_down_arrow_end_of_input(self): + # fmt: off + code = ( + 'def f():\n' + ' ...\n' + ) + # fmt: on + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (0, 2)) + console.move_cursor.assert_called_once_with(0, 2) + + def test_left_arrow_simple(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (4, 0)) + console.move_cursor.assert_called_once_with(4, 0) + + def test_right_arrow_end_of_line(self): + events = itertools.chain( + code_to_events("11+11"), + [ + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, console = handle_all_events(events) + self.assertEqual(reader.cxy, (5, 0)) + console.move_cursor.assert_called_once_with(5, 0) + + def test_cursor_position_simple_character(self): + events = itertools.chain(code_to_events("k")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 1 for simple character + self.assertEqual(reader.cxy, (1, 0)) + + def test_cursor_position_double_width_character(self): + events = itertools.chain(code_to_events("樂")) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_character_move_left(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 0) + self.assertEqual(reader.cxy, (0, 0)) + + def test_cursor_position_double_width_character_move_left_right(self): + events = itertools.chain( + code_to_events("樂"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 1) + + # 2 for wide character + self.assertEqual(reader.cxy, (2, 0)) + + def test_cursor_position_double_width_characters_move_up(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor at end of first line + self.assertEqual(reader.pos, len(for_loop)) + self.assertEqual(reader.cxy, (len(for_loop), 0)) + + def test_cursor_position_double_width_characters_move_up_down(self): + for_loop = "for _ in _:" + + # fmt: off + code = ( + f"{for_loop}\n" + " ' 可口可乐; 可口可樂'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # cursor here (showing 2nd line only): + # < ' 可口可乐; 可口可樂'> + # ^ + self.assertEqual(reader.pos, 19) + self.assertEqual(reader.cxy, (10, 1)) + + def test_cursor_position_multiple_double_width_characters_move_left(self): + events = itertools.chain( + code_to_events("' 可口可乐; 可口可樂'"), + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")), + ], + ) + + reader, _ = handle_all_events(events) + self.assertEqual(reader.pos, 10) + + # 1 for quote, 1 for space, 2 per wide character, + # 1 for semicolon, 1 for space, 2 per wide character + self.assertEqual(reader.cxy, (16, 0)) + + def test_cursor_position_move_up_to_eol(self): + first_line = "for _ in _:" + second_line = " hello" + + # fmt: off + code = ( + f"{first_line}\n" + f"{second_line}\n" + " h\n" + " hel" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 1, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual( + reader.pos, len(first_line) + len(second_line) + 1 + ) # +1 for newline + self.assertEqual(reader.cxy, (len(second_line), 1)) + + def test_cursor_position_move_down_to_eol(self): + last_line = " hel" + + # fmt: off + code = ( + "for _ in _:\n" + " hello\n" + " h\n" + f"{last_line}" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader, _ = handle_all_events(events) + + # Cursor should be at end of line 3, even though line 2 is shorter + # for _ in _: + # hello + # h + # hel + self.assertEqual(reader.pos, len(code)) + self.assertEqual(reader.cxy, (len(last_line), 3)) + + def test_cursor_position_multiple_mixed_lines_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " x = '可口可乐; 可口可樂'\n" + " y = 'abckdfjskldfjslkdjf'" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + 13 * [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], + ) + + reader, _ = handle_all_events(events) + + # By moving left, we're before the s: + # y = 'abckdfjskldfjslkdjf' + # ^ + # And we should move before the semi-colon despite the different offset + # x = '可口可乐; 可口可樂' + # ^ + self.assertEqual(reader.pos, 22) + self.assertEqual(reader.cxy, (15, 1)) + + def test_cursor_position_after_wrap_and_move_up(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = itertools.chain( + code_to_events(code), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + ], + ) + reader, _ = handle_events_narrow_console(events) + + # The code looks like this: + # def foo()\ + # : + # hello + # After moving up we should be after the colon in line 2 + self.assertEqual(reader.pos, 10) + self.assertEqual(reader.cxy, (1, 1)) + + +class TestPyReplOutput(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_basic(self): + reader = self.prepare_reader(code_to_events("1+1\n")) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_multiline_edit(self): + events = itertools.chain( + code_to_events("def f():\n ...\n\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + Event(evt="key", data="g", raw=bytearray(b"g")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "def f():\n ...\n ") + output = multiline_input(reader) + self.assertEqual(output, "def g():\n ...\n ") + + def test_history_navigation_with_up_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_navigation_with_down_arrow(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n"), + [ + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="up", raw=bytearray(b"\x1bOA")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_history_search(self): + events = itertools.chain( + code_to_events("1+1\n2+2\n3+3\n"), + [ + Event(evt="key", data="\x12", raw=bytearray(b"\x12")), + Event(evt="key", data="1", raw=bytearray(b"1")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + Event(evt="key", data="\n", raw=bytearray(b"\n")), + ], + ) + + reader = self.prepare_reader(events) + + output = multiline_input(reader) + self.assertEqual(output, "1+1") + output = multiline_input(reader) + self.assertEqual(output, "2+2") + output = multiline_input(reader) + self.assertEqual(output, "3+3") + output = multiline_input(reader) + self.assertEqual(output, "1+1") + + def test_control_character(self): + events = code_to_events("c\x1d\n") + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, "c\x1d") + + +class TestPyReplCompleter(TestCase): + def prepare_reader(self, events, namespace): + console = FakeConsole(events) + config = ReadlineConfig() + config.readline_completer = rlcompleter.Completer(namespace).complete + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_simple_completion(self): + events = code_to_events("os.geten\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.getenv") + + def test_completion_with_many_options(self): + events = code_to_events("os.\t\tO_AS\t\n") + + namespace = {"os": os} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.O_ASYNC") + + def test_empty_namespace_completion(self): + events = code_to_events("os.geten\t\n") + namespace = {} + reader = self.prepare_reader(events, namespace) + + output = multiline_input(reader, namespace) + self.assertEqual(output, "os.geten") + + def test_global_namespace_completion(self): + events = code_to_events("py\t\n") + namespace = {"python": None} + reader = self.prepare_reader(events, namespace) + output = multiline_input(reader, namespace) + self.assertEqual(output, "python") + + +@patch("_pyrepl.curses.tigetstr", lambda x: b"") +class TestUnivEventQueue(TestCase): + def test_get(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.get(), event) + + def test_empty(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + self.assertTrue(eq.empty()) + eq.insert(Event("key", "a", b"a")) + self.assertFalse(eq.empty()) + + def test_flush_buf(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.buf.extend(b"test") + self.assertEqual(eq.flush_buf(), b"test") + self.assertEqual(eq.buf, bytearray()) + + def test_insert(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + event = Event("key", "a", b"a") + eq.insert(event) + self.assertEqual(eq.events[0], event) + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"a": "b"} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "b") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_without_key_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"c": "d"} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "a") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertTrue(eq.empty()) + eq.push("b") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "c") + eq.push("d") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "d") + + @patch("_pyrepl.unix_eventqueue.keymap") + def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): + mock_keymap.compile_keymap.return_value = {"a": "b"} + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {b"a": {b"b": "c"}} + eq.push("a") + self.assertTrue(mock_keymap.compile_keymap.called) + self.assertTrue(eq.empty()) + eq.flush_buf() + eq.push("\033") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\033") + eq.push("b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "b") + + def test_push_special_key(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("A") + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + + def test_push_unrecognized_escape_sequence(self): + eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq.keymap = {} + eq.push("\x1b") + eq.push("[") + eq.push("Z") + self.assertEqual(len(eq.events), 3) + self.assertEqual(eq.events[0].evt, "key") + self.assertEqual(eq.events[0].data, "\x1b") + self.assertEqual(eq.events[1].evt, "key") + self.assertEqual(eq.events[1].data, "[") + self.assertEqual(eq.events[2].evt, "key") + self.assertEqual(eq.events[2].data, "Z") + + +class TestPasteEvent(TestCase): + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_paste(self): + # fmt: off + code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n' + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines(self): + # fmt: off + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n' + ) + # fmt: on + + events = itertools.chain( + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events(code), + [ + Event(evt="key", data="f3", raw=bytearray(b"\x1bOR")), + ], + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, code) + + def test_paste_mid_newlines_not_in_paste_mode(self): + # fmt: off + code = ( + 'def f():\n' + ' x = y\n' + ' \n' + ' y = z\n\n' + ) + + expected = ( + 'def f():\n' + ' x = y\n' + ' ' + ) + # fmt: on + + events = code_to_events(code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, expected) + + def test_paste_not_in_paste_mode(self): + # fmt: off + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:\n' + ' pass\n\n' + ) + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + ' if x%2:\n' + ' print(x)\n' + ' else:' + ) + # fmt: on + + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + + +class TestReader(TestCase): + def assert_screen_equals(self, reader, expected): + actual = reader.calc_screen() + expected = expected.split("\n") + self.assertListEqual(actual, expected) + + def test_calc_screen_wrap_simple(self): + events = code_to_events(10 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_calc_screen_wrap_wide_characters(self): + events = code_to_events(8 * "a" + "樂") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{8*"a"}\\\n樂") + + def test_calc_screen_wrap_three_lines(self): + events = code_to_events(20 * "a") + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\n{9*"a"}\\\naa") + + def test_calc_screen_wrap_three_lines_mixed_character(self): + # fmt: off + code = ( + "def f():\n" + f" {8*"a"}\n" + f" {5*"樂"}" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + + # fmt: off + self.assert_screen_equals(reader, ( + "def f():\n" + f" {7*"a"}\\\n" + "a\n" + f" {3*"樂"}\\\n" + "樂樂" + )) + # fmt: on + + def test_calc_screen_backspace(self): + events = itertools.chain( + code_to_events("aaa"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "aa") + + def test_calc_screen_wrap_removes_after_backspace(self): + events = itertools.chain( + code_to_events(10 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, 9 * "a") + + def test_calc_screen_backspace_in_second_line_after_wrap(self): + events = itertools.chain( + code_to_events(11 * "a"), + [ + Event(evt="key", data="backspace", raw=bytearray(b"\x7f")), + ], + ) + reader, _ = handle_events_narrow_console(events) + self.assert_screen_equals(reader, f"{9*"a"}\\\na") + + def test_setpos_for_xy_simple(self): + events = code_to_events("11+11") + reader, _ = handle_all_events(events) + reader.setpos_from_xy(0, 0) + self.assertEqual(reader.pos, 0) + + def test_setpos_from_xy_multiple_lines(self): + # fmt: off + code = ( + "def foo():\n" + " return 1" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_all_events(events) + reader.setpos_from_xy(2, 1) + self.assertEqual(reader.pos, 13) + + def test_setpos_from_xy_after_wrap(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(2, 2) + self.assertEqual(reader.pos, 13) + + def test_setpos_fromxy_in_wrapped_line(self): + # fmt: off + code = ( + "def foo():\n" + " hello" + ) + # fmt: on + + events = code_to_events(code) + reader, _ = handle_events_narrow_console(events) + reader.setpos_from_xy(0, 1) + self.assertEqual(reader.pos, 9) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 8969e0174c98c1e..7e4851058e0109d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -500,7 +500,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -539,7 +539,7 @@ def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=, tb=, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), diff --git a/Lib/traceback.py b/Lib/traceback.py index 8403173ade7b6c9..1878779e154d048 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -140,7 +140,7 @@ def _print_exception_bltin(exc, /): def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ - chain=True): + chain=True, **kwargs): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -149,9 +149,10 @@ def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ + colorize = kwargs.get("colorize", False) value, tb = _parse_value_tb(exc, value, tb) te = TracebackException(type(value), value, tb, limit=limit, compact=True) - return list(te.format(chain=chain)) + return list(te.format(chain=chain, colorize=colorize)) def format_exception_only(exc, /, value=_sentinel, *, show_group=False): diff --git a/Makefile.pre.in b/Makefile.pre.in index 924ed1f5621255d..903227709c79ec2 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2339,6 +2339,7 @@ LIBSUBDIRS= asyncio \ xmlrpc \ zipfile zipfile/_path \ zoneinfo \ + _pyrepl \ __phello__ TESTSUBDIRS= idlelib/idle_test \ test \ diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst new file mode 100644 index 000000000000000..9176da155d48fbe --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst @@ -0,0 +1,4 @@ +The :term:`interactive` interpreter is now implemented in Python, which +allows for a number of new features like colors, multiline input, history +viewing, and paste mode. Contributed by Pablo Galindo and Łukasz Langa based +on code from the PyPy project. diff --git a/Modules/main.c b/Modules/main.c index df2ce5502450889..8eded2639ad90a2 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -513,8 +513,13 @@ pymain_run_stdin(PyConfig *config) return pymain_exit_err_print(); } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + if (!isatty(fileno(stdin)) + || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + return (run != 0); + } + int run = pymain_run_module(L"_pyrepl", 0); return (run != 0); } @@ -537,9 +542,15 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int res = PyRun_AnyFileFlags(stdin, "", &cf); - *exitcode = (res != 0); + if (!isatty(fileno(stdin))) { + PyCompilerFlags cf = _PyCompilerFlags_INIT; + int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); + *exitcode = (run != 0); + return; + } + int run = pymain_run_module(L"_pyrepl", 0); + *exitcode = (run != 0); + return; } diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 0a8704c71e488cb..56a831eb2ea06ec 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1485,6 +1485,24 @@ sys__get_cpu_count_config(PyObject *module, PyObject *Py_UNUSED(ignored)) return return_value; } +PyDoc_STRVAR(sys__baserepl__doc__, +"_baserepl($module, /)\n" +"--\n" +"\n" +"Private function for getting the base REPL"); + +#define SYS__BASEREPL_METHODDEF \ + {"_baserepl", (PyCFunction)sys__baserepl, METH_NOARGS, sys__baserepl__doc__}, + +static PyObject * +sys__baserepl_impl(PyObject *module); + +static PyObject * +sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__baserepl_impl(module); +} + PyDoc_STRVAR(sys__is_gil_enabled__doc__, "_is_gil_enabled($module, /)\n" "--\n" @@ -1556,4 +1574,4 @@ sys__is_gil_enabled(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=352ac7a0085e8a1f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ef7c35945443d300 input=a9049054013a1b77]*/ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 31213aec3cd9c30..ce7f194e929c9c9 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -83,8 +83,6 @@ _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, return res; } - -/* Parse input from a file and execute it */ int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index ba320842173b484..9686d10563aa4d9 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -65,6 +65,7 @@ static const char* _Py_stdlib_module_names[] = { "_pydecimal", "_pyio", "_pylong", +"_pyrepl", "_queue", "_random", "_scproxy", diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 17c4a5f46c09df1..69bee32987fe6bd 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2395,6 +2395,21 @@ sys__get_cpu_count_config_impl(PyObject *module) return config->cpu_count; } +/*[clinic input] +sys._baserepl + +Private function for getting the base REPL +[clinic start generated code]*/ + +static PyObject * +sys__baserepl_impl(PyObject *module) +/*[clinic end generated code: output=f19a36375ebe0a45 input=ade0ebb9fab56f3c]*/ +{ + PyCompilerFlags cf = _PyCompilerFlags_INIT; + PyRun_AnyFileExFlags(stdin, "", 0, &cf); + Py_RETURN_NONE; +} + /*[clinic input] sys._is_gil_enabled -> bool @@ -2579,6 +2594,7 @@ static PyMethodDef sys_methods[] = { SYS_UNRAISABLEHOOK_METHODDEF SYS_GET_INT_MAX_STR_DIGITS_METHODDEF SYS_SET_INT_MAX_STR_DIGITS_METHODDEF + SYS__BASEREPL_METHODDEF #ifdef Py_STATS SYS__STATS_ON_METHODDEF SYS__STATS_OFF_METHODDEF From d8d94911e2393bd30ca58a32b33d792307fdc00d Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sun, 5 May 2024 20:57:19 +0100 Subject: [PATCH 133/220] Move pathlib implementation out of `__init__.py` (#118582) Use the `__init__.py` file only for imports that define the API, following the example of asyncio. --- Lib/pathlib/__init__.py | 865 +--------------------------------------- Lib/pathlib/_abc.py | 32 +- Lib/pathlib/_glob.py | 59 ++- Lib/pathlib/_local.py | 859 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 910 insertions(+), 905 deletions(-) create mode 100644 Lib/pathlib/_local.py diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 381b08cbe8e01f2..4b3edf535a61aaf 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,865 +5,8 @@ operating systems. """ -import io -import ntpath -import operator -import os -import posixpath -import sys -import warnings -from itertools import chain -from _collections_abc import Sequence +from ._abc import * +from ._local import * -try: - import pwd -except ImportError: - pwd = None -try: - import grp -except ImportError: - grp = None - -from . import _abc, _glob - - -__all__ = [ - "UnsupportedOperation", - "PurePath", "PurePosixPath", "PureWindowsPath", - "Path", "PosixPath", "WindowsPath", - ] - - -class _PathParents(Sequence): - """This object provides sequence-like access to the logical ancestors - of a path. Don't try to construct it yourself.""" - __slots__ = ('_path', '_drv', '_root', '_tail') - - def __init__(self, path): - self._path = path - self._drv = path.drive - self._root = path.root - self._tail = path._tail - - def __len__(self): - return len(self._tail) - - def __getitem__(self, idx): - if isinstance(idx, slice): - return tuple(self[i] for i in range(*idx.indices(len(self)))) - - if idx >= len(self) or idx < -len(self): - raise IndexError(idx) - if idx < 0: - idx += len(self) - return self._path._from_parsed_parts(self._drv, self._root, - self._tail[:-idx - 1]) - - def __repr__(self): - return "<{}.parents>".format(type(self._path).__name__) - - -UnsupportedOperation = _abc.UnsupportedOperation - - -class PurePath(_abc.PurePathBase): - """Base class for manipulating paths without I/O. - - PurePath represents a filesystem path and offers operations which - don't imply any actual filesystem I/O. Depending on your system, - instantiating a PurePath will return either a PurePosixPath or a - PureWindowsPath object. You can also instantiate either of these classes - directly, regardless of your system. - """ - - __slots__ = ( - # The `_raw_paths` slot stores unnormalized string paths. This is set - # in the `__init__()` method. - '_raw_paths', - - # The `_drv`, `_root` and `_tail_cached` slots store parsed and - # normalized parts of the path. They are set when any of the `drive`, - # `root` or `_tail` properties are accessed for the first time. The - # three-part division corresponds to the result of - # `os.path.splitroot()`, except that the tail is further split on path - # separators (i.e. it is a list of strings), and that the root and - # tail are normalized. - '_drv', '_root', '_tail_cached', - - # The `_str` slot stores the string representation of the path, - # computed from the drive, root and tail when `__str__()` is called - # for the first time. It's used to implement `_str_normcase` - '_str', - - # The `_str_normcase_cached` slot stores the string path with - # normalized case. It is set when the `_str_normcase` property is - # accessed for the first time. It's used to implement `__eq__()` - # `__hash__()`, and `_parts_normcase` - '_str_normcase_cached', - - # The `_parts_normcase_cached` slot stores the case-normalized - # string path after splitting on path separators. It's set when the - # `_parts_normcase` property is accessed for the first time. It's used - # to implement comparison methods like `__lt__()`. - '_parts_normcase_cached', - - # The `_hash` slot stores the hash of the case-normalized string - # path. It's set when `__hash__()` is called for the first time. - '_hash', - ) - parser = os.path - _globber = _glob.Globber - - def __new__(cls, *args, **kwargs): - """Construct a PurePath from one or several strings and or existing - PurePath objects. The strings and path objects are combined so as - to yield a canonicalized path, which is incorporated into the - new PurePath object. - """ - if cls is PurePath: - cls = PureWindowsPath if os.name == 'nt' else PurePosixPath - return object.__new__(cls) - - def __init__(self, *args): - paths = [] - for arg in args: - if isinstance(arg, PurePath): - if arg.parser is ntpath and self.parser is posixpath: - # GH-103631: Convert separators for backwards compatibility. - paths.extend(path.replace('\\', '/') for path in arg._raw_paths) - else: - paths.extend(arg._raw_paths) - else: - try: - path = os.fspath(arg) - except TypeError: - path = arg - if not isinstance(path, str): - raise TypeError( - "argument should be a str or an os.PathLike " - "object where __fspath__ returns a str, " - f"not {type(path).__name__!r}") - paths.append(path) - # Avoid calling super().__init__, as an optimisation - self._raw_paths = paths - - def joinpath(self, *pathsegments): - """Combine this path with one or several arguments, and return a - new path representing either a subpath (if all arguments are relative - paths) or a totally different path (if one of the arguments is - anchored). - """ - return self.with_segments(self, *pathsegments) - - def __truediv__(self, key): - try: - return self.with_segments(self, key) - except TypeError: - return NotImplemented - - def __rtruediv__(self, key): - try: - return self.with_segments(key, self) - except TypeError: - return NotImplemented - - def __reduce__(self): - return self.__class__, tuple(self._raw_paths) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, self.as_posix()) - - def __fspath__(self): - return str(self) - - def __bytes__(self): - """Return the bytes representation of the path. This is only - recommended to use under Unix.""" - return os.fsencode(self) - - @property - def _str_normcase(self): - # String with normalized case, for hashing and equality checks - try: - return self._str_normcase_cached - except AttributeError: - if _abc._is_case_sensitive(self.parser): - self._str_normcase_cached = str(self) - else: - self._str_normcase_cached = str(self).lower() - return self._str_normcase_cached - - def __hash__(self): - try: - return self._hash - except AttributeError: - self._hash = hash(self._str_normcase) - return self._hash - - def __eq__(self, other): - if not isinstance(other, PurePath): - return NotImplemented - return self._str_normcase == other._str_normcase and self.parser is other.parser - - @property - def _parts_normcase(self): - # Cached parts with normalized case, for comparisons. - try: - return self._parts_normcase_cached - except AttributeError: - self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) - return self._parts_normcase_cached - - def __lt__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase < other._parts_normcase - - def __le__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase <= other._parts_normcase - - def __gt__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase > other._parts_normcase - - def __ge__(self, other): - if not isinstance(other, PurePath) or self.parser is not other.parser: - return NotImplemented - return self._parts_normcase >= other._parts_normcase - - def __str__(self): - """Return the string representation of the path, suitable for - passing to system calls.""" - try: - return self._str - except AttributeError: - self._str = self._format_parsed_parts(self.drive, self.root, - self._tail) or '.' - return self._str - - @classmethod - def _format_parsed_parts(cls, drv, root, tail): - if drv or root: - return drv + root + cls.parser.sep.join(tail) - elif tail and cls.parser.splitdrive(tail[0])[0]: - tail = ['.'] + tail - return cls.parser.sep.join(tail) - - def _from_parsed_parts(self, drv, root, tail): - path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) - path._drv = drv - path._root = root - path._tail_cached = tail - return path - - def _from_parsed_string(self, path_str): - path = self.with_segments(path_str) - path._str = path_str or '.' - return path - - @classmethod - def _parse_path(cls, path): - if not path: - return '', '', [] - sep = cls.parser.sep - altsep = cls.parser.altsep - if altsep: - path = path.replace(altsep, sep) - drv, root, rel = cls.parser.splitroot(path) - if not root and drv.startswith(sep) and not drv.endswith(sep): - drv_parts = drv.split(sep) - if len(drv_parts) == 4 and drv_parts[2] not in '?.': - # e.g. //server/share - root = sep - elif len(drv_parts) == 6: - # e.g. //?/unc/server/share - root = sep - parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] - return drv, root, parsed - - @property - def _raw_path(self): - """The joined but unnormalized path.""" - paths = self._raw_paths - if len(paths) == 0: - path = '' - elif len(paths) == 1: - path = paths[0] - else: - path = self.parser.join(*paths) - return path - - @property - def drive(self): - """The drive prefix (letter or UNC path), if any.""" - try: - return self._drv - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._drv - - @property - def root(self): - """The root of the path, if any.""" - try: - return self._root - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._root - - @property - def _tail(self): - try: - return self._tail_cached - except AttributeError: - self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) - return self._tail_cached - - @property - def anchor(self): - """The concatenation of the drive and root, or ''.""" - return self.drive + self.root - - @property - def parts(self): - """An object providing sequence-like access to the - components in the filesystem path.""" - if self.drive or self.root: - return (self.drive + self.root,) + tuple(self._tail) - else: - return tuple(self._tail) - - @property - def parent(self): - """The logical parent of the path.""" - drv = self.drive - root = self.root - tail = self._tail - if not tail: - return self - return self._from_parsed_parts(drv, root, tail[:-1]) - - @property - def parents(self): - """A sequence of this path's logical parents.""" - # The value of this property should not be cached on the path object, - # as doing so would introduce a reference cycle. - return _PathParents(self) - - @property - def name(self): - """The final path component, if any.""" - tail = self._tail - if not tail: - return '' - return tail[-1] - - def with_name(self, name): - """Return a new path with the file name changed.""" - p = self.parser - if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': - raise ValueError(f"Invalid name {name!r}") - tail = self._tail.copy() - if not tail: - raise ValueError(f"{self!r} has an empty name") - tail[-1] = name - return self._from_parsed_parts(self.drive, self.root, tail) - - def relative_to(self, other, /, *_deprecated, walk_up=False): - """Return the relative path to another path identified by the passed - arguments. If the operation is not possible (because this is not - related to the other path), raise ValueError. - - The *walk_up* parameter controls whether `..` may be used to resolve - the path. - """ - if _deprecated: - msg = ("support for supplying more than one positional argument " - "to pathlib.PurePath.relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - for step, path in enumerate(chain([other], other.parents)): - if path == self or path in self.parents: - break - elif not walk_up: - raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") - elif path.name == '..': - raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") - else: - raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") - parts = ['..'] * step + self._tail[len(path._tail):] - return self._from_parsed_parts('', '', parts) - - def is_relative_to(self, other, /, *_deprecated): - """Return True if the path is relative to another path or False. - """ - if _deprecated: - msg = ("support for supplying more than one argument to " - "pathlib.PurePath.is_relative_to() is deprecated and " - "scheduled for removal in Python 3.14") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - other = self.with_segments(other, *_deprecated) - elif not isinstance(other, PurePath): - other = self.with_segments(other) - return other == self or other in self.parents - - def is_absolute(self): - """True if the path is absolute (has both a root and, if applicable, - a drive).""" - if self.parser is posixpath: - # Optimization: work with raw paths on POSIX. - for path in self._raw_paths: - if path.startswith('/'): - return True - return False - return self.parser.isabs(self) - - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " - "for removal in Python 3.15. Use os.path.isreserved() to " - "detect reserved paths on Windows.") - warnings.warn(msg, DeprecationWarning, stacklevel=2) - if self.parser is ntpath: - return self.parser.isreserved(self) - return False - - def as_uri(self): - """Return the path as a URI.""" - if not self.is_absolute(): - raise ValueError("relative path can't be expressed as a file URI") - - drive = self.drive - if len(drive) == 2 and drive[1] == ':': - # It's a path on a local drive => 'file:///c:/a/b' - prefix = 'file:///' + drive - path = self.as_posix()[2:] - elif drive: - # It's a path on a network drive => 'file://host/share/a/b' - prefix = 'file:' - path = self.as_posix() - else: - # It's a posix path => 'file:///etc/hosts' - prefix = 'file://' - path = str(self) - from urllib.parse import quote_from_bytes - return prefix + quote_from_bytes(os.fsencode(path)) - - @property - def _pattern_str(self): - """The path expressed as a string, for use in pattern-matching.""" - # The string representation of an empty path is a single dot ('.'). Empty - # paths shouldn't match wildcards, so we change it to the empty string. - path_str = str(self) - return '' if path_str == '.' else path_str - -# Subclassing os.PathLike makes isinstance() checks slower, -# which in turn makes Path construction slower. Register instead! -os.PathLike.register(PurePath) - - -class PurePosixPath(PurePath): - """PurePath subclass for non-Windows systems. - - On a POSIX system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - parser = posixpath - __slots__ = () - - -class PureWindowsPath(PurePath): - """PurePath subclass for Windows systems. - - On a Windows system, instantiating a PurePath should return this object. - However, you can also instantiate it directly on any system. - """ - parser = ntpath - __slots__ = () - - -class Path(_abc.PathBase, PurePath): - """PurePath subclass that can make system calls. - - Path represents a filesystem path but unlike PurePath, also offers - methods to do system calls on path objects. Depending on your system, - instantiating a Path will return either a PosixPath or a WindowsPath - object. You can also instantiate a PosixPath or WindowsPath directly, - but cannot instantiate a WindowsPath on a POSIX system or vice versa. - """ - __slots__ = () - as_uri = PurePath.as_uri - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" - - def __init__(self, *args, **kwargs): - if kwargs: - msg = ("support for supplying keyword arguments to pathlib.PurePath " - "is deprecated and scheduled for removal in Python {remove}") - warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) - super().__init__(*args) - - def __new__(cls, *args, **kwargs): - if cls is Path: - cls = WindowsPath if os.name == 'nt' else PosixPath - return object.__new__(cls) - - def stat(self, *, follow_symlinks=True): - """ - Return the result of the stat() system call on this path, like - os.stat() does. - """ - return os.stat(self, follow_symlinks=follow_symlinks) - - def is_mount(self): - """ - Check if this path is a mount point - """ - return os.path.ismount(self) - - def is_junction(self): - """ - Whether this path is a junction. - """ - return os.path.isjunction(self) - - def open(self, mode='r', buffering=-1, encoding=None, - errors=None, newline=None): - """ - Open the file pointed to by this path and return a file object, as - the built-in open() function does. - """ - if "b" not in mode: - encoding = io.text_encoding(encoding) - return io.open(self, mode, buffering, encoding, errors, newline) - - def read_text(self, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, read it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.read_text(self, encoding, errors, newline) - - def write_text(self, data, encoding=None, errors=None, newline=None): - """ - Open the file in text mode, write to it, and close the file. - """ - # Call io.text_encoding() here to ensure any warning is raised at an - # appropriate stack level. - encoding = io.text_encoding(encoding) - return _abc.PathBase.write_text(self, data, encoding, errors, newline) - - _remove_leading_dot = operator.itemgetter(slice(2, None)) - _remove_trailing_slash = operator.itemgetter(slice(-1)) - - def _filter_trailing_slash(self, paths): - sep = self.parser.sep - anchor_len = len(self.anchor) - for path_str in paths: - if len(path_str) > anchor_len and path_str[-1] == sep: - path_str = path_str[:-1] - yield path_str - - def iterdir(self): - """Yield path objects of the directory contents. - - The children are yielded in arbitrary order, and the - special entries '.' and '..' are not included. - """ - root_dir = str(self) - with os.scandir(root_dir) as scandir_it: - paths = [entry.path for entry in scandir_it] - if root_dir == '.': - paths = map(self._remove_leading_dot, paths) - return map(self._from_parsed_string, paths) - - def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): - """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given relative pattern. - """ - sys.audit("pathlib.Path.glob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - if pattern.anchor: - raise NotImplementedError("Non-relative patterns are unsupported") - parts = pattern._tail.copy() - if not parts: - raise ValueError("Unacceptable pattern: {!r}".format(pattern)) - raw = pattern._raw_path - if raw[-1] in (self.parser.sep, self.parser.altsep): - # GH-65238: pathlib doesn't preserve trailing slash. Add it back. - parts.append('') - select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) - root = str(self) - paths = select(root) - - # Normalize results - if root == '.': - paths = map(self._remove_leading_dot, paths) - if parts[-1] == '': - paths = map(self._remove_trailing_slash, paths) - elif parts[-1] == '**': - paths = self._filter_trailing_slash(paths) - paths = map(self._from_parsed_string, paths) - return paths - - def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): - """Recursively yield all existing files (of any kind, including - directories) matching the given relative pattern, anywhere in - this subtree. - """ - sys.audit("pathlib.Path.rglob", self, pattern) - if not isinstance(pattern, PurePath): - pattern = self.with_segments(pattern) - pattern = '**' / pattern - return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) - - def walk(self, top_down=True, on_error=None, follow_symlinks=False): - """Walk the directory tree from this directory, similar to os.walk().""" - sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) - root_dir = str(self) - results = self._globber.walk(root_dir, top_down, on_error, follow_symlinks) - for path_str, dirnames, filenames in results: - if root_dir == '.': - path_str = path_str[2:] - yield self._from_parsed_string(path_str), dirnames, filenames - - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - if self.root: - drive = os.path.splitroot(os.getcwd())[0] - return self._from_parsed_parts(drive, self.root, self._tail) - if self.drive: - # There is a CWD on each drive-letter drive. - cwd = os.path.abspath(self.drive) - else: - cwd = os.getcwd() - if not self._tail: - # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). - # We pass only one argument to with_segments() to avoid the cost - # of joining, and we exploit the fact that getcwd() returns a - # fully-normalized string by storing it in _str. This is used to - # implement Path.cwd(). - return self._from_parsed_string(cwd) - drive, root, rel = os.path.splitroot(cwd) - if not rel: - return self._from_parsed_parts(drive, root, self._tail) - tail = rel.split(self.parser.sep) - tail.extend(self._tail) - return self._from_parsed_parts(drive, root, tail) - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - - return self.with_segments(os.path.realpath(self, strict=strict)) - - if pwd: - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - uid = self.stat(follow_symlinks=follow_symlinks).st_uid - return pwd.getpwuid(uid).pw_name - - if grp: - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - gid = self.stat(follow_symlinks=follow_symlinks).st_gid - return grp.getgrgid(gid).gr_name - - if hasattr(os, "readlink"): - def readlink(self): - """ - Return the path to which the symbolic link points. - """ - return self.with_segments(os.readlink(self)) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - - if exist_ok: - # First try to bump modification time - # Implementation note: GNU touch uses the UTIME_NOW option of - # the utimensat() / futimens() functions. - try: - os.utime(self, None) - except OSError: - # Avoid exception chaining - pass - else: - return - flags = os.O_CREAT | os.O_WRONLY - if not exist_ok: - flags |= os.O_EXCL - fd = os.open(self, flags, mode) - os.close(fd) - - def mkdir(self, mode=0o777, parents=False, exist_ok=False): - """ - Create a new directory at this given path. - """ - try: - os.mkdir(self, mode) - except FileNotFoundError: - if not parents or self.parent == self: - raise - self.parent.mkdir(parents=True, exist_ok=True) - self.mkdir(mode, parents=False, exist_ok=exist_ok) - except OSError: - # Cannot rely on checking for EEXIST, since the operating system - # could give priority to other errors like EACCES or EROFS - if not exist_ok or not self.is_dir(): - raise - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - os.chmod(self, mode, follow_symlinks=follow_symlinks) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - try: - os.unlink(self) - except FileNotFoundError: - if not missing_ok: - raise - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - os.rmdir(self) - - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.rename(self, target) - return self.with_segments(target) - - def replace(self, target): - """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - os.replace(self, target) - return self.with_segments(target) - - if hasattr(os, "symlink"): - def symlink_to(self, target, target_is_directory=False): - """ - Make this path a symlink pointing to the target path. - Note the order of arguments (link, target) is the reverse of os.symlink. - """ - os.symlink(target, self, target_is_directory) - - if hasattr(os, "link"): - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - os.link(target, self) - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - if (not (self.drive or self.root) and - self._tail and self._tail[0][:1] == '~'): - homedir = os.path.expanduser(self._tail[0]) - if homedir[:1] == "~": - raise RuntimeError("Could not determine home directory.") - drv, root, tail = self._parse_path(homedir) - return self._from_parsed_parts(drv, root, tail + self._tail[1:]) - - return self - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - if not uri.startswith('file:'): - raise ValueError(f"URI does not start with 'file:': {uri!r}") - path = uri[5:] - if path[:3] == '///': - # Remove empty authority - path = path[2:] - elif path[:12] == '//localhost/': - # Remove 'localhost' authority - path = path[11:] - if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): - # Remove slash before DOS device/UNC path - path = path[1:] - if path[1:2] == '|': - # Replace bar with colon in DOS drive - path = path[:1] + ':' + path[2:] - from urllib.parse import unquote_to_bytes - path = cls(os.fsdecode(unquote_to_bytes(path))) - if not path.is_absolute(): - raise ValueError(f"URI is not absolute: {uri!r}") - return path - - -class PosixPath(Path, PurePosixPath): - """Path subclass for non-Windows systems. - - On a POSIX system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name == 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") - -class WindowsPath(Path, PureWindowsPath): - """Path subclass for Windows systems. - - On a Windows system, instantiating a Path should return this object. - """ - __slots__ = () - - if os.name != 'nt': - def __new__(cls, *args, **kwargs): - raise UnsupportedOperation( - f"cannot instantiate {cls.__name__!r} on your system") +__all__ = (_abc.__all__ + + _local.__all__) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 591df443be093af..9ef016a060f1d72 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,11 +12,13 @@ """ import functools -import operator from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from . import _glob +from ._glob import Globber, no_recurse_symlinks + + +__all__ = ["UnsupportedOperation"] # # Internals @@ -44,30 +46,6 @@ def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' -class Globber(_glob.Globber): - lstat = operator.methodcaller('lstat') - add_slash = operator.methodcaller('joinpath', '') - - @staticmethod - def scandir(path): - # Emulate os.scandir(), which returns an object that can be used as a - # context manager. This method is called by walk() and glob(). - from contextlib import nullcontext - return nullcontext(path.iterdir()) - - @staticmethod - def concat_path(path, text): - """Appends text to the given path. - """ - return path.with_segments(path._raw_path + text) - - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). - """ - return entry - - class UnsupportedOperation(NotImplementedError): """An exception that is raised when an unsupported operation is called on a path object. @@ -693,7 +671,7 @@ def _glob_selector(self, parts, case_sensitive, recurse_symlinks): # know the case sensitivity of the underlying filesystem, so we # must use scandir() for everything, including non-wildcard parts. case_pedantic = True - recursive = True if recurse_symlinks else _glob.no_recurse_symlinks + recursive = True if recurse_symlinks else no_recurse_symlinks globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) return globber.selector(parts) diff --git a/Lib/pathlib/_glob.py b/Lib/pathlib/_glob.py index 73ccc0677920ce9..49a3851e80a038d 100644 --- a/Lib/pathlib/_glob.py +++ b/Lib/pathlib/_glob.py @@ -87,24 +87,28 @@ def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): # Low-level methods - lstat = staticmethod(os.lstat) - scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') - concat_path = operator.add + lstat = operator.methodcaller('lstat') + add_slash = operator.methodcaller('joinpath', '') - if os.name == 'nt': - @staticmethod - def add_slash(pathname): - tail = os.path.splitroot(pathname)[2] - if not tail or tail[-1] in '\\/': - return pathname - return f'{pathname}\\' - else: - @staticmethod - def add_slash(pathname): - if not pathname or pathname[-1] == '/': - return pathname - return f'{pathname}/' + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + from contextlib import nullcontext + return nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path. + """ + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir(). + """ + return entry # High-level methods @@ -303,3 +307,24 @@ def walk(cls, root, top_down, on_error, follow_symlinks): if dirnames: prefix = cls.add_slash(path) paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] + + +class StringGlobber(Globber): + lstat = staticmethod(os.lstat) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 000000000000000..e708191213a8b47 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,859 @@ +import io +import ntpath +import operator +import os +import posixpath +import sys +import warnings +from itertools import chain +from _collections_abc import Sequence + +try: + import pwd +except ImportError: + pwd = None +try: + import grp +except ImportError: + grp = None + +from ._abc import UnsupportedOperation, PurePathBase, PathBase +from ._glob import StringGlobber + + +__all__ = [ + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", + ] + + +class _PathParents(Sequence): + """This object provides sequence-like access to the logical ancestors + of a path. Don't try to construct it yourself.""" + __slots__ = ('_path', '_drv', '_root', '_tail') + + def __init__(self, path): + self._path = path + self._drv = path.drive + self._root = path.root + self._tail = path._tail + + def __len__(self): + return len(self._tail) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return tuple(self[i] for i in range(*idx.indices(len(self)))) + + if idx >= len(self) or idx < -len(self): + raise IndexError(idx) + if idx < 0: + idx += len(self) + return self._path._from_parsed_parts(self._drv, self._root, + self._tail[:-idx - 1]) + + def __repr__(self): + return "<{}.parents>".format(type(self._path).__name__) + + +class PurePath(PurePathBase): + """Base class for manipulating paths without I/O. + + PurePath represents a filesystem path and offers operations which + don't imply any actual filesystem I/O. Depending on your system, + instantiating a PurePath will return either a PurePosixPath or a + PureWindowsPath object. You can also instantiate either of these classes + directly, regardless of your system. + """ + + __slots__ = ( + # The `_raw_paths` slot stores unnormalized string paths. This is set + # in the `__init__()` method. + '_raw_paths', + + # The `_drv`, `_root` and `_tail_cached` slots store parsed and + # normalized parts of the path. They are set when any of the `drive`, + # `root` or `_tail` properties are accessed for the first time. The + # three-part division corresponds to the result of + # `os.path.splitroot()`, except that the tail is further split on path + # separators (i.e. it is a list of strings), and that the root and + # tail are normalized. + '_drv', '_root', '_tail_cached', + + # The `_str` slot stores the string representation of the path, + # computed from the drive, root and tail when `__str__()` is called + # for the first time. It's used to implement `_str_normcase` + '_str', + + # The `_str_normcase_cached` slot stores the string path with + # normalized case. It is set when the `_str_normcase` property is + # accessed for the first time. It's used to implement `__eq__()` + # `__hash__()`, and `_parts_normcase` + '_str_normcase_cached', + + # The `_parts_normcase_cached` slot stores the case-normalized + # string path after splitting on path separators. It's set when the + # `_parts_normcase` property is accessed for the first time. It's used + # to implement comparison methods like `__lt__()`. + '_parts_normcase_cached', + + # The `_hash` slot stores the hash of the case-normalized string + # path. It's set when `__hash__()` is called for the first time. + '_hash', + ) + parser = os.path + _globber = StringGlobber + + def __new__(cls, *args, **kwargs): + """Construct a PurePath from one or several strings and or existing + PurePath objects. The strings and path objects are combined so as + to yield a canonicalized path, which is incorporated into the + new PurePath object. + """ + if cls is PurePath: + cls = PureWindowsPath if os.name == 'nt' else PurePosixPath + return object.__new__(cls) + + def __init__(self, *args): + paths = [] + for arg in args: + if isinstance(arg, PurePath): + if arg.parser is ntpath and self.parser is posixpath: + # GH-103631: Convert separators for backwards compatibility. + paths.extend(path.replace('\\', '/') for path in arg._raw_paths) + else: + paths.extend(arg._raw_paths) + else: + try: + path = os.fspath(arg) + except TypeError: + path = arg + if not isinstance(path, str): + raise TypeError( + "argument should be a str or an os.PathLike " + "object where __fspath__ returns a str, " + f"not {type(path).__name__!r}") + paths.append(path) + # Avoid calling super().__init__, as an optimisation + self._raw_paths = paths + + def joinpath(self, *pathsegments): + """Combine this path with one or several arguments, and return a + new path representing either a subpath (if all arguments are relative + paths) or a totally different path (if one of the arguments is + anchored). + """ + return self.with_segments(self, *pathsegments) + + def __truediv__(self, key): + try: + return self.with_segments(self, key) + except TypeError: + return NotImplemented + + def __rtruediv__(self, key): + try: + return self.with_segments(key, self) + except TypeError: + return NotImplemented + + def __reduce__(self): + return self.__class__, tuple(self._raw_paths) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, self.as_posix()) + + def __fspath__(self): + return str(self) + + def __bytes__(self): + """Return the bytes representation of the path. This is only + recommended to use under Unix.""" + return os.fsencode(self) + + @property + def _str_normcase(self): + # String with normalized case, for hashing and equality checks + try: + return self._str_normcase_cached + except AttributeError: + if self.parser is posixpath: + self._str_normcase_cached = str(self) + else: + self._str_normcase_cached = str(self).lower() + return self._str_normcase_cached + + def __hash__(self): + try: + return self._hash + except AttributeError: + self._hash = hash(self._str_normcase) + return self._hash + + def __eq__(self, other): + if not isinstance(other, PurePath): + return NotImplemented + return self._str_normcase == other._str_normcase and self.parser is other.parser + + @property + def _parts_normcase(self): + # Cached parts with normalized case, for comparisons. + try: + return self._parts_normcase_cached + except AttributeError: + self._parts_normcase_cached = self._str_normcase.split(self.parser.sep) + return self._parts_normcase_cached + + def __lt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase < other._parts_normcase + + def __le__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase <= other._parts_normcase + + def __gt__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase > other._parts_normcase + + def __ge__(self, other): + if not isinstance(other, PurePath) or self.parser is not other.parser: + return NotImplemented + return self._parts_normcase >= other._parts_normcase + + def __str__(self): + """Return the string representation of the path, suitable for + passing to system calls.""" + try: + return self._str + except AttributeError: + self._str = self._format_parsed_parts(self.drive, self.root, + self._tail) or '.' + return self._str + + @classmethod + def _format_parsed_parts(cls, drv, root, tail): + if drv or root: + return drv + root + cls.parser.sep.join(tail) + elif tail and cls.parser.splitdrive(tail[0])[0]: + tail = ['.'] + tail + return cls.parser.sep.join(tail) + + def _from_parsed_parts(self, drv, root, tail): + path = self._from_parsed_string(self._format_parsed_parts(drv, root, tail)) + path._drv = drv + path._root = root + path._tail_cached = tail + return path + + def _from_parsed_string(self, path_str): + path = self.with_segments(path_str) + path._str = path_str or '.' + return path + + @classmethod + def _parse_path(cls, path): + if not path: + return '', '', [] + sep = cls.parser.sep + altsep = cls.parser.altsep + if altsep: + path = path.replace(altsep, sep) + drv, root, rel = cls.parser.splitroot(path) + if not root and drv.startswith(sep) and not drv.endswith(sep): + drv_parts = drv.split(sep) + if len(drv_parts) == 4 and drv_parts[2] not in '?.': + # e.g. //server/share + root = sep + elif len(drv_parts) == 6: + # e.g. //?/unc/server/share + root = sep + parsed = [sys.intern(str(x)) for x in rel.split(sep) if x and x != '.'] + return drv, root, parsed + + @property + def _raw_path(self): + """The joined but unnormalized path.""" + paths = self._raw_paths + if len(paths) == 0: + path = '' + elif len(paths) == 1: + path = paths[0] + else: + path = self.parser.join(*paths) + return path + + @property + def drive(self): + """The drive prefix (letter or UNC path), if any.""" + try: + return self._drv + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._drv + + @property + def root(self): + """The root of the path, if any.""" + try: + return self._root + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._root + + @property + def _tail(self): + try: + return self._tail_cached + except AttributeError: + self._drv, self._root, self._tail_cached = self._parse_path(self._raw_path) + return self._tail_cached + + @property + def anchor(self): + """The concatenation of the drive and root, or ''.""" + return self.drive + self.root + + @property + def parts(self): + """An object providing sequence-like access to the + components in the filesystem path.""" + if self.drive or self.root: + return (self.drive + self.root,) + tuple(self._tail) + else: + return tuple(self._tail) + + @property + def parent(self): + """The logical parent of the path.""" + drv = self.drive + root = self.root + tail = self._tail + if not tail: + return self + return self._from_parsed_parts(drv, root, tail[:-1]) + + @property + def parents(self): + """A sequence of this path's logical parents.""" + # The value of this property should not be cached on the path object, + # as doing so would introduce a reference cycle. + return _PathParents(self) + + @property + def name(self): + """The final path component, if any.""" + tail = self._tail + if not tail: + return '' + return tail[-1] + + def with_name(self, name): + """Return a new path with the file name changed.""" + p = self.parser + if not name or p.sep in name or (p.altsep and p.altsep in name) or name == '.': + raise ValueError(f"Invalid name {name!r}") + tail = self._tail.copy() + if not tail: + raise ValueError(f"{self!r} has an empty name") + tail[-1] = name + return self._from_parsed_parts(self.drive, self.root, tail) + + def relative_to(self, other, /, *_deprecated, walk_up=False): + """Return the relative path to another path identified by the passed + arguments. If the operation is not possible (because this is not + related to the other path), raise ValueError. + + The *walk_up* parameter controls whether `..` may be used to resolve + the path. + """ + if _deprecated: + msg = ("support for supplying more than one positional argument " + "to pathlib.PurePath.relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + for step, path in enumerate(chain([other], other.parents)): + if path == self or path in self.parents: + break + elif not walk_up: + raise ValueError(f"{str(self)!r} is not in the subpath of {str(other)!r}") + elif path.name == '..': + raise ValueError(f"'..' segment in {str(other)!r} cannot be walked") + else: + raise ValueError(f"{str(self)!r} and {str(other)!r} have different anchors") + parts = ['..'] * step + self._tail[len(path._tail):] + return self._from_parsed_parts('', '', parts) + + def is_relative_to(self, other, /, *_deprecated): + """Return True if the path is relative to another path or False. + """ + if _deprecated: + msg = ("support for supplying more than one argument to " + "pathlib.PurePath.is_relative_to() is deprecated and " + "scheduled for removal in Python 3.14") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + other = self.with_segments(other, *_deprecated) + elif not isinstance(other, PurePath): + other = self.with_segments(other) + return other == self or other in self.parents + + def is_absolute(self): + """True if the path is absolute (has both a root and, if applicable, + a drive).""" + if self.parser is posixpath: + # Optimization: work with raw paths on POSIX. + for path in self._raw_paths: + if path.startswith('/'): + return True + return False + return self.parser.isabs(self) + + def is_reserved(self): + """Return True if the path contains one of the special names reserved + by the system, if any.""" + msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " + "for removal in Python 3.15. Use os.path.isreserved() to " + "detect reserved paths on Windows.") + warnings.warn(msg, DeprecationWarning, stacklevel=2) + if self.parser is ntpath: + return self.parser.isreserved(self) + return False + + def as_uri(self): + """Return the path as a URI.""" + if not self.is_absolute(): + raise ValueError("relative path can't be expressed as a file URI") + + drive = self.drive + if len(drive) == 2 and drive[1] == ':': + # It's a path on a local drive => 'file:///c:/a/b' + prefix = 'file:///' + drive + path = self.as_posix()[2:] + elif drive: + # It's a path on a network drive => 'file://host/share/a/b' + prefix = 'file:' + path = self.as_posix() + else: + # It's a posix path => 'file:///etc/hosts' + prefix = 'file://' + path = str(self) + from urllib.parse import quote_from_bytes + return prefix + quote_from_bytes(os.fsencode(path)) + + @property + def _pattern_str(self): + """The path expressed as a string, for use in pattern-matching.""" + # The string representation of an empty path is a single dot ('.'). Empty + # paths shouldn't match wildcards, so we change it to the empty string. + path_str = str(self) + return '' if path_str == '.' else path_str + +# Subclassing os.PathLike makes isinstance() checks slower, +# which in turn makes Path construction slower. Register instead! +os.PathLike.register(PurePath) + + +class PurePosixPath(PurePath): + """PurePath subclass for non-Windows systems. + + On a POSIX system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = posixpath + __slots__ = () + + +class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ + parser = ntpath + __slots__ = () + + +class Path(PathBase, PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ + __slots__ = () + as_uri = PurePath.as_uri + + @classmethod + def _unsupported_msg(cls, attribute): + return f"{cls.__name__}.{attribute} is unsupported on this system" + + def __init__(self, *args, **kwargs): + if kwargs: + msg = ("support for supplying keyword arguments to pathlib.PurePath " + "is deprecated and scheduled for removal in Python {remove}") + warnings._deprecated("pathlib.PurePath(**kwargs)", msg, remove=(3, 14)) + super().__init__(*args) + + def __new__(cls, *args, **kwargs): + if cls is Path: + cls = WindowsPath if os.name == 'nt' else PosixPath + return object.__new__(cls) + + def stat(self, *, follow_symlinks=True): + """ + Return the result of the stat() system call on this path, like + os.stat() does. + """ + return os.stat(self, follow_symlinks=follow_symlinks) + + def is_mount(self): + """ + Check if this path is a mount point + """ + return os.path.ismount(self) + + def is_junction(self): + """ + Whether this path is a junction. + """ + return os.path.isjunction(self) + + def open(self, mode='r', buffering=-1, encoding=None, + errors=None, newline=None): + """ + Open the file pointed to by this path and return a file object, as + the built-in open() function does. + """ + if "b" not in mode: + encoding = io.text_encoding(encoding) + return io.open(self, mode, buffering, encoding, errors, newline) + + def read_text(self, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, read it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.read_text(self, encoding, errors, newline) + + def write_text(self, data, encoding=None, errors=None, newline=None): + """ + Open the file in text mode, write to it, and close the file. + """ + # Call io.text_encoding() here to ensure any warning is raised at an + # appropriate stack level. + encoding = io.text_encoding(encoding) + return PathBase.write_text(self, data, encoding, errors, newline) + + _remove_leading_dot = operator.itemgetter(slice(2, None)) + _remove_trailing_slash = operator.itemgetter(slice(-1)) + + def _filter_trailing_slash(self, paths): + sep = self.parser.sep + anchor_len = len(self.anchor) + for path_str in paths: + if len(path_str) > anchor_len and path_str[-1] == sep: + path_str = path_str[:-1] + yield path_str + + def iterdir(self): + """Yield path objects of the directory contents. + + The children are yielded in arbitrary order, and the + special entries '.' and '..' are not included. + """ + root_dir = str(self) + with os.scandir(root_dir) as scandir_it: + paths = [entry.path for entry in scandir_it] + if root_dir == '.': + paths = map(self._remove_leading_dot, paths) + return map(self._from_parsed_string, paths) + + def glob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Iterate over this subtree and yield all existing files (of any + kind, including directories) matching the given relative pattern. + """ + sys.audit("pathlib.Path.glob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + if pattern.anchor: + raise NotImplementedError("Non-relative patterns are unsupported") + parts = pattern._tail.copy() + if not parts: + raise ValueError("Unacceptable pattern: {!r}".format(pattern)) + raw = pattern._raw_path + if raw[-1] in (self.parser.sep, self.parser.altsep): + # GH-65238: pathlib doesn't preserve trailing slash. Add it back. + parts.append('') + select = self._glob_selector(parts[::-1], case_sensitive, recurse_symlinks) + root = str(self) + paths = select(root) + + # Normalize results + if root == '.': + paths = map(self._remove_leading_dot, paths) + if parts[-1] == '': + paths = map(self._remove_trailing_slash, paths) + elif parts[-1] == '**': + paths = self._filter_trailing_slash(paths) + paths = map(self._from_parsed_string, paths) + return paths + + def rglob(self, pattern, *, case_sensitive=None, recurse_symlinks=False): + """Recursively yield all existing files (of any kind, including + directories) matching the given relative pattern, anywhere in + this subtree. + """ + sys.audit("pathlib.Path.rglob", self, pattern) + if not isinstance(pattern, PurePath): + pattern = self.with_segments(pattern) + pattern = '**' / pattern + return self.glob(pattern, case_sensitive=case_sensitive, recurse_symlinks=recurse_symlinks) + + def walk(self, top_down=True, on_error=None, follow_symlinks=False): + """Walk the directory tree from this directory, similar to os.walk().""" + sys.audit("pathlib.Path.walk", self, on_error, follow_symlinks) + root_dir = str(self) + results = self._globber.walk(root_dir, top_down, on_error, follow_symlinks) + for path_str, dirnames, filenames in results: + if root_dir == '.': + path_str = path_str[2:] + yield self._from_parsed_string(path_str), dirnames, filenames + + def absolute(self): + """Return an absolute version of this path + No normalization or symlink resolution is performed. + + Use resolve() to resolve symlinks and remove '..' segments. + """ + if self.is_absolute(): + return self + if self.root: + drive = os.path.splitroot(os.getcwd())[0] + return self._from_parsed_parts(drive, self.root, self._tail) + if self.drive: + # There is a CWD on each drive-letter drive. + cwd = os.path.abspath(self.drive) + else: + cwd = os.getcwd() + if not self._tail: + # Fast path for "empty" paths, e.g. Path("."), Path("") or Path(). + # We pass only one argument to with_segments() to avoid the cost + # of joining, and we exploit the fact that getcwd() returns a + # fully-normalized string by storing it in _str. This is used to + # implement Path.cwd(). + return self._from_parsed_string(cwd) + drive, root, rel = os.path.splitroot(cwd) + if not rel: + return self._from_parsed_parts(drive, root, self._tail) + tail = rel.split(self.parser.sep) + tail.extend(self._tail) + return self._from_parsed_parts(drive, root, tail) + + def resolve(self, strict=False): + """ + Make the path absolute, resolving all symlinks on the way and also + normalizing it. + """ + + return self.with_segments(os.path.realpath(self, strict=strict)) + + if pwd: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + uid = self.stat(follow_symlinks=follow_symlinks).st_uid + return pwd.getpwuid(uid).pw_name + + if grp: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + gid = self.stat(follow_symlinks=follow_symlinks).st_gid + return grp.getgrgid(gid).gr_name + + if hasattr(os, "readlink"): + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + return self.with_segments(os.readlink(self)) + + def touch(self, mode=0o666, exist_ok=True): + """ + Create this file with the given access mode, if it doesn't exist. + """ + + if exist_ok: + # First try to bump modification time + # Implementation note: GNU touch uses the UTIME_NOW option of + # the utimensat() / futimens() functions. + try: + os.utime(self, None) + except OSError: + # Avoid exception chaining + pass + else: + return + flags = os.O_CREAT | os.O_WRONLY + if not exist_ok: + flags |= os.O_EXCL + fd = os.open(self, flags, mode) + os.close(fd) + + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + try: + os.mkdir(self, mode) + except FileNotFoundError: + if not parents or self.parent == self: + raise + self.parent.mkdir(parents=True, exist_ok=True) + self.mkdir(mode, parents=False, exist_ok=exist_ok) + except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS + if not exist_ok or not self.is_dir(): + raise + + def chmod(self, mode, *, follow_symlinks=True): + """ + Change the permissions of the path, like os.chmod(). + """ + os.chmod(self, mode, follow_symlinks=follow_symlinks) + + def unlink(self, missing_ok=False): + """ + Remove this file or link. + If the path is a directory, use rmdir() instead. + """ + try: + os.unlink(self) + except FileNotFoundError: + if not missing_ok: + raise + + def rmdir(self): + """ + Remove this directory. The directory must be empty. + """ + os.rmdir(self) + + def rename(self, target): + """ + Rename this path to the target path. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.rename(self, target) + return self.with_segments(target) + + def replace(self, target): + """ + Rename this path to the target path, overwriting if that path exists. + + The target path may be absolute or relative. Relative paths are + interpreted relative to the current working directory, *not* the + directory of the Path object. + + Returns the new Path instance pointing to the target path. + """ + os.replace(self, target) + return self.with_segments(target) + + if hasattr(os, "symlink"): + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + os.symlink(target, self, target_is_directory) + + if hasattr(os, "link"): + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + os.link(target, self) + + def expanduser(self): + """ Return a new path with expanded ~ and ~user constructs + (as returned by os.path.expanduser) + """ + if (not (self.drive or self.root) and + self._tail and self._tail[0][:1] == '~'): + homedir = os.path.expanduser(self._tail[0]) + if homedir[:1] == "~": + raise RuntimeError("Could not determine home directory.") + drv, root, tail = self._parse_path(homedir) + return self._from_parsed_parts(drv, root, tail + self._tail[1:]) + + return self + + @classmethod + def from_uri(cls, uri): + """Return a new path from the given 'file' URI.""" + if not uri.startswith('file:'): + raise ValueError(f"URI does not start with 'file:': {uri!r}") + path = uri[5:] + if path[:3] == '///': + # Remove empty authority + path = path[2:] + elif path[:12] == '//localhost/': + # Remove 'localhost' authority + path = path[11:] + if path[:3] == '///' or (path[:1] == '/' and path[2:3] in ':|'): + # Remove slash before DOS device/UNC path + path = path[1:] + if path[1:2] == '|': + # Replace bar with colon in DOS drive + path = path[:1] + ':' + path[2:] + from urllib.parse import unquote_to_bytes + path = cls(os.fsdecode(unquote_to_bytes(path))) + if not path.is_absolute(): + raise ValueError(f"URI is not absolute: {uri!r}") + return path + + +class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name == 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") + +class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ + __slots__ = () + + if os.name != 'nt': + def __new__(cls, *args, **kwargs): + raise UnsupportedOperation( + f"cannot instantiate {cls.__name__!r} on your system") From b4f8eb0de2c775e4fb11b15d26b9f21195732d47 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 5 May 2024 14:31:35 -0700 Subject: [PATCH 134/220] gh-118605: Fix reference leak in FrameLocalsProxy (#118607) Also add some error checks. --- Objects/frameobject.c | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 1cb00e318d9163b..fdac86961f860c0 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -232,6 +232,8 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) Py_DECREF(value); } + Py_DECREF(iter); + return 0; } @@ -242,11 +244,18 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + if (names == NULL) { + return NULL; + } + for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *val = framelocalsproxy_getval(frame->f_frame, co, i); if (val) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); - PyList_Append(names, name); + if (PyList_Append(names, name) < 0) { + Py_DECREF(names); + return NULL; + } } } @@ -258,7 +267,10 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) if (frame->f_extra_locals) { assert(PyDict_Check(frame->f_extra_locals)); while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { - PyList_Append(names, key); + if (PyList_Append(names, key) < 0) { + Py_DECREF(names); + return NULL; + } } } @@ -306,7 +318,15 @@ framelocalsproxy_visit(PyObject *self, visitproc visit, void *arg) static PyObject * framelocalsproxy_iter(PyObject *self) { - return PyObject_GetIter(framelocalsproxy_keys(self, NULL)); + PyObject* keys = framelocalsproxy_keys(self, NULL); + if (keys == NULL) { + return NULL; + } + + PyObject* iter = PyObject_GetIter(keys); + Py_XDECREF(keys); + + return iter; } static PyObject * @@ -567,6 +587,11 @@ static PyObject* framelocalsproxy_reversed(PyObject *self, PyObject *__unused) { PyObject *result = framelocalsproxy_keys(self, NULL); + + if (result == NULL) { + return NULL; + } + if (PyList_Reverse(result) < 0) { Py_DECREF(result); return NULL; From b744fa5d3ec2294f3ef94733b728481807ca3039 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 5 May 2024 14:43:42 -0700 Subject: [PATCH 135/220] gh-111140: minor docs typos cleanup in the C example API calls. (#118612) --- Doc/c-api/long.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 1eb8f191c3ca327..522c028cfb8d406 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -390,7 +390,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Usage example:: int32_t value; - Py_ssize_t bytes = PyLong_AsNativeBits(pylong, &value, sizeof(value), -1); + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, &value, sizeof(value), -1); if (bytes < 0) { // Failed. A Python exception was set with the reason. return NULL; @@ -418,7 +418,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. called twice: first to determine the buffer size, then to fill it:: // Ask how much space we need. - Py_ssize_t expected = PyLong_AsNativeBits(pylong, NULL, 0, -1); + Py_ssize_t expected = PyLong_AsNativeBytes(pylong, NULL, 0, -1); if (expected < 0) { // Failed. A Python exception was set with the reason. return NULL; @@ -430,7 +430,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. return NULL; } // Safely get the entire value. - Py_ssize_t bytes = PyLong_AsNativeBits(pylong, bignum, expected, -1); + Py_ssize_t bytes = PyLong_AsNativeBytes(pylong, bignum, expected, -1); if (bytes < 0) { // Exception has been set. free(bignum); return NULL; From 2eb99914c4d9457c6e26ce522e9034309517aae5 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 5 May 2024 23:13:50 +0100 Subject: [PATCH 136/220] Add Lysandros Nikolaou to the News entry of gh-111201 (#118616) --- .../2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst index 9176da155d48fbe..6f1d29997e30b1a 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-00-41-17.gh-issue-111201.cQsh5U.rst @@ -1,4 +1,4 @@ The :term:`interactive` interpreter is now implemented in Python, which allows for a number of new features like colors, multiline input, history -viewing, and paste mode. Contributed by Pablo Galindo and Łukasz Langa based -on code from the PyPy project. +viewing, and paste mode. Contributed by Pablo Galindo, Łukasz Langa and +Lysandros Nikolaou based on code from the PyPy project. From dd37e85518bb5f3c9eff44ed1dbebecdd1d2aa06 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 6 May 2024 10:34:59 +0800 Subject: [PATCH 137/220] Correct timing sensitivity in iOS testing Makefile target. (GH-118620) --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 903227709c79ec2..ddc8b2feda51c49 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2040,7 +2040,7 @@ testuniversal: all # a full Xcode install that has an iPhone SE (3rd edition) simulator available. # This must be run *after* a `make install` has completed the build. The # `--with-framework-name` argument *cannot* be used when configuring the build. -XCFOLDER=iOSTestbed.$(MULTIARCH).$(shell date +%s) +XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult .PHONY: testios testios: From 1506d5adc457a806e85886018804483e808d85c5 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Mon, 6 May 2024 11:58:14 +0800 Subject: [PATCH 138/220] Set a DerivedData path for iOS test builds. (GH-118621) --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index ddc8b2feda51c49..88ec2009fbfffe4 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2067,7 +2067,7 @@ testios: # Run the test suite for the Xcode project, targeting the iOS simulator. # If the suite fails, touch a file in the test folder as a marker - if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) ; then \ + if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ touch $(XCFOLDER)/failed; \ fi From a8e5fed1007233f677d3539162257b99b920df00 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Mon, 6 May 2024 10:34:56 +0300 Subject: [PATCH 139/220] gh-118613: Fix error handling of `_PyEval_GetFrameLocals` in `ceval.c` (#118614) --- Python/ceval.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Python/ceval.c b/Python/ceval.c index 0d02a9887bef7ad..128e0417a9fd634 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -2496,17 +2496,21 @@ _PyEval_GetFrameLocals(void) if (PyFrameLocalsProxy_Check(locals)) { PyObject* ret = PyDict_New(); - if (PyDict_Update(ret, locals)) { + if (ret == NULL) { + Py_DECREF(locals); + return NULL; + } + if (PyDict_Update(ret, locals) < 0) { Py_DECREF(ret); + Py_DECREF(locals); return NULL; } Py_DECREF(locals); return ret; - } else if (PyMapping_Check(locals)) { - return locals; } - return NULL; + assert(PyMapping_Check(locals)); + return locals; } PyObject * From 7758be431807d574e0f1bbab003796585ae46719 Mon Sep 17 00:00:00 2001 From: Terry Jan Reedy Date: Mon, 6 May 2024 03:55:56 -0400 Subject: [PATCH 140/220] gh-78955: Use user-selected color theme for Help => IDLE Doc (#9502) --- Lib/idlelib/News3.txt | 2 ++ Lib/idlelib/help.py | 7 +++++-- .../next/IDLE/2018-09-23-01-36-39.bpo-34774.VeM-X-.rst | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2018-09-23-01-36-39.bpo-34774.VeM-X-.rst diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 241b1f48e5c1d84..fb07d7b3b3fad83 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -4,6 +4,8 @@ Released on 2024-10-xx ========================= +gh-78955: Use user-selected color theme for Help => IDLE Doc. + gh-96905: In idlelib code, stop redefining built-ins 'dict' and 'object'. gh-72284: Improve the lists of features, editor key bindings, diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index bdf4b2b29f11a26..d8613b2eadd6aa5 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -33,6 +33,7 @@ from tkinter import font as tkfont from idlelib.config import idleConf +from idlelib.colorizer import color_config ## About IDLE ## @@ -177,14 +178,16 @@ def __init__(self, parent, filename): normalfont = self.findfont(['TkDefaultFont', 'arial', 'helvetica']) fixedfont = self.findfont(['TkFixedFont', 'monaco', 'courier']) + color_config(self) self['font'] = (normalfont, 12) self.tag_configure('em', font=(normalfont, 12, 'italic')) self.tag_configure('h1', font=(normalfont, 20, 'bold')) self.tag_configure('h2', font=(normalfont, 18, 'bold')) self.tag_configure('h3', font=(normalfont, 15, 'bold')) - self.tag_configure('pre', font=(fixedfont, 12), background='#f6f6ff') + self.tag_configure('pre', font=(fixedfont, 12)) + preback = self['selectbackground'] self.tag_configure('preblock', font=(fixedfont, 10), lmargin1=25, - borderwidth=1, relief='solid', background='#eeffcc') + background=preback) self.tag_configure('l1', lmargin1=25, lmargin2=25) self.tag_configure('l2', lmargin1=50, lmargin2=50) self.tag_configure('l3', lmargin1=75, lmargin2=75) diff --git a/Misc/NEWS.d/next/IDLE/2018-09-23-01-36-39.bpo-34774.VeM-X-.rst b/Misc/NEWS.d/next/IDLE/2018-09-23-01-36-39.bpo-34774.VeM-X-.rst new file mode 100644 index 000000000000000..cac44b13f3f1fa9 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2018-09-23-01-36-39.bpo-34774.VeM-X-.rst @@ -0,0 +1 @@ +Use user-selected color theme for Help => IDLE Doc. From 05adfbba2abafcdd271bf144a7b3f80bcd927288 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 6 May 2024 10:04:39 +0200 Subject: [PATCH 141/220] gh-95382: Improve performance of json encoder with indent (GH-118105) --- Lib/json/encoder.py | 14 +- ...4-05-03-18-01-26.gh-issue-95382.73FSEv.rst | 2 + Modules/_json.c | 136 ++++++++++++------ 3 files changed, 105 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-03-18-01-26.gh-issue-95382.73FSEv.rst diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 597849eca0524a3..323332f064edf8f 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -244,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -272,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-03-18-01-26.gh-issue-95382.73FSEv.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-03-18-01-26.gh-issue-95382.73FSEv.rst new file mode 100644 index 000000000000000..097a663e3f5e249 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-03-18-01-26.gh-issue-95382.73FSEv.rst @@ -0,0 +1,2 @@ +Improve performance of :func:`json.dumps` and :func:`json.dump` when using the argument *indent*. Depending on the data the encoding using +:func:`json.dumps` with *indent* can be up to 2 to 3 times faster. diff --git a/Modules/_json.c b/Modules/_json.c index fc39f624b723f50..e33ef1f5eea92f0 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -85,11 +85,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyEncoderObject *self); static int -encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *seq, PyObject *newline_indent); static int -encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *obj, PyObject *newline_indent); static int -encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, PyObject *dct, PyObject *newline_indent); static PyObject * _encoded_const(PyObject *obj); static void @@ -1251,6 +1251,17 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)s; } +static PyObject * +_create_newline_indent(PyObject *indent, Py_ssize_t indent_level) +{ + PyObject *newline_indent = PyUnicode_FromOrdinal('\n'); + if (newline_indent != NULL && indent_level) { + PyUnicode_AppendAndDel(&newline_indent, + PySequence_Repeat(indent, indent_level)); + } + return newline_indent; +} + static PyObject * encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) { @@ -1267,10 +1278,20 @@ encoder_call(PyEncoderObject *self, PyObject *args, PyObject *kwds) _PyUnicodeWriter_Init(&writer); writer.overallocate = 1; - if (encoder_listencode_obj(self, &writer, obj, indent_level)) { + PyObject *newline_indent = NULL; + if (self->indent != Py_None) { + newline_indent = _create_newline_indent(self->indent, indent_level); + if (newline_indent == NULL) { + _PyUnicodeWriter_Dealloc(&writer); + return NULL; + } + } + if (encoder_listencode_obj(self, &writer, obj, newline_indent)) { _PyUnicodeWriter_Dealloc(&writer); + Py_XDECREF(newline_indent); return NULL; } + Py_XDECREF(newline_indent); result = PyTuple_New(1); if (result == NULL || @@ -1358,7 +1379,7 @@ _steal_accumulate(_PyUnicodeWriter *writer, PyObject *stolen) static int encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *obj, Py_ssize_t indent_level) + PyObject *obj, PyObject *newline_indent) { /* Encode Python object obj to a JSON term */ PyObject *newobj; @@ -1394,14 +1415,14 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, writer, obj, indent_level); + rv = encoder_listencode_list(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (_Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, writer, obj, indent_level); + rv = encoder_listencode_dict(s, writer, obj, newline_indent); _Py_LeaveRecursiveCall(); return rv; } @@ -1435,7 +1456,7 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_XDECREF(ident); return -1; } - rv = encoder_listencode_obj(s, writer, newobj, indent_level); + rv = encoder_listencode_obj(s, writer, newobj, newline_indent); _Py_LeaveRecursiveCall(); Py_DECREF(newobj); @@ -1456,7 +1477,9 @@ encoder_listencode_obj(PyEncoderObject *s, _PyUnicodeWriter *writer, static int encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *first, - PyObject *key, PyObject *value, Py_ssize_t indent_level) + PyObject *key, PyObject *value, + PyObject *newline_indent, + PyObject *item_separator) { PyObject *keystr = NULL; PyObject *encoded; @@ -1493,7 +1516,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir *first = false; } else { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator) < 0) { + if (_PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { Py_DECREF(keystr); return -1; } @@ -1511,7 +1534,7 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir if (_PyUnicodeWriter_WriteStr(writer, s->key_separator) < 0) { return -1; } - if (encoder_listencode_obj(s, writer, value, indent_level) < 0) { + if (encoder_listencode_obj(s, writer, value, newline_indent) < 0) { return -1; } return 0; @@ -1519,13 +1542,15 @@ encoder_encode_key_value(PyEncoderObject *s, _PyUnicodeWriter *writer, bool *fir static int encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *dct, Py_ssize_t indent_level) + PyObject *dct, PyObject *newline_indent) { /* Encode Python dict dct a JSON term */ PyObject *ident = NULL; PyObject *items = NULL; PyObject *key, *value; bool first = true; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; if (PyDict_GET_SIZE(dct) == 0) /* Fast path */ return _PyUnicodeWriter_WriteASCIIString(writer, "{}", 2); @@ -1549,14 +1574,21 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, '{')) goto bail; + PyObject *current_item_separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + separator_indent = PyUnicode_Concat(current_item_separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + // update item separator with a borrowed reference + current_item_separator = separator_indent; + if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1574,7 +1606,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, key = PyTuple_GET_ITEM(item, 0); value = PyTuple_GET_ITEM(item, 1); - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } Py_CLEAR(items); @@ -1582,7 +1616,9 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { - if (encoder_encode_key_value(s, writer, &first, key, value, indent_level) < 0) + if (encoder_encode_key_value(s, writer, &first, key, value, + new_newline_indent, + current_item_separator) < 0) goto bail; } } @@ -1592,12 +1628,15 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + + if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ if (_PyUnicodeWriter_WriteChar(writer, '}')) goto bail; return 0; @@ -1605,16 +1644,20 @@ encoder_listencode_dict(PyEncoderObject *s, _PyUnicodeWriter *writer, bail: Py_XDECREF(items); Py_XDECREF(ident); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } static int encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, - PyObject *seq, Py_ssize_t indent_level) + PyObject *seq, PyObject *newline_indent) { PyObject *ident = NULL; PyObject *s_fast = NULL; Py_ssize_t i; + PyObject *new_newline_indent = NULL; + PyObject *separator_indent = NULL; ident = NULL; s_fast = PySequence_Fast(seq, "_iterencode_list needs a sequence"); @@ -1643,22 +1686,31 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, if (_PyUnicodeWriter_WriteChar(writer, '[')) goto bail; + + PyObject *separator = s->item_separator; // borrowed reference if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ - indent_level += 1; - /* - newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) - separator = _item_separator + newline_indent - buf += newline_indent - */ + new_newline_indent = PyUnicode_Concat(newline_indent, s->indent); + if (new_newline_indent == NULL) { + goto bail; + } + + if (_PyUnicodeWriter_WriteStr(writer, new_newline_indent) < 0) { + goto bail; + } + + separator_indent = PyUnicode_Concat(separator, new_newline_indent); + if (separator_indent == NULL) { + goto bail; + } + separator = separator_indent; // assign separator with borrowed reference } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); if (i) { - if (_PyUnicodeWriter_WriteStr(writer, s->item_separator)) + if (_PyUnicodeWriter_WriteStr(writer, separator) < 0) goto bail; } - if (encoder_listencode_obj(s, writer, obj, indent_level)) + if (encoder_listencode_obj(s, writer, obj, new_newline_indent)) goto bail; } if (ident != NULL) { @@ -1667,12 +1719,14 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, Py_CLEAR(ident); } - /* TODO: DOES NOT RUN if (s->indent != Py_None) { - indent_level -= 1; + Py_CLEAR(new_newline_indent); + Py_CLEAR(separator_indent); + if (_PyUnicodeWriter_WriteStr(writer, newline_indent) < 0) { + goto bail; + } + } - yield '\n' + (' ' * (_indent * _current_indent_level)) - }*/ if (_PyUnicodeWriter_WriteChar(writer, ']')) goto bail; Py_DECREF(s_fast); @@ -1681,6 +1735,8 @@ encoder_listencode_list(PyEncoderObject *s, _PyUnicodeWriter *writer, bail: Py_XDECREF(ident); Py_DECREF(s_fast); + Py_XDECREF(separator_indent); + Py_XDECREF(new_newline_indent); return -1; } @@ -1721,7 +1777,7 @@ encoder_clear(PyEncoderObject *self) return 0; } -PyDoc_STRVAR(encoder_doc, "_iterencode(obj, _current_indent_level) -> iterable"); +PyDoc_STRVAR(encoder_doc, "Encoder(markers, default, encoder, indent, key_separator, item_separator, sort_keys, skipkeys, allow_nan)"); static PyType_Slot PyEncoderType_slots[] = { {Py_tp_doc, (void *)encoder_doc}, From 716ec4bfcf1a564db9936122c442baa99f9c4a8c Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 6 May 2024 10:39:43 +0200 Subject: [PATCH 142/220] gh-115119: Bump CI to use Ubuntu 22.04 (#118631) Ubuntu 22.04 ships with mpdecimal 2.5.1, installable using 'apt install libmpdec-dev'. --- .github/workflows/build.yml | 6 +++--- .github/workflows/reusable-ubuntu.yml | 2 +- .github/workflows/reusable-wasi.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 299c02e0944d429..aac395d5638750e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -250,7 +250,7 @@ jobs: build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' @@ -316,7 +316,7 @@ jobs: test_hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' @@ -429,7 +429,7 @@ jobs: build_asan: name: 'Address sanitizer' - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 021859179af7ccc..fa450ed33763219 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -12,7 +12,7 @@ jobs: build_ubuntu_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.13 diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 4a509a8acfee961..c389fe9e173b38d 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -9,7 +9,7 @@ jobs: build_wasi_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: WASMTIME_VERSION: 18.0.3 WASI_SDK_VERSION: 21 From 153b3f75306b5d26e29ea157105d0fdc247ef853 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 12:02:37 +0300 Subject: [PATCH 143/220] gh-118465: Add __firstlineno__ attribute to class (GH-118475) It is set by compiler with the line number of the first line of the class definition. --- Doc/reference/datamodel.rst | 4 + Doc/whatsnew/3.13.rst | 5 ++ .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 3 + Lib/enum.py | 2 +- Lib/importlib/_bootstrap_external.py | 3 +- Lib/inspect.py | 83 ++----------------- Lib/pydoc.py | 2 +- Lib/test/test_compile.py | 5 +- Lib/test/test_descr.py | 8 +- Lib/test/test_inspect/test_inspect.py | 15 ++++ Lib/test/test_metaclass.py | 8 +- Lib/typing.py | 2 +- ...-05-01-17-12-36.gh-issue-118465.g3Q8iE.rst | 2 + Python/compile.c | 5 ++ 17 files changed, 61 insertions(+), 89 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-01-17-12-36.gh-issue-118465.g3Q8iE.rst diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index f9438a141657be1..f5e871607320568 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -971,6 +971,7 @@ A class object can be called (see above) to yield a class instance (see below). single: __annotations__ (class attribute) single: __type_params__ (class attribute) single: __static_attributes__ (class attribute) + single: __firstlineno__ (class attribute) Special attributes: @@ -1005,6 +1006,9 @@ Special attributes: A tuple containing names of attributes of this class which are accessed through ``self.X`` from any function in its body. + :attr:`__firstlineno__` + The line number of the first line of the class definition, including decorators. + Class instances --------------- diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 11c3f931909a347..558565ccbbeeb51 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -328,6 +328,11 @@ Other Language Changes class scopes are not inlined into their parent scope. (Contributed by Jelle Zijlstra in :gh:`109118` and :gh:`118160`.) +* Classes have a new :attr:`!__firstlineno__` attribute, + populated by the compiler, with the line number of the first line + of the class definition. + (Contributed by Serhiy Storchaka in :gh:`118465`.) + * ``from __future__ import ...`` statements are now just normal relative imports if dots are present before the module name. (Contributed by Jeremiah Gabriel Pascual in :gh:`118216`.) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 4a6f40c84088e84..ca7355b2b61aa75 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -624,6 +624,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__eq__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__exit__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__file__)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__firstlineno__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__float__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__floordiv__)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(__format__)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 8332cdf874c0c98..fbb25285f0f282a 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -113,6 +113,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(__eq__) STRUCT_FOR_ID(__exit__) STRUCT_FOR_ID(__file__) + STRUCT_FOR_ID(__firstlineno__) STRUCT_FOR_ID(__float__) STRUCT_FOR_ID(__floordiv__) STRUCT_FOR_ID(__format__) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 103279a4cf228b1..508da40c53422d8 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -622,6 +622,7 @@ extern "C" { INIT_ID(__eq__), \ INIT_ID(__exit__), \ INIT_ID(__file__), \ + INIT_ID(__firstlineno__), \ INIT_ID(__float__), \ INIT_ID(__floordiv__), \ INIT_ID(__format__), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index a180054d407b39a..cc2fc15ac5cabff 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -180,6 +180,9 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(__file__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); + string = &_Py_ID(__firstlineno__); + assert(_PyUnicode_CheckConsistency(string, 1)); + _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(__float__); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/enum.py b/Lib/enum.py index 98a49eafbb98973..5485306c18843b2 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -2035,7 +2035,7 @@ def _test_simple_enum(checked_enum, simple_enum): ) for key in set(checked_keys + simple_keys): if key in ('__module__', '_member_map_', '_value2member_map_', '__doc__', - '__static_attributes__'): + '__static_attributes__', '__firstlineno__'): # keys known to be different, or very long continue elif key in member_names: diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 0a11dc9efc252c6..30a8cd4475a104e 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -471,6 +471,7 @@ def _write_atomic(path, data, mode=0o666): # Python 3.13a1 3567 (Reimplement line number propagation by the compiler) # Python 3.13a1 3568 (Change semantics of END_FOR) # Python 3.13a5 3569 (Specialize CONTAINS_OP) +# Python 3.13a6 3570 (Add __firstlineno__ class attribute) # Python 3.14 will start with 3600 @@ -487,7 +488,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3569).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3570).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/inspect.py b/Lib/inspect.py index a0c80bd5c8b6014..84260b251a4fb89 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1035,79 +1035,6 @@ class ClassFoundException(Exception): pass -class _ClassFinder(ast.NodeVisitor): - - def __init__(self, cls, tree, lines, qualname): - self.stack = [] - self.cls = cls - self.tree = tree - self.lines = lines - self.qualname = qualname - self.lineno_found = [] - - def visit_FunctionDef(self, node): - self.stack.append(node.name) - self.stack.append('') - self.generic_visit(node) - self.stack.pop() - self.stack.pop() - - visit_AsyncFunctionDef = visit_FunctionDef - - def visit_ClassDef(self, node): - self.stack.append(node.name) - if self.qualname == '.'.join(self.stack): - # Return the decorator for the class if present - if node.decorator_list: - line_number = node.decorator_list[0].lineno - else: - line_number = node.lineno - - # decrement by one since lines starts with indexing by zero - self.lineno_found.append((line_number - 1, node.end_lineno)) - self.generic_visit(node) - self.stack.pop() - - def get_lineno(self): - self.visit(self.tree) - lineno_found_number = len(self.lineno_found) - if lineno_found_number == 0: - raise OSError('could not find class definition') - elif lineno_found_number == 1: - return self.lineno_found[0][0] - else: - # We have multiple candidates for the class definition. - # Now we have to guess. - - # First, let's see if there are any method definitions - for member in self.cls.__dict__.values(): - if (isinstance(member, types.FunctionType) and - member.__module__ == self.cls.__module__): - for lineno, end_lineno in self.lineno_found: - if lineno <= member.__code__.co_firstlineno <= end_lineno: - return lineno - - class_strings = [(''.join(self.lines[lineno: end_lineno]), lineno) - for lineno, end_lineno in self.lineno_found] - - # Maybe the class has a docstring and it's unique? - if self.cls.__doc__: - ret = None - for candidate, lineno in class_strings: - if self.cls.__doc__.strip() in candidate: - if ret is None: - ret = lineno - else: - break - else: - if ret is not None: - return ret - - # We are out of ideas, just return the last one found, which is - # slightly better than previous ones - return self.lineno_found[-1][0] - - def findsource(object): """Return the entire source file and starting line number for an object. @@ -1140,11 +1067,11 @@ def findsource(object): return lines, 0 if isclass(object): - qualname = object.__qualname__ - source = ''.join(lines) - tree = ast.parse(source) - class_finder = _ClassFinder(object, tree, lines, qualname) - return lines, class_finder.get_lineno() + try: + firstlineno = object.__firstlineno__ + except AttributeError: + raise OSError('source code not available') + return lines, object.__firstlineno__ - 1 if ismethod(object): object = object.__func__ diff --git a/Lib/pydoc.py b/Lib/pydoc.py index eaaf8249b205bc5..55ccf2152c26cb1 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -326,7 +326,7 @@ def visiblename(name, all=None, obj=None): '__date__', '__doc__', '__file__', '__spec__', '__loader__', '__module__', '__name__', '__package__', '__path__', '__qualname__', '__slots__', '__version__', - '__static_attributes__'}: + '__static_attributes__', '__firstlineno__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 484d72e63411a80..1f4368b15f473c7 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1958,7 +1958,10 @@ def test_column_offset_deduplication(self): def test_load_super_attr(self): source = "class C:\n def __init__(self):\n super().__init__()" - code = compile(source, "", "exec").co_consts[0].co_consts[1] + for const in compile(source, "", "exec").co_consts[0].co_consts: + if isinstance(const, types.CodeType): + code = const + break self.assertOpcodeSourcePositionIs( code, "LOAD_GLOBAL", line=3, end_line=3, column=4, end_column=9 ) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 93f66a721e81087..18144c8cbb2f0af 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5088,7 +5088,8 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', '__static_attributes__', '__weakref__', 'meth']) @@ -5099,7 +5100,7 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 6) + self.assertEqual(len(values), 7) @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') @@ -5109,7 +5110,8 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', '__static_attributes__', '__weakref__', 'meth']) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index d12240353ff8324..82e466e978624fb 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -817,6 +817,21 @@ def monkey(filename, module_globals=None): def test_getsource_on_code_object(self): self.assertSourceEqual(mod.eggs.__code__, 12, 18) + def test_getsource_on_generated_class(self): + A = type('A', (), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + class TestGetsourceInteractive(unittest.TestCase): def test_getclasses_interactive(self): # bpo-44648: simulate a REPL session; diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 70f9c5d9400bf61..b37b7defe84d1cc 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -164,6 +164,7 @@ ... d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['foo'] = 4 d['foo'] = 42 d['bar'] = 123 @@ -183,12 +184,12 @@ ... b = 24 ... meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] kw: [] >>> type(C) is dict True >>> print(sorted(C.items())) - [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] + [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 42), ('b', 24)] >>> And again, with a __prepare__ attribute. @@ -206,12 +207,13 @@ prepare: C () [('other', 'booh')] d['__module__'] = 'test.test_metaclass' d['__qualname__'] = 'C' + d['__firstlineno__'] = 1 d['a'] = 1 d['a'] = 2 d['b'] = 3 d['__static_attributes__'] = () meta: C () - ns: [('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] + ns: [('__firstlineno__', 1), ('__module__', 'test.test_metaclass'), ('__qualname__', 'C'), ('__static_attributes__', ()), ('a', 2), ('b', 3)] kw: [('other', 'booh')] >>> diff --git a/Lib/typing.py b/Lib/typing.py index 3f6ff491e7b918c..ff0e9b811a58670 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1860,7 +1860,7 @@ class _TypingEllipsis: '__abstractmethods__', '__annotations__', '__dict__', '__doc__', '__init__', '__module__', '__new__', '__slots__', '__subclasshook__', '__weakref__', '__class_getitem__', - '__match_args__', '__static_attributes__', + '__match_args__', '__static_attributes__', '__firstlineno__', }) # These special attributes will be not collected as protocol members. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-01-17-12-36.gh-issue-118465.g3Q8iE.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-17-12-36.gh-issue-118465.g3Q8iE.rst new file mode 100644 index 000000000000000..705a90ed58bda0a --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-17-12-36.gh-issue-118465.g3Q8iE.rst @@ -0,0 +1,2 @@ +Compiler populates the new ``__firstlineno__`` field on a class with the +line number of the first line of the class definition. diff --git a/Python/compile.c b/Python/compile.c index 35a7848f021c314..79f3baadca6b4ad 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2502,6 +2502,11 @@ compiler_class_body(struct compiler *c, stmt_ty s, int firstlineno) compiler_exit_scope(c); return ERROR; } + ADDOP_LOAD_CONST_NEW(c, loc, PyLong_FromLong(c->u->u_metadata.u_firstlineno)); + if (compiler_nameop(c, loc, &_Py_ID(__firstlineno__), Store) < 0) { + compiler_exit_scope(c); + return ERROR; + } asdl_type_param_seq *type_params = s->v.ClassDef.type_params; if (asdl_seq_LEN(type_params) > 0) { if (!compiler_set_type_params_in_class(c, loc)) { From 8e750b83a89ad62a6a209357dcbadc91a2ceaedb Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 6 May 2024 11:50:42 +0200 Subject: [PATCH 144/220] gh-111201: Fix event queue tests for pyrepl (#118635) --- Lib/test/test_pyrepl.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index c53bdef0ca9cb19..3cba37c70e95574 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -2,6 +2,7 @@ import os import rlcompleter import sys +import tempfile import unittest from code import InteractiveConsole from functools import partial @@ -603,26 +604,32 @@ def test_global_namespace_completion(self): @patch("_pyrepl.curses.tigetstr", lambda x: b"") class TestUnivEventQueue(TestCase): + def setUp(self): + self.file = tempfile.TemporaryFile() + + def tearDown(self) -> None: + self.file.close() + def test_get(self): - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") event = Event("key", "a", b"a") eq.insert(event) self.assertEqual(eq.get(), event) def test_empty(self): - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") self.assertTrue(eq.empty()) eq.insert(Event("key", "a", b"a")) self.assertFalse(eq.empty()) def test_flush_buf(self): - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.buf.extend(b"test") self.assertEqual(eq.flush_buf(), b"test") self.assertEqual(eq.buf, bytearray()) def test_insert(self): - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") event = Event("key", "a", b"a") eq.insert(event) self.assertEqual(eq.events[0], event) @@ -630,30 +637,30 @@ def test_insert(self): @patch("_pyrepl.unix_eventqueue.keymap") def test_push_with_key_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.keymap = {b"a": "b"} eq.push("a") - self.assertTrue(mock_keymap.compile_keymap.called) + mock_keymap.compile_keymap.assert_called() self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "b") @patch("_pyrepl.unix_eventqueue.keymap") def test_push_without_key_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.keymap = {b"c": "d"} eq.push("a") - self.assertTrue(mock_keymap.compile_keymap.called) + mock_keymap.compile_keymap.assert_called() self.assertEqual(eq.events[0].evt, "key") self.assertEqual(eq.events[0].data, "a") @patch("_pyrepl.unix_eventqueue.keymap") def test_push_with_keymap_in_keymap(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.keymap = {b"a": {b"b": "c"}} eq.push("a") - self.assertTrue(mock_keymap.compile_keymap.called) + mock_keymap.compile_keymap.assert_called() self.assertTrue(eq.empty()) eq.push("b") self.assertEqual(eq.events[0].evt, "key") @@ -665,10 +672,10 @@ def test_push_with_keymap_in_keymap(self, mock_keymap): @patch("_pyrepl.unix_eventqueue.keymap") def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): mock_keymap.compile_keymap.return_value = {"a": "b"} - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.keymap = {b"a": {b"b": "c"}} eq.push("a") - self.assertTrue(mock_keymap.compile_keymap.called) + mock_keymap.compile_keymap.assert_called() self.assertTrue(eq.empty()) eq.flush_buf() eq.push("\033") @@ -679,7 +686,7 @@ def test_push_with_keymap_in_keymap_and_escape(self, mock_keymap): self.assertEqual(eq.events[1].data, "b") def test_push_special_key(self): - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.keymap = {} eq.push("\x1b") eq.push("[") @@ -688,7 +695,7 @@ def test_push_special_key(self): self.assertEqual(eq.events[0].data, "\x1b") def test_push_unrecognized_escape_sequence(self): - eq = EventQueue(sys.stdout.fileno(), "utf-8") + eq = EventQueue(self.file.fileno(), "utf-8") eq.keymap = {} eq.push("\x1b") eq.push("[") From 757fd3e0101c85081dfec60cbab26519b7a40305 Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Mon, 6 May 2024 20:06:07 +1000 Subject: [PATCH 145/220] gh-74929: locals() documentation update for PEP 667 (#118265) Documentation wording taken from the withdrawn PEP 558 (which had switched to the Python level semantics proposed in PEP 667 prior to its withdrawal). --- Doc/library/functions.rst | 47 +++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 4fdf30e0d252ae1..0c7ef67774cd057 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -636,8 +636,7 @@ are always available. They are listed here in alphabetical order. .. note:: - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. + The default *locals* act as described for function :func:`locals` below. Pass an explicit *locals* dictionary if you need to see effects of the code on *locals* after function :func:`exec` returns. @@ -1049,14 +1048,44 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. Note that at the module level, :func:`locals` - and :func:`globals` are the same dictionary. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using ``exec()`` or ``eval()`` with a + single namespace, this function returns the same namespace as ``globals()``. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + namespaces, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as bound, + rebound, or deleted local variables, and binding, rebinding, or deleting + local variables will immediately affect the contents of the returned mapping + object. + + At function scope (including for generators and coroutines), each call to + ``locals()`` instead returns a fresh dictionary containing the current + bindings of the function's local variables and any nonlocal cell references. + In this case, name binding changes made via the returned dict are *not* + written back to the corresponding local variables or nonlocal cell + references, and binding, rebinding, or deleting local variables and nonlocal + cell references does *not* affect the contents of previously returned + dictionaries. + + .. versionchanged:: 3.13 + In previous versions, the semantics of mutating the mapping object + returned from this function were formally undefined. In CPython + specifically, the mapping returned at function scope could be + implicitly refreshed by other operations, such as calling ``locals()`` + again. Obtaining the legacy CPython behaviour now requires explicit + calls to update the initially returned dictionary with the results + of subsequent calls to ``locals()``. - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. .. function:: map(function, iterable, *iterables) From afbe5bf9c8c621bb19833a06e296245c41f0dbc5 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 6 May 2024 03:16:48 -0700 Subject: [PATCH 146/220] gh-74929: Make containment checks more efficient in `FrameLocalsProxy` (#118624) Properly implement the `sq_contains` slot for frame locals proxy containment checks. --- Objects/frameobject.c | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index fdac86961f860c0..aac809b459a430e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -488,7 +488,7 @@ framelocalsproxy_length(PyObject *self) return size; } -static PyObject* +static int framelocalsproxy_contains(PyObject *self, PyObject *key) { PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; @@ -496,21 +496,25 @@ framelocalsproxy_contains(PyObject *self, PyObject *key) if (PyUnicode_CheckExact(key)) { int i = framelocalsproxy_getkeyindex(frame, key, true); if (i >= 0) { - Py_RETURN_TRUE; + return 1; } } PyObject *extra = ((PyFrameObject*)frame)->f_extra_locals; if (extra != NULL) { - int result = PyDict_Contains(extra, key); - if (result < 0) { - return NULL; - } else if (result > 0) { - Py_RETURN_TRUE; - } + return PyDict_Contains(extra, key); } - Py_RETURN_FALSE; + return 0; +} + +static PyObject* framelocalsproxy___contains__(PyObject *self, PyObject *key) +{ + int result = framelocalsproxy_contains(self, key); + if (result < 0) { + return NULL; + } + return PyBool_FromLong(result); } static PyObject* @@ -604,6 +608,10 @@ static PyNumberMethods framelocalsproxy_as_number = { .nb_inplace_or = framelocalsproxy_inplace_or, }; +static PySequenceMethods framelocalsproxy_as_sequence = { + .sq_contains = framelocalsproxy_contains, +}; + static PyMappingMethods framelocalsproxy_as_mapping = { framelocalsproxy_length, // mp_length framelocalsproxy_getitem, // mp_subscript @@ -611,7 +619,7 @@ static PyMappingMethods framelocalsproxy_as_mapping = { }; static PyMethodDef framelocalsproxy_methods[] = { - {"__contains__", framelocalsproxy_contains, METH_O | METH_COEXIST, + {"__contains__", framelocalsproxy___contains__, METH_O | METH_COEXIST, NULL}, {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, NULL}, @@ -639,6 +647,7 @@ PyTypeObject PyFrameLocalsProxy_Type = { .tp_dealloc = (destructor)framelocalsproxy_dealloc, .tp_repr = &framelocalsproxy_repr, .tp_as_number = &framelocalsproxy_as_number, + .tp_as_sequence = &framelocalsproxy_as_sequence, .tp_as_mapping = &framelocalsproxy_as_mapping, .tp_getattro = PyObject_GenericGetAttr, .tp_setattro = PyObject_GenericSetAttr, From d3c7821335f88d92297622ad1ec17b33b9baf5cd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 6 May 2024 11:56:47 +0100 Subject: [PATCH 147/220] gh-111201: Use a more common constant in completion tests in test_pyrepl (#118638) --- Lib/test/test_pyrepl.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 3cba37c70e95574..2b217abdf1543b4 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -578,13 +578,18 @@ def test_simple_completion(self): self.assertEqual(output, "os.getenv") def test_completion_with_many_options(self): - events = code_to_events("os.\t\tO_AS\t\n") + # Test with something that initially displays many options + # and then complete from one of them. The first time tab is + # pressed, the options are displayed (which corresponds to + # when the repl shows [ not unique ]) and the second completes + # from one of them. + events = code_to_events("os.\t\tO_AP\t\n") namespace = {"os": os} reader = self.prepare_reader(events, namespace) output = multiline_input(reader, namespace) - self.assertEqual(output, "os.O_ASYNC") + self.assertEqual(output, "os.O_APPEND") def test_empty_namespace_completion(self): events = code_to_events("os.geten\t\n") From d6fa1d4beef2bf9d83048469667e0ba5f2b41068 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 15:50:52 +0300 Subject: [PATCH 148/220] gh-66543: Add mimetypes.guess_file_type() (GH-117258) --- Doc/includes/email-dir.py | 2 +- Doc/library/mimetypes.rst | 31 +++++++- Doc/library/wsgiref.rst | 2 +- Doc/whatsnew/3.13.rst | 11 +++ Lib/http/server.py | 2 +- Lib/mimetypes.py | 34 ++++++-- Lib/test/test_mimetypes.py | 79 +++++++++++++------ ...4-03-26-15-29-39.gh-issue-66543.OZBhU5.rst | 3 + 8 files changed, 129 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-03-26-15-29-39.gh-issue-66543.OZBhU5.rst diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py index 2fc1570e654db6a..aa2a5c7cda52aae 100644 --- a/Doc/includes/email-dir.py +++ b/Doc/includes/email-dir.py @@ -53,7 +53,7 @@ def main(): # Guess the content type based on the file's extension. Encoding # will be ignored, although we should check for simple things like # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) + ctype, encoding = mimetypes.guess_file_type(path) if ctype is None or encoding is not None: # No guess could be made, or the file is encoded (compressed), so # use a generic bag-of-bits type. diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index f610032acbe4177..a24eab21d573437 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -52,7 +52,22 @@ the information :func:`init` sets up. are also recognized. .. versionchanged:: 3.8 - Added support for url being a :term:`path-like object`. + Added support for *url* being a :term:`path-like object`. + + .. deprecated:: 3.13 + Passing a file path instead of URL is :term:`soft deprecated`. + Use :func:`guess_file_type` for this. + + +.. function:: guess_file_type(path, *, strict=True) + + .. index:: pair: MIME; headers + + Guess the type of a file based on its path, given by *path*. + Similar to the :func:`guess_type` function, but accepts a path instead of URL. + Path can be a string, a bytes object or a :term:`path-like object`. + + .. versionadded:: 3.13 .. function:: guess_all_extensions(type, strict=True) @@ -61,7 +76,7 @@ the information :func:`init` sets up. return value is a list of strings giving all possible filename extensions, including the leading dot (``'.'``). The extensions are not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. + type *type* by :func:`guess_type` and :func:`guess_file_type`. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -72,8 +87,8 @@ the information :func:`init` sets up. return value is a string giving a filename extension, including the leading dot (``'.'``). The extension is not guaranteed to have been associated with any particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. + :func:`guess_type` and :func:`guess_file_type`. + If no extension can be guessed for *type*, ``None`` is returned. The optional *strict* argument has the same meaning as with the :func:`guess_type` function. @@ -238,6 +253,14 @@ than one MIME-type database; it provides an interface similar to the one of the the object. + .. method:: MimeTypes.guess_file_type(path, *, strict=True) + + Similar to the :func:`guess_file_type` function, using the tables stored + as part of the object. + + .. versionadded:: 3.13 + + .. method:: MimeTypes.guess_all_extensions(type, strict=True) Similar to the :func:`guess_all_extensions` function, using the tables stored diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index c2b0ba7046967ed..7fe84a2de1fcebf 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -865,7 +865,7 @@ directory and port number (default: 8000) on the command line:: fn = os.path.join(path, environ["PATH_INFO"][1:]) if "." not in fn.split(os.path.sep)[-1]: fn = os.path.join(fn, "index.html") - mime_type = mimetypes.guess_type(fn)[0] + mime_type = mimetypes.guess_file_type(fn)[0] # Return 200 OK if file exists, otherwise 404 Not Found if os.path.exists(fn): diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 558565ccbbeeb51..0b75665ab9c9ba9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -623,6 +623,13 @@ math "fusedMultiplyAdd" operation for special cases. (Contributed by Mark Dickinson and Victor Stinner in :gh:`73468`.) +mimetypes +--------- + +* Add the :func:`~mimetypes.guess_file_type` function which works with file path. + Passing file path instead of URL in :func:`~mimetypes.guess_type` is :term:`soft deprecated`. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + mmap ---- @@ -1167,6 +1174,10 @@ Deprecated .. Add deprecations above alphabetically, not here at the end. +* Passing file path instead of URL in :func:`~mimetypes.guess_type` is :term:`soft deprecated`. + Use :func:`~mimetypes.guess_file_type` instead. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + Pending Removal in Python 3.14 ------------------------------ diff --git a/Lib/http/server.py b/Lib/http/server.py index ee7a9b6aa55b887..7d0da5052d2d4d5 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -897,7 +897,7 @@ def guess_type(self, path): ext = ext.lower() if ext in self.extensions_map: return self.extensions_map[ext] - guess, _ = mimetypes.guess_type(path) + guess, _ = mimetypes.guess_file_type(path) if guess: return guess return 'application/octet-stream' diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 65a049ae9945f66..8604000ed77a196 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -40,7 +40,7 @@ __all__ = [ "knownfiles", "inited", "MimeTypes", - "guess_type", "guess_all_extensions", "guess_extension", + "guess_type", "guess_file_type", "guess_all_extensions", "guess_extension", "add_type", "init", "read_mime_types", "suffix_map", "encodings_map", "types_map", "common_types" ] @@ -119,14 +119,14 @@ def guess_type(self, url, strict=True): Optional `strict' argument when False adds a bunch of commonly found, but non-standard types. """ + # TODO: Deprecate accepting file paths (in particular path-like objects). url = os.fspath(url) p = urllib.parse.urlparse(url) if p.scheme and len(p.scheme) > 1: scheme = p.scheme url = p.path else: - scheme = None - url = os.path.splitdrive(url)[1] + return self.guess_file_type(url, strict=strict) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data @@ -146,13 +146,25 @@ def guess_type(self, url, strict=True): if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None - base, ext = posixpath.splitext(url) + return self._guess_file_type(url, strict, posixpath.splitext) + + def guess_file_type(self, path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + path = os.fsdecode(path) + path = os.path.splitdrive(path)[1] + return self._guess_file_type(path, strict, os.path.splitext) + + def _guess_file_type(self, path, strict, splitext): + base, ext = splitext(path) while (ext_lower := ext.lower()) in self.suffix_map: - base, ext = posixpath.splitext(base + self.suffix_map[ext_lower]) + base, ext = splitext(base + self.suffix_map[ext_lower]) # encodings_map is case sensitive if ext in self.encodings_map: encoding = self.encodings_map[ext] - base, ext = posixpath.splitext(base) + base, ext = splitext(base) else: encoding = None ext = ext.lower() @@ -310,6 +322,16 @@ def guess_type(url, strict=True): return _db.guess_type(url, strict) +def guess_file_type(path, *, strict=True): + """Guess the type of a file based on its path. + + Similar to guess_type(), but takes file path istead of URL. + """ + if _db is None: + init() + return _db.guess_file_type(path, strict=strict) + + def guess_all_extensions(type, strict=True): """Guess the extensions for a file based on its MIME type. diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index bf6eae7d0ac9c67..2e0ad0606ae9c22 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -36,20 +36,28 @@ def setUp(self): def test_case_sensitivity(self): eq = self.assertEqual - eq(self.db.guess_type("foobar.HTML"), self.db.guess_type("foobar.html")) - eq(self.db.guess_type("foobar.TGZ"), self.db.guess_type("foobar.tgz")) - eq(self.db.guess_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) def test_default_data(self): eq = self.assertEqual - eq(self.db.guess_type("foo.html"), ("text/html", None)) - eq(self.db.guess_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual @@ -63,7 +71,7 @@ def test_file_parsing(self): eq = self.assertEqual sio = io.StringIO("x-application/x-unittest pyunit\n") self.db.readfp(sio) - eq(self.db.guess_type("foo.pyunit"), + eq(self.db.guess_file_type("foo.pyunit"), ("x-application/x-unittest", None)) eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") @@ -95,12 +103,12 @@ def test_read_mime_types(self): def test_non_standard_types(self): eq = self.assertEqual # First try strict - eq(self.db.guess_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) eq(self.db.guess_extension('image/jpg', strict=True), None) # And then non-strict - eq(self.db.guess_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') @@ -124,15 +132,26 @@ def test_filename_with_url_delimiters(self): '//share/server/', '\\\\share\\server\\'): path = prefix + name with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) eq(self.db.guess_type(path), gzip_expected) expected = (None, None) if os.name == 'nt' else gzip_expected for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): path = prefix + name with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') result = self.db.guess_type('http://host.html') msg = 'URL only has a host name, not a file' self.assertSequenceEqual(result, (None, None), msg) @@ -242,22 +261,38 @@ def test_init_stability(self): def test_path_like_ob(self): filename = "LICENSE.txt" - filepath = pathlib.Path(filename) - filepath_with_abs_dir = pathlib.Path('/dir/'+filename) - filepath_relative = pathlib.Path('../dir/'+filename) - path_dir = pathlib.Path('./') + filepath = os_helper.FakePath(filename) + filepath_with_abs_dir = os_helper.FakePath('/dir/'+filename) + filepath_relative = os_helper.FakePath('../dir/'+filename) + path_dir = os_helper.FakePath('./') - expected = self.db.guess_type(filename) + expected = self.db.guess_file_type(filename) + self.assertEqual(self.db.guess_file_type(filepath), expected) self.assertEqual(self.db.guess_type(filepath), expected) + self.assertEqual(self.db.guess_file_type( + filepath_with_abs_dir), expected) self.assertEqual(self.db.guess_type( filepath_with_abs_dir), expected) + self.assertEqual(self.db.guess_file_type(filepath_relative), expected) self.assertEqual(self.db.guess_type(filepath_relative), expected) + + self.assertEqual(self.db.guess_file_type(path_dir), (None, None)) self.assertEqual(self.db.guess_type(path_dir), (None, None)) + def test_bytes_path(self): + self.assertEqual(self.db.guess_file_type(b'foo.html'), + self.db.guess_file_type('foo.html')) + self.assertEqual(self.db.guess_file_type(b'foo.tar.gz'), + self.db.guess_file_type('foo.tar.gz')) + self.assertEqual(self.db.guess_file_type(b'foo.tgz'), + self.db.guess_file_type('foo.tgz')) + def test_keywords_args_api(self): + self.assertEqual(self.db.guess_file_type( + path="foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_type( - url="foo.html", strict=True), ("text/html", None)) + url="scheme:foo.html", strict=True), ("text/html", None)) self.assertEqual(self.db.guess_all_extensions( type='image/jpg', strict=True), []) self.assertEqual(self.db.guess_extension( diff --git a/Misc/NEWS.d/next/Library/2024-03-26-15-29-39.gh-issue-66543.OZBhU5.rst b/Misc/NEWS.d/next/Library/2024-03-26-15-29-39.gh-issue-66543.OZBhU5.rst new file mode 100644 index 000000000000000..12ea50858142816 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-26-15-29-39.gh-issue-66543.OZBhU5.rst @@ -0,0 +1,3 @@ +Add the :func:`mimetypes.guess_file_type` function which works with file +path. Passing file path instead of URL in :func:`~mimetypes.guess_type` is +:term:`soft deprecated`. From 0085c3ae8f067abd4f6540d0f6dd2fb13107618e Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 15:53:15 +0300 Subject: [PATCH 149/220] gh-116871: Improve name suggestions in tracebacks (GH-116930) Only include underscored names in name suggestions for AttributeError and ImportError if the original name was underscored. --- Lib/test/test_traceback.py | 32 +++++++++++++++++++ Lib/traceback.py | 11 +++++++ ...-03-17-18-24-23.gh-issue-116871.9uSl8M.rst | 2 ++ 3 files changed, 45 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-03-17-18-24-23.gh-issue-116871.9uSl8M.rst diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 7e4851058e0109d..5987ec382e6c853 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -3882,6 +3882,27 @@ class CaseChangeOverSubstitution: actual = self.get_suggestion(cls(), 'bluch') self.assertIn(suggestion, actual) + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None @@ -4074,6 +4095,17 @@ def test_import_from_suggestions(self): actual = self.get_import_from_suggestion(code, 'bluch') self.assertIn(suggestion, actual) + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" diff --git a/Lib/traceback.py b/Lib/traceback.py index 1878779e154d048..9401b461497cc1e 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -1469,12 +1469,23 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): obj = exc_value.obj try: d = dir(obj) + hide_underscored = (wrong_name[:1] != '_') + if hide_underscored and tb is not None: + while tb.tb_next is not None: + tb = tb.tb_next + frame = tb.tb_frame + if 'self' in frame.f_locals and frame.f_locals['self'] is obj: + hide_underscored = False + if hide_underscored: + d = [x for x in d if x[:1] != '_'] except Exception: return None elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) d = dir(mod) + if wrong_name[:1] != '_': + d = [x for x in d if x[:1] != '_'] except Exception: return None else: diff --git a/Misc/NEWS.d/next/Library/2024-03-17-18-24-23.gh-issue-116871.9uSl8M.rst b/Misc/NEWS.d/next/Library/2024-03-17-18-24-23.gh-issue-116871.9uSl8M.rst new file mode 100644 index 000000000000000..f3caa63187d78bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-17-18-24-23.gh-issue-116871.9uSl8M.rst @@ -0,0 +1,2 @@ +Name suggestions for :exc:`AttributeError` and :exc:`ImportError` now only +include underscored names if the original name was underscored. From c3f4a6b52418d9b9f091f864cb6340d0d5fc6966 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 6 May 2024 21:58:27 +0800 Subject: [PATCH 150/220] Fix typo in Lib/zipfile/_path/__init__.py (#118622) --- Lib/zipfile/_path/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 4c167563b6b7621..79ebb777354e032 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -262,7 +262,7 @@ class Path: >>> str(path.parent) 'mem' - If the zipfile has no filename, such attribtues are not + If the zipfile has no filename, such attributes are not valid and accessing them will raise an Exception. >>> zf.filename = None From 09871c922393cba4c85bc29d210d76425e076c1d Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 6 May 2024 15:30:30 +0100 Subject: [PATCH 151/220] gh-118518: Correct type of perf_profiling in config (#118646) --- Python/initconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 1880a28fea81f69..0e53d60e3537af3 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -60,7 +60,7 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(hash_seed, ULONG), SPEC(faulthandler, BOOL), SPEC(tracemalloc, UINT), - SPEC(perf_profiling, BOOL), + SPEC(perf_profiling, UINT), SPEC(import_time, BOOL), SPEC(code_debug_ranges, BOOL), SPEC(show_ref_count, BOOL), From 1b639a04cab0e858d90e2ac459fb34b73700701f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 17:33:15 +0300 Subject: [PATCH 152/220] gh-118225: Support more options for copying images in Tkinter (GH-118228) * Add the PhotoImage method copy_replace() to copy a region from one image to other image, possibly with pixel zooming and/or subsampling. * Add from_coords parameter to PhotoImage methods copy(), zoom() and subsample(). * Add zoom and subsample parameters to PhotoImage method copy(). --- Doc/library/tkinter.rst | 9 ++ Doc/whatsnew/3.13.rst | 9 ++ Lib/test/test_tkinter/test_images.py | 151 +++++++++++++++++- Lib/tkinter/__init__.py | 111 +++++++++++-- ...-04-24-16-07-26.gh-issue-118225.KdrcgL.rst | 5 + 5 files changed, 268 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-24-16-07-26.gh-issue-118225.KdrcgL.rst diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index e084d8554c7c09f..7e5dee1b562df8b 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -979,6 +979,15 @@ of :class:`tkinter.Image`: Either type of image is created through either the ``file`` or the ``data`` option (other options are available as well). +.. versionchanged:: 3.13 + Added the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy()`, + :meth:`!zoom()` and :meth:`!subsample()`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy()`. + The image object can then be used wherever an ``image`` option is supported by some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0b75665ab9c9ba9..67e7fbebc92ef46 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -882,6 +882,15 @@ tkinter * Add the :meth:`!after_info` method for Tkinter widgets. (Contributed by Cheryl Sabella in :gh:`77020`.) +* Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region + from one image to other image, possibly with pixel zooming and/or + subsampling. + Add *from_coords* parameter to :class:`!PhotoImage` methods :meth:`!copy()`, + :meth:`!zoom()` and :meth:`!subsample()`. + Add *zoom* and *subsample* parameters to :class:`!PhotoImage` method + :meth:`!copy()`. + (Contributed by Serhiy Storchaka in :gh:`118225`.) + traceback --------- diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index ef1c99f57c6f47f..2a59d014f4aa2a0 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -302,7 +302,37 @@ def test_copy(self): image2 = image.copy() self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) - self.assertEqual(image.get(4, 6), image.get(4, 6)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = image.copy(from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(3, 2), image.get(3*2+2, 2*2+3)) + + image2 = image.copy(from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) def test_subsample(self): image = self.create() @@ -316,6 +346,13 @@ def test_subsample(self): self.assertEqual(image2.height(), 8) self.assertEqual(image2.get(2, 3), image.get(4, 6)) + image2 = image.subsample(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + def test_zoom(self): image = self.create() image2 = image.zoom(2, 3) @@ -330,6 +367,118 @@ def test_zoom(self): self.assertEqual(image2.get(8, 12), image.get(4, 6)) self.assertEqual(image2.get(9, 13), image.get(4, 6)) + image2 = image.zoom(2, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + def test_copy_replace(self): + image = self.create() + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), to=(0, 0, 100, 50)) + self.assertEqual(image2.width(), 100) + self.assertEqual(image2.height(), 50) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12, 4+8), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2+12*2, 4), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2, 4+8*3), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), zoom=2) + self.assertEqual(image2.width(), 24) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(23, 15), image.get(13, 10)) + self.assertEqual(image2.get(2*2, 4*2), image.get(2+2, 4+3)) + self.assertEqual(image2.get(2*2+1, 4*2+1), image.get(6+2, 2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2) + self.assertEqual(image2.width(), 6) + self.assertEqual(image2.height(), 4) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(5, 3), image.get(12, 9)) + self.assertEqual(image2.get(1, 2), image.get(1*2+2, 2*2+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.copy_replace(image, from_coords=(2, 3, 14, 11), subsample=2, zoom=3) + self.assertEqual(image2.width(), 18) + self.assertEqual(image2.height(), 12) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(17, 11), image.get(12, 9)) + self.assertEqual(image2.get(3*3, 2*3), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(3*3+2, 2*3+2), image.get(3*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3, 2*3), image.get(1*2+2, 2*2+3)) + self.assertEqual(image2.get(1*3+2, 2*3+2), image.get(1*2+2, 2*2+3)) + + def checkImgTrans(self, image, expected): + actual = {(x, y) + for x in range(image.width()) + for y in range(image.height()) + if image.transparency_get(x, y)} + self.assertEqual(actual, expected) + + def test_copy_replace_compositingrule(self): + image1 = tkinter.PhotoImage(master=self.root, width=2, height=2) + image1.blank() + image1.put('black', to=(0, 0, 2, 2)) + image1.transparency_set(0, 0, True) + + # default compositingrule + image2 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image2.blank() + image2.put('white', to=(0, 0, 2, 2)) + image2.copy_replace(image1, to=(1, 1)) + self.checkImgTrans(image2, {(0, 2), (2, 0)}) + + image3 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image3.blank() + image3.put('white', to=(0, 0, 2, 2)) + image3.copy_replace(image1, to=(1, 1), compositingrule='overlay') + self.checkImgTrans(image3, {(0, 2), (2, 0)}) + + image4 = tkinter.PhotoImage(master=self.root, width=3, height=3) + image4.blank() + image4.put('white', to=(0, 0, 2, 2)) + image4.copy_replace(image1, to=(1, 1), compositingrule='set') + self.checkImgTrans(image4, {(0, 2), (1, 1), (2, 0)}) + def test_put(self): image = self.create() image.put('{red green} {blue yellow}', to=(4, 6)) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 70a1ed46fd07743..35da83107bd99ef 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4278,33 +4278,112 @@ def cget(self, option): def __getitem__(self, key): return self.tk.call(self.name, 'cget', '-' + key) - # XXX copy -from, -to, ...? - def copy(self): - """Return a new PhotoImage with the same image as this widget.""" + def copy(self, *, from_coords=None, zoom=None, subsample=None): + """Return a new PhotoImage with the same image as this widget. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + """ destImage = PhotoImage(master=self.tk) - self.tk.call(destImage, 'copy', self.name) + destImage.copy_replace(self, from_coords=from_coords, + zoom=zoom, subsample=subsample) return destImage - def zoom(self, x, y=''): + def zoom(self, x, y='', *, from_coords=None): """Return a new PhotoImage with the same image as this widget - but zoom it with a factor of x in the X direction and y in the Y - direction. If y is not given, the default value is the same as x. + but zoom it with a factor of X in the X direction and Y in the Y + direction. If Y is not given, the default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-zoom',x,y) - return destImage + return self.copy(zoom=(x, y), from_coords=from_coords) - def subsample(self, x, y=''): + def subsample(self, x, y='', *, from_coords=None): """Return a new PhotoImage based on the same image as this widget - but use only every Xth or Yth pixel. If y is not given, the - default value is the same as x. + but use only every Xth or Yth pixel. If Y is not given, the + default value is the same as X. + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied, as in the copy() method. """ - destImage = PhotoImage(master=self.tk) if y=='': y=x - self.tk.call(destImage, 'copy', self.name, '-subsample',x,y) - return destImage + return self.copy(subsample=(x, y), from_coords=from_coords) + + def copy_replace(self, sourceImage, *, from_coords=None, to=None, shrink=False, + zoom=None, subsample=None, compositingrule=None): + """Copy a region from the source image (which must be a PhotoImage) to + this image, possibly with pixel zooming and/or subsampling. If no + options are specified, this command copies the whole of the source + image into this image, starting at coordinates (0, 0). + + The FROM_COORDS option specifies a rectangular sub-region of the + source image to be copied. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is the bottom-right corner of the source image. + The pixels copied will include the left and top edges of the + specified rectangle but not the bottom or right edges. If the + FROM_COORDS option is not given, the default is the whole source + image. + + The TO option specifies a rectangular sub-region of the destination + image to be affected. It must be a tuple or a list of 1 to 4 + integers (x1, y1, x2, y2). (x1, y1) and (x2, y2) specify diagonally + opposite corners of the rectangle. If x2 and y2 are not specified, + the default value is (x1,y1) plus the size of the source region + (after subsampling and zooming, if specified). If x2 and y2 are + specified, the source region will be replicated if necessary to fill + the destination region in a tiled fashion. + + If SHRINK is true, the size of the destination image should be + reduced, if necessary, so that the region being copied into is at + the bottom-right corner of the image. + + If SUBSAMPLE or ZOOM are specified, the image is transformed as in + the subsample() or zoom() methods. The value must be a single + integer or a pair of integers. + + The COMPOSITINGRULE option specifies how transparent pixels in the + source image are combined with the destination image. When a + compositing rule of 'overlay' is set, the old contents of the + destination image are visible, as if the source image were printed + on a piece of transparent film and placed over the top of the + destination. When a compositing rule of 'set' is set, the old + contents of the destination image are discarded and the source image + is used as-is. The default compositing rule is 'overlay'. + """ + options = [] + if from_coords is not None: + options.extend(('-from', *from_coords)) + if to is not None: + options.extend(('-to', *to)) + if shrink: + options.append('-shrink') + if zoom is not None: + if not isinstance(zoom, (tuple, list)): + zoom = (zoom,) + options.extend(('-zoom', *zoom)) + if subsample is not None: + if not isinstance(subsample, (tuple, list)): + subsample = (subsample,) + options.extend(('-subsample', *subsample)) + if compositingrule: + options.extend(('-compositingrule', compositingrule)) + self.tk.call(self.name, 'copy', sourceImage, *options) def get(self, x, y): """Return the color (red, green, blue) of the pixel at X,Y.""" diff --git a/Misc/NEWS.d/next/Library/2024-04-24-16-07-26.gh-issue-118225.KdrcgL.rst b/Misc/NEWS.d/next/Library/2024-04-24-16-07-26.gh-issue-118225.KdrcgL.rst new file mode 100644 index 000000000000000..a4671a301abb8ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-24-16-07-26.gh-issue-118225.KdrcgL.rst @@ -0,0 +1,5 @@ +Add the :class:`!PhotoImage` method :meth:`!copy_replace` to copy a region +from one image to other image, possibly with pixel zooming and/or +subsampling. Add *from_coords* parameter to :class:`!PhotoImage` methods +:meth:`!copy()`, :meth:`!zoom()` and :meth:`!subsample()`. Add *zoom* and +*subsample* parameters to :class:`!PhotoImage` method :meth:`!copy()`. From fc50f1bdbad3aa52d7cbd3cb836a35806266ec54 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 6 May 2024 23:00:31 +0800 Subject: [PATCH 153/220] Fix typo in Doc/library/asyncio-task.rst (GH-118627) --- Doc/library/asyncio-task.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 3d300c37419f13e..188649a2968df8c 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -1386,7 +1386,7 @@ Task Object with :meth:`uncancel`. :class:`TaskGroup` context managers use :func:`uncancel` in a similar fashion. - If end-user code is, for some reason, suppresing cancellation by + If end-user code is, for some reason, suppressing cancellation by catching :exc:`CancelledError`, it needs to call this method to remove the cancellation state. From 709ca90a00e66cea432096a7ba61aa6459d2a9a7 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 18:06:06 +0300 Subject: [PATCH 154/220] gh-118271: Support more options for reading/writing images in Tkinter (GH-118273) * Add PhotoImage.read() to read an image from a file. * Add PhotoImage.data() to get the image data. * Add background and grayscale parameters to PhotoImage.write(). --- Doc/whatsnew/3.13.rst | 6 + Lib/test/test_tkinter/test_images.py | 106 ++++++++++++++- Lib/tkinter/__init__.py | 122 ++++++++++++++++-- ...-04-25-11-49-11.gh-issue-118271.5N2Xcy.rst | 4 + 4 files changed, 222 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-25-11-49-11.gh-issue-118271.5N2Xcy.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 67e7fbebc92ef46..d573d8109edef86 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -891,6 +891,12 @@ tkinter :meth:`!copy()`. (Contributed by Serhiy Storchaka in :gh:`118225`.) +* Add the :class:`!PhotoImage` methods :meth:`!read` to read + an image from a file and :meth:`!data` to get the image data. + Add *background* and *grayscale* parameters to :class:`!PhotoImage` method + :meth:`!write`. + (Contributed by Serhiy Storchaka in :gh:`118271`.) + traceback --------- diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index 2a59d014f4aa2a0..b8e549e314d27d9 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -505,6 +505,50 @@ def test_get(self): self.assertRaises(tkinter.TclError, image.get, 16, 15) self.assertRaises(tkinter.TclError, image.get, 15, 16) + def test_read(self): + # Due to the Tk bug https://core.tcl-lang.org/tk/tktview/1576528 + # the -from option does not work correctly for GIF and PNG files. + # Use the PPM file for this test. + testfile = support.findfile('python.ppm', subdir='tkinterdata') + image = tkinter.PhotoImage(master=self.root, file=testfile) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + self.assertRaises(tkinter.TclError, image2.read, self.testfile, 'ppm') + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11)) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root, file=testfile) + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + image2.read(testfile, from_coords=(2, 3, 14, 11), shrink=True) + self.assertEqual(image2.width(), 12) + self.assertEqual(image2.height(), 8) + self.assertEqual(image2.get(0, 0), image.get(2, 3)) + self.assertEqual(image2.get(11, 7), image.get(13, 10)) + self.assertEqual(image2.get(2, 4), image.get(2+2, 4+3)) + + image2 = tkinter.PhotoImage(master=self.root) + image2.read(testfile, from_coords=(2, 3, 14, 11), to=(3, 6)) + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 15) + self.assertEqual(image2.height(), 14) + self.assertEqual(image2.get(0+3, 0+6), image.get(2, 3)) + self.assertEqual(image2.get(11+3, 7+6), image.get(13, 10)) + self.assertEqual(image2.get(2+3, 4+6), image.get(2+2, 4+3)) + def test_write(self): filename = os_helper.TESTFN import locale @@ -516,19 +560,58 @@ def test_write(self): image.write(filename) image2 = tkinter.PhotoImage('::img::test2', master=self.root, - format='ppm', - file=filename) + format='ppm', file=filename) self.assertEqual(str(image2), '::img::test2') self.assertEqual(image2.type(), 'photo') self.assertEqual(image2.width(), 16) self.assertEqual(image2.height(), 16) self.assertEqual(image2.get(0, 0), image.get(0, 0)) - self.assertEqual(image2.get(15, 8), image.get(15, 8)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) image.write(filename, format='gif', from_coords=(4, 6, 6, 9)) image3 = tkinter.PhotoImage('::img::test3', master=self.root, - format='gif', - file=filename) + format='gif', file=filename) + self.assertEqual(str(image3), '::img::test3') + self.assertEqual(image3.type(), 'photo') + self.assertEqual(image3.width(), 2) + self.assertEqual(image3.height(), 3) + self.assertEqual(image3.get(0, 0), image.get(4, 6)) + self.assertEqual(image3.get(1, 2), image.get(5, 8)) + + image.write(filename, background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', file=filename) + self.assertEqual(image4.get(0, 0), (255, 0, 0)) + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + image.write(filename, grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', file=filename) + c = image5.get(4, 6) + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_data(self): + image = self.create() + + data = image.data() + self.assertIsInstance(data, tuple) + for row in data: + self.assertIsInstance(row, str) + self.assertEqual(data[6].split()[4], '#%02x%02x%02x' % image.get(4, 6)) + + data = image.data('ppm') + image2 = tkinter.PhotoImage('::img::test2', master=self.root, + format='ppm', data=data) + self.assertEqual(str(image2), '::img::test2') + self.assertEqual(image2.type(), 'photo') + self.assertEqual(image2.width(), 16) + self.assertEqual(image2.height(), 16) + self.assertEqual(image2.get(0, 0), image.get(0, 0)) + self.assertEqual(image2.get(4, 6), image.get(4, 6)) + + data = image.data(format='gif', from_coords=(4, 6, 6, 9)) + image3 = tkinter.PhotoImage('::img::test3', master=self.root, + format='gif', data=data) self.assertEqual(str(image3), '::img::test3') self.assertEqual(image3.type(), 'photo') self.assertEqual(image3.width(), 2) @@ -536,6 +619,19 @@ def test_write(self): self.assertEqual(image3.get(0, 0), image.get(4, 6)) self.assertEqual(image3.get(1, 2), image.get(5, 8)) + data = image.data('ppm', background='#ff0000') + image4 = tkinter.PhotoImage('::img::test4', master=self.root, + format='ppm', data=data) + self.assertEqual(image4.get(0, 0), (255, 0, 0)) + self.assertEqual(image4.get(4, 6), image.get(4, 6)) + + data = image.data('ppm', grayscale=True) + image5 = tkinter.PhotoImage('::img::test5', master=self.root, + format='ppm', data=data) + c = image5.get(4, 6) + self.assertTrue(c[0] == c[1] == c[2], c) + + def test_transparency(self): image = self.create() self.assertEqual(image.transparency_get(0, 0), True) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 35da83107bd99ef..5031085ac3e9279 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -4398,17 +4398,117 @@ def put(self, data, to=None): to = to[1:] args = args + ('-to',) + tuple(to) self.tk.call(args) - # XXX read - - def write(self, filename, format=None, from_coords=None): - """Write image to file FILENAME in FORMAT starting from - position FROM_COORDS.""" - args = (self.name, 'write', filename) - if format: - args = args + ('-format', format) - if from_coords: - args = args + ('-from',) + tuple(from_coords) - self.tk.call(args) + + def read(self, filename, format=None, *, from_coords=None, to=None, shrink=False): + """Reads image data from the file named FILENAME into the image. + + The FORMAT option specifies the format of the image data in the + file. + + The FROM_COORDS option specifies a rectangular sub-region of the image + file data to be copied to the destination image. It must be a tuple + or a list of 1 to 4 integers (x1, y1, x2, y2). (x1, y1) and + (x2, y2) specify diagonally opposite corners of the rectangle. If + x2 and y2 are not specified, the default value is the bottom-right + corner of the source image. The default, if this option is not + specified, is the whole of the image in the image file. + + The TO option specifies the coordinates of the top-left corner of + the region of the image into which data from filename are to be + read. The default is (0, 0). + + If SHRINK is true, the size of the destination image will be + reduced, if necessary, so that the region into which the image file + data are read is at the bottom-right corner of the image. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if shrink: + options += ('-shrink',) + if to is not None: + options += ('-to', *to) + self.tk.call(self.name, 'read', filename, *options) + + def write(self, filename, format=None, from_coords=None, *, + background=None, grayscale=False): + """Writes image data from the image to a file named FILENAME. + + The FORMAT option specifies the name of the image file format + handler to be used to write the data to the file. If this option + is not given, the format is guessed from the file extension. + + The FROM_COORDS option specifies a rectangular region of the image + to be written to the image file. It must be a tuple or a list of 1 + to 4 integers (x1, y1, x2, y2). If only x1 and y1 are specified, + the region extends from (x1,y1) to the bottom-right corner of the + image. If all four coordinates are given, they specify diagonally + opposite corners of the rectangular region. The default, if this + option is not given, is the whole image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + self.tk.call(self.name, 'write', filename, *options) + + def data(self, format=None, *, from_coords=None, + background=None, grayscale=False): + """Returns image data. + + The FORMAT option specifies the name of the image file format + handler to be used. If this option is not given, this method uses + a format that consists of a tuple (one element per row) of strings + containings space separated (one element per pixel/column) colors + in “#RRGGBB” format (where RR is a pair of hexadecimal digits for + the red channel, GG for green, and BB for blue). + + The FROM_COORDS option specifies a rectangular region of the image + to be returned. It must be a tuple or a list of 1 to 4 integers + (x1, y1, x2, y2). If only x1 and y1 are specified, the region + extends from (x1,y1) to the bottom-right corner of the image. If + all four coordinates are given, they specify diagonally opposite + corners of the rectangular region, including (x1, y1) and excluding + (x2, y2). The default, if this option is not given, is the whole + image. + + If BACKGROUND is specified, the data will not contain any + transparency information. In all transparent pixels the color will + be replaced by the specified color. + + If GRAYSCALE is true, the data will not contain color information. + All pixel data will be transformed into grayscale. + """ + options = () + if format is not None: + options += ('-format', format) + if from_coords is not None: + options += ('-from', *from_coords) + if grayscale: + options += ('-grayscale',) + if background is not None: + options += ('-background', background) + data = self.tk.call(self.name, 'data', *options) + if isinstance(data, str): # For wantobjects = 0. + if format is None: + data = self.tk.splitlist(data) + else: + data = bytes(data, 'latin1') + return data def transparency_get(self, x, y): """Return True if the pixel at x,y is transparent.""" diff --git a/Misc/NEWS.d/next/Library/2024-04-25-11-49-11.gh-issue-118271.5N2Xcy.rst b/Misc/NEWS.d/next/Library/2024-04-25-11-49-11.gh-issue-118271.5N2Xcy.rst new file mode 100644 index 000000000000000..7f116028d979fbb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-25-11-49-11.gh-issue-118271.5N2Xcy.rst @@ -0,0 +1,4 @@ +Add the :class:`!PhotoImage` methods :meth:`~tkinter.PhotoImage.read` to +read an image from a file and :meth:`~tkinter.PhotoImage.data` to get the +image data. Add *background* and *grayscale* parameters to +:class:`!PhotoImage` method :meth:`~tkinter.PhotoImage.write`. From 7528b84e947f727734bd802356e380673553387d Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 6 May 2024 08:46:15 -0700 Subject: [PATCH 155/220] gh-118609: Add proper error check for framelocalsproxy (#118615) --- Objects/frameobject.c | 145 ++++++++++++++++++++++++++++-------------- 1 file changed, 99 insertions(+), 46 deletions(-) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index aac809b459a430e..869cdec8aa70627 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -189,7 +189,7 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) assert(PyDict_Check(extra)); - return PyDict_SetItem(extra, key, value) < 0; + return PyDict_SetItem(extra, key, value); } static int @@ -200,19 +200,19 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) } PyObject *keys = PyMapping_Keys(other); - PyObject *iter = NULL; - PyObject *key = NULL; - PyObject *value = NULL; - - assert(keys != NULL); + if (keys == NULL) { + return -1; + } - iter = PyObject_GetIter(keys); + PyObject *iter = PyObject_GetIter(keys); Py_DECREF(keys); - if (iter == NULL) { return -1; } + PyObject *key = NULL; + PyObject *value = NULL; + while ((key = PyIter_Next(iter)) != NULL) { value = PyObject_GetItem(other, key); if (value == NULL) { @@ -238,12 +238,11 @@ framelocalsproxy_merge(PyObject* self, PyObject* other) } static PyObject * -framelocalsproxy_keys(PyObject *self, PyObject *__unused) +framelocalsproxy_keys(PyObject *self, void *Py_UNUSED(ignored)) { - PyObject *names = PyList_New(0); PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); - + PyObject *names = PyList_New(0); if (names == NULL) { return NULL; } @@ -260,12 +259,13 @@ framelocalsproxy_keys(PyObject *self, PyObject *__unused) } // Iterate through the extra locals - Py_ssize_t i = 0; - PyObject *key = NULL; - PyObject *value = NULL; - if (frame->f_extra_locals) { assert(PyDict_Check(frame->f_extra_locals)); + + Py_ssize_t i = 0; + PyObject *key = NULL; + PyObject *value = NULL; + while (PyDict_Next(frame->f_extra_locals, &i, &key, &value)) { if (PyList_Append(names, key) < 0) { Py_DECREF(names); @@ -341,9 +341,16 @@ framelocalsproxy_richcompare(PyObject *self, PyObject *other, int op) } } else if (PyDict_Check(other)) { PyObject *dct = PyDict_New(); - PyObject *result = NULL; - PyDict_Update(dct, self); - result = PyObject_RichCompare(dct, other, op); + if (dct == NULL) { + return NULL; + } + + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + return NULL; + } + + PyObject *result = PyObject_RichCompare(dct, other, op); Py_DECREF(dct); return result; } @@ -360,14 +367,22 @@ framelocalsproxy_repr(PyObject *self) } PyObject *dct = PyDict_New(); - PyObject *repr = NULL; + if (dct == NULL) { + Py_ReprLeave(self); + return NULL; + } - if (PyDict_Update(dct, self) == 0) { - repr = PyObject_Repr(dct); + if (PyDict_Update(dct, self) < 0) { + Py_DECREF(dct); + Py_ReprLeave(self); + return NULL; } - Py_ReprLeave(self); + PyObject *repr = PyObject_Repr(dct); Py_DECREF(dct); + + Py_ReprLeave(self); + return repr; } @@ -379,6 +394,10 @@ framelocalsproxy_or(PyObject *self, PyObject *other) } PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + if (PyDict_Update(result, self) < 0) { Py_DECREF(result); return NULL; @@ -407,27 +426,35 @@ framelocalsproxy_inplace_or(PyObject *self, PyObject *other) } static PyObject* -framelocalsproxy_values(PyObject *self, PyObject *__unused) +framelocalsproxy_values(PyObject *self, void *Py_UNUSED(ignored)) { - PyObject *values = PyList_New(0); PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *values = PyList_New(0); + if (values == NULL) { + return NULL; + } for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *value = framelocalsproxy_getval(frame->f_frame, co, i); if (value) { - PyList_Append(values, value); + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } } } // Iterate through the extra locals - Py_ssize_t j = 0; - PyObject *key = NULL; - PyObject *value = NULL; - if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { - PyList_Append(values, value); + if (PyList_Append(values, value) < 0) { + Py_DECREF(values); + return NULL; + } } } @@ -435,11 +462,14 @@ framelocalsproxy_values(PyObject *self, PyObject *__unused) } static PyObject * -framelocalsproxy_items(PyObject *self, PyObject *__unused) +framelocalsproxy_items(PyObject *self, void *Py_UNUSED(ignored)) { - PyObject *items = PyList_New(0); PyFrameObject *frame = ((PyFrameLocalsProxyObject*)self)->frame; PyCodeObject *co = _PyFrame_GetCode(frame->f_frame); + PyObject *items = PyList_New(0); + if (items == NULL) { + return NULL; + } for (int i = 0; i < co->co_nlocalsplus; i++) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); @@ -447,20 +477,39 @@ framelocalsproxy_items(PyObject *self, PyObject *__unused) if (value) { PyObject *pair = PyTuple_Pack(2, name, value); - PyList_Append(items, pair); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + Py_DECREF(pair); } } // Iterate through the extra locals - Py_ssize_t j = 0; - PyObject *key = NULL; - PyObject *value = NULL; - if (frame->f_extra_locals) { + Py_ssize_t j = 0; + PyObject *key = NULL; + PyObject *value = NULL; while (PyDict_Next(frame->f_extra_locals, &j, &key, &value)) { PyObject *pair = PyTuple_Pack(2, key, value); - PyList_Append(items, pair); + if (pair == NULL) { + Py_DECREF(items); + return NULL; + } + + if (PyList_Append(items, pair) < 0) { + Py_DECREF(items); + Py_DECREF(pair); + return NULL; + } + Py_DECREF(pair); } } @@ -588,7 +637,7 @@ framelocalsproxy_setdefault(PyObject* self, PyObject *const *args, Py_ssize_t na } static PyObject* -framelocalsproxy_reversed(PyObject *self, PyObject *__unused) +framelocalsproxy_reversed(PyObject *self, void *Py_UNUSED(ignored)) { PyObject *result = framelocalsproxy_keys(self, NULL); @@ -623,19 +672,19 @@ static PyMethodDef framelocalsproxy_methods[] = { NULL}, {"__getitem__", framelocalsproxy_getitem, METH_O | METH_COEXIST, NULL}, - {"__reversed__", framelocalsproxy_reversed, METH_NOARGS, + {"update", framelocalsproxy_update, METH_O, NULL}, - {"keys", framelocalsproxy_keys, METH_NOARGS, + {"__reversed__", _PyCFunction_CAST(framelocalsproxy_reversed), METH_NOARGS, NULL}, - {"values", framelocalsproxy_values, METH_NOARGS, + {"keys", _PyCFunction_CAST(framelocalsproxy_keys), METH_NOARGS, NULL}, - {"items", framelocalsproxy_items, METH_NOARGS, + {"values", _PyCFunction_CAST(framelocalsproxy_values), METH_NOARGS, NULL}, - {"update", framelocalsproxy_update, METH_O, + {"items", _PyCFunction_CAST(framelocalsproxy_items), METH_NOARGS, NULL}, - {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, + {"get", _PyCFunction_CAST(framelocalsproxy_get), METH_FASTCALL, NULL}, - {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, + {"setdefault", _PyCFunction_CAST(framelocalsproxy_setdefault), METH_FASTCALL, NULL}, {NULL, NULL} /* sentinel */ }; @@ -666,6 +715,10 @@ PyObject * _PyFrameLocalsProxy_New(PyFrameObject *frame) { PyObject* args = PyTuple_Pack(1, frame); + if (args == NULL) { + return NULL; + } + PyObject* proxy = (PyObject*)framelocalsproxy_new(&PyFrameLocalsProxy_Type, args, NULL); Py_DECREF(args); return proxy; From 3ed3bc379a0c4ce7a107dd4bc276554fbb477998 Mon Sep 17 00:00:00 2001 From: Kerim Kabirov Date: Mon, 6 May 2024 18:55:22 +0200 Subject: [PATCH 156/220] GH-115577 Clarify netloc term usage in urllib.parse docs (GH-117632) --- Doc/library/urllib.parse.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 3c898c3e8263046..59fb14960ba9f6a 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -31,6 +31,11 @@ The :mod:`urllib.parse` module defines functions that fall into two broad categories: URL parsing and URL quoting. These are covered in detail in the following sections. +This module's functions use the deprecated term ``netloc`` (or ``net_loc``), +which was introduced in :rfc:`1808`. However, this term has been obsoleted by +:rfc:`3986`, which introduced the term ``authority`` as its replacement. +The use of ``netloc`` is continued for backward compatibility. + URL Parsing ----------- From 417dd3aca7bb910691d34e54fee108d7bbc627d3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Mon, 6 May 2024 18:59:36 +0200 Subject: [PATCH 157/220] gh-116322: Rename PyModule_ExperimentalSetGIL to PyUnstable_Module_SetGIL (GH-118645) --- Doc/c-api/module.rst | 2 +- Include/moduleobject.h | 2 +- .../2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst | 2 +- Modules/_cursesmodule.c | 2 +- Modules/_datetimemodule.c | 2 +- Modules/_testbuffer.c | 2 +- Modules/_testcapimodule.c | 2 +- Modules/_testclinic.c | 2 +- Modules/_testclinic_limited.c | 2 +- Modules/_testexternalinspection.c | 2 +- Modules/_testlimitedcapi.c | 2 +- Modules/_testmultiphase.c | 2 +- Modules/_testsinglephase.c | 6 +++--- Modules/_tkinter.c | 2 +- Modules/_tracemalloc.c | 2 +- Modules/readline.c | 2 +- Objects/moduleobject.c | 2 +- Python/bltinmodule.c | 2 +- Python/sysmodule.c | 2 +- 19 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 86308d921f47f2f..6fe1ce9e9948320 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -634,7 +634,7 @@ state: .. versionadded:: 3.9 -.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil) +.. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) Indicate that *module* does or does not support running without the global interpreter lock (GIL), using one of the values from diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 6afa3c7be37ee7c..2a17c891dda811d 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -101,7 +101,7 @@ struct PyModuleDef_Slot { #endif #if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -PyAPI_FUNC(int) PyModule_ExperimentalSetGIL(PyObject *module, void *gil); +PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif struct PyModuleDef { diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst index 2d3bf411a5a1628..1f718a237800df9 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst @@ -1,5 +1,5 @@ Extension modules may indicate to the runtime that they can run without the GIL. Multi-phase init modules do so by calling providing ``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init -modules call ``PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED)`` from +modules call ``PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED)`` from their init function. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 69b97042b89dc77..ee3d4c6eae7546f 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -4753,7 +4753,7 @@ PyInit__curses(void) if (m == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif /* Add some symbolic constants to the module */ diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 00015c5d8c23cbc..8164715a66ff09d 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -6985,7 +6985,7 @@ PyInit__datetime(void) if (mod == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif if (_datetime_exec(mod) < 0) { diff --git a/Modules/_testbuffer.c b/Modules/_testbuffer.c index 35d4ffecad6b15d..54ee468803261a7 100644 --- a/Modules/_testbuffer.c +++ b/Modules/_testbuffer.c @@ -2902,7 +2902,7 @@ PyInit__testbuffer(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif if (_testbuffer_exec(mod) < 0) { Py_DECREF(mod); diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e7e342e529eff47..ff31724c0e9ff9f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3936,7 +3936,7 @@ PyInit__testcapi(void) if (m == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif Py_SET_TYPE(&_HashInheritanceTester_Type, &PyType_Type); diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index c7af552f029b811..4187e13231dc69a 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1956,7 +1956,7 @@ PyInit__testclinic(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif if (PyModule_AddType(m, &TestClass) < 0) { goto error; diff --git a/Modules/_testclinic_limited.c b/Modules/_testclinic_limited.c index d5f98085f842255..370433b3e2a0d94 100644 --- a/Modules/_testclinic_limited.c +++ b/Modules/_testclinic_limited.c @@ -147,7 +147,7 @@ PyInit__testclinic_limited(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif return m; } diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index d9c65fe253f8d3b..696c00c237dccf5 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -631,7 +631,7 @@ PyInit__testexternalinspection(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif int rc = PyModule_AddIntConstant(mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index f88476f4be20546..fb5cdb6ca9e1d34 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -26,7 +26,7 @@ PyInit__testlimitedcapi(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif if (_PyTestLimitedCAPI_Init_Abstract(mod) < 0) { diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index ca3d83233c5dd03..886b260aceb20d8 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -901,7 +901,7 @@ PyInit__test_module_state_shared(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif if (PyModule_AddObjectRef(module, "Error", PyExc_Exception) < 0) { diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index c0eb266751e9e6e..448be502466e799 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -472,7 +472,7 @@ init__testsinglephase_basic(PyModuleDef *def) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif module_state *state = &global_state.module; @@ -566,7 +566,7 @@ PyInit__testsinglephase_with_reinit(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif assert(get_module_state(module) == NULL); @@ -631,7 +631,7 @@ PyInit__testsinglephase_with_state(void) return NULL; } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(module, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif module_state *state = get_module_state(module); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index ecb7ca8de62ef1b..3d1cff2fa52464a 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -3206,7 +3206,7 @@ PyInit__tkinter(void) if (m == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif Tkinter_TclError = PyErr_NewException("_tkinter.TclError", NULL, NULL); diff --git a/Modules/_tracemalloc.c b/Modules/_tracemalloc.c index 55028dc3a65ff46..887a1e820e250ed 100644 --- a/Modules/_tracemalloc.c +++ b/Modules/_tracemalloc.c @@ -220,7 +220,7 @@ PyInit__tracemalloc(void) if (m == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif if (_PyTraceMalloc_Init() < 0) { diff --git a/Modules/readline.c b/Modules/readline.c index f59f8a9834caafc..35655c70a4618fa 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -1553,7 +1553,7 @@ PyInit_readline(void) if (m == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(m, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif if (PyModule_AddIntConstant(m, "_READLINE_VERSION", diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index a570b13e1208637..46995b948a28e71 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -439,7 +439,7 @@ PyModule_FromDefAndSpec2(PyModuleDef* def, PyObject *spec, int module_api_versio #ifdef Py_GIL_DISABLED int -PyModule_ExperimentalSetGIL(PyObject *module, void *gil) +PyUnstable_Module_SetGIL(PyObject *module, void *gil) { if (!PyModule_Check(module)) { PyErr_BadInternalCall(); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 88d858dc944863f..e7b60ebca5e2d7e 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -3125,7 +3125,7 @@ _PyBuiltin_Init(PyInterpreterState *interp) if (mod == NULL) return NULL; #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); #endif dict = PyModule_GetDict(mod); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 69bee32987fe6bd..417040c891f8031 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -3775,7 +3775,7 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) return _PyStatus_ERR("failed to create a module object"); } #ifdef Py_GIL_DISABLED - PyModule_ExperimentalSetGIL(sysmod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(sysmod, Py_MOD_GIL_NOT_USED); #endif PyObject *sysdict = PyModule_GetDict(sysmod); From 1ff626ebda465931ff3e4922e8e87d586eb6244c Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 20:12:51 +0300 Subject: [PATCH 158/220] gh-71592: Add ability to trace Tcl commands executed by Tkinter (GH-118291) This is an experimental feature, for internal use. Setting tkinter._debug = True before creating the root window enables printing every executed Tcl command (or a Tcl command equivalent to the used Tcl C API). This will help to convert a Tkinter example into Tcl script to check whether the issue is caused by Tkinter or exists in the underlying Tcl/Tk library. --- Lib/tkinter/__init__.py | 19 ++++- Modules/_tkinter.c | 148 +++++++++++++++++++++++++++++++++++- Modules/clinic/_tkinter.c.h | 29 ++++++- 3 files changed, 190 insertions(+), 6 deletions(-) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index 5031085ac3e9279..dc6ee9a1b472257 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -41,6 +41,7 @@ import re wantobjects = 1 +_debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) TclVersion = float(_tkinter.TCL_VERSION) @@ -69,7 +70,10 @@ def _stringify(value): else: value = '{%s}' % _join(value) else: - value = str(value) + if isinstance(value, bytes): + value = str(value, 'latin1') + else: + value = str(value) if not value: value = '{}' elif _magic_re.search(value): @@ -411,7 +415,6 @@ def __del__(self): self._tk.globalunsetvar(self._name) if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self._tk.deletecommand(name) self._tclCommands = None @@ -683,7 +686,6 @@ def destroy(self): this widget in the Tcl interpreter.""" if self._tclCommands is not None: for name in self._tclCommands: - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) self._tclCommands = None @@ -691,7 +693,6 @@ def deletecommand(self, name): """Internal function. Delete the Tcl command provided in NAME.""" - #print '- Tkinter: deleted command', name self.tk.deletecommand(name) try: self._tclCommands.remove(name) @@ -2450,6 +2451,8 @@ def __init__(self, screenName=None, baseName=None, className='Tk', baseName = baseName + ext interactive = False self.tk = _tkinter.create(screenName, baseName, className, interactive, wantobjects, useTk, sync, use) + if _debug: + self.tk.settrace(_print_command) if useTk: self._loadtk() if not sys.flags.ignore_environment: @@ -2536,6 +2539,14 @@ def __getattr__(self, attr): "Delegate attribute access to the interpreter object" return getattr(self.tk, attr) + +def _print_command(cmd, *, file=sys.stderr): + # Print executed Tcl/Tk commands. + assert isinstance(cmd, tuple) + cmd = _join(cmd) + print(cmd, file=file) + + # Ideally, the classes Pack, Place and Grid disappear, the # pack/place/grid methods are defined on the Widget class, and # everybody uses w.pack_whatever(...) instead of Pack.whatever(w, diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 3d1cff2fa52464a..163449a927cf188 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -306,6 +306,7 @@ typedef struct { int threaded; /* True if tcl_platform[threaded] */ Tcl_ThreadId thread_id; int dispatching; + PyObject *trace; /* We cannot include tclInt.h, as this is internal. So we cache interesting types here. */ const Tcl_ObjType *OldBooleanType; @@ -570,6 +571,7 @@ Tkapp_New(const char *screenName, const char *className, TCL_GLOBAL_ONLY) != NULL; v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; + v->trace = NULL; #ifndef TCL_THREADS if (v->threaded) { @@ -1306,6 +1308,29 @@ Tkapp_ObjectResult(TkappObject *self) return res; } +static int +Tkapp_Trace(TkappObject *self, PyObject *args) +{ + if (args == NULL) { + return 0; + } + if (self->trace) { + PyObject *res = PyObject_CallObject(self->trace, args); + if (res == NULL) { + Py_DECREF(args); + return 0; + } + Py_DECREF(res); + } + Py_DECREF(args); + return 1; +} + +#define TRACE(_self, ARGS) do { \ + if ((_self)->trace && !Tkapp_Trace((_self), Py_BuildValue ARGS)) { \ + return NULL; \ + } \ + } while (0) /* Tkapp_CallProc is the event procedure that is executed in the context of the Tcl interpreter thread. Initially, it holds the Tcl lock, and doesn't @@ -1320,7 +1345,12 @@ Tkapp_CallProc(Tcl_Event *evPtr, int flags) int objc; int i; ENTER_PYTHON - objv = Tkapp_CallArgs(e->args, objStore, &objc); + if (e->self->trace && !Tkapp_Trace(e->self, PyTuple_Pack(1, e->args))) { + objv = NULL; + } + else { + objv = Tkapp_CallArgs(e->args, objStore, &objc); + } if (!objv) { *(e->exc) = PyErr_GetRaisedException(); *(e->res) = NULL; @@ -1413,6 +1443,7 @@ Tkapp_Call(PyObject *selfptr, PyObject *args) } else { + TRACE(self, ("(O)", args)); objv = Tkapp_CallArgs(args, objStore, &objc); if (!objv) @@ -1455,6 +1486,8 @@ _tkinter_tkapp_eval_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "eval", script)); + ENTER_TCL err = Tcl_Eval(Tkapp_Interp(self), script); ENTER_OVERLAP @@ -1484,6 +1517,8 @@ _tkinter_tkapp_evalfile_impl(TkappObject *self, const char *fileName) CHECK_STRING_LENGTH(fileName); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "source", fileName)); + ENTER_TCL err = Tcl_EvalFile(Tkapp_Interp(self), fileName); ENTER_OVERLAP @@ -1513,6 +1548,8 @@ _tkinter_tkapp_record_impl(TkappObject *self, const char *script) CHECK_STRING_LENGTH(script); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ssss))", "history", "add", script, "exec")); + ENTER_TCL err = Tcl_RecordAndEval(Tkapp_Interp(self), script, TCL_NO_EVAL); ENTER_OVERLAP @@ -1702,6 +1739,15 @@ SetVar(TkappObject *self, PyObject *args, int flags) newval = AsObj(newValue); if (newval == NULL) return NULL; + + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((ssssO))", "uplevel", "#0", "set", + name1, newValue)); + } + else { + TRACE((TkappObject *)self, ("((ssO))", "set", name1, newValue)); + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, NULL, newval, flags); @@ -1719,8 +1765,22 @@ SetVar(TkappObject *self, PyObject *args, int flags) return NULL; CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + /* XXX must hold tcl lock already??? */ newval = AsObj(newValue); + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + TRACE((TkappObject *)self, ("((sssNO))", "uplevel", "#0", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + else { + TRACE((TkappObject *)self, ("((sNO))", "set", + PyUnicode_FromFormat("%s(%s)", name1, name2), + newValue)); + } + } + ENTER_TCL ok = Tcl_SetVar2Ex(Tkapp_Interp(self), name1, name2, newval, flags); ENTER_OVERLAP @@ -1807,6 +1867,28 @@ UnsetVar(TkappObject *self, PyObject *args, int flags) CHECK_STRING_LENGTH(name1); CHECK_STRING_LENGTH(name2); + + if (((TkappObject *)self)->trace) { + if (flags & TCL_GLOBAL_ONLY) { + if (name2) { + TRACE((TkappObject *)self, ("((sssN))", "uplevel", "#0", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ssss))", "uplevel", "#0", "unset", name1)); + } + } + else { + if (name2) { + TRACE((TkappObject *)self, ("((sN))", "unset", + PyUnicode_FromFormat("%s(%s)", name1, name2))); + } + else { + TRACE((TkappObject *)self, ("((ss))", "unset", name1)); + } + } + } + ENTER_TCL code = Tcl_UnsetVar2(Tkapp_Interp(self), name1, name2, flags); ENTER_OVERLAP @@ -1973,6 +2055,8 @@ _tkinter_tkapp_exprstring_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprString(Tkapp_Interp(self), s); ENTER_OVERLAP @@ -2003,6 +2087,8 @@ _tkinter_tkapp_exprlong_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprLong(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2032,6 +2118,9 @@ _tkinter_tkapp_exprdouble_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprDouble(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2061,6 +2150,9 @@ _tkinter_tkapp_exprboolean_impl(TkappObject *self, const char *s) CHECK_STRING_LENGTH(s); CHECK_TCL_APPARTMENT; + + TRACE(self, ("((ss))", "expr", s)); + ENTER_TCL retval = Tcl_ExprBoolean(Tkapp_Interp(self), s, &v); ENTER_OVERLAP @@ -2286,6 +2378,8 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, !WaitForMainloop(self)) return NULL; + TRACE(self, ("((ss()O))", "proc", name, func)); + data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); @@ -2344,6 +2438,8 @@ _tkinter_tkapp_deletecommand_impl(TkappObject *self, const char *name) CHECK_STRING_LENGTH(name); + TRACE(self, ("((sss))", "rename", name, "")); + if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { Tcl_Condition cond = NULL; CommandEvent *ev; @@ -2469,6 +2565,8 @@ _tkinter_tkapp_createfilehandler_impl(TkappObject *self, PyObject *file, return NULL; } + TRACE(self, ("((ssiiO))", "#", "createfilehandler", tfile, mask, func)); + data = NewFHCD(func, file, tfile); if (data == NULL) return NULL; @@ -2500,6 +2598,8 @@ _tkinter_tkapp_deletefilehandler(TkappObject *self, PyObject *file) if (tfile < 0) return NULL; + TRACE(self, ("((ssi))", "#", "deletefilehandler", tfile)); + DeleteFHCD(tfile); /* Ought to check for null Tcl_File object... */ @@ -2534,6 +2634,7 @@ _tkinter_tktimertoken_deletetimerhandler_impl(TkttObject *self) PyObject *func = v->func; if (v->token != NULL) { + /* TRACE(...) */ Tcl_DeleteTimerHandler(v->token); v->token = NULL; } @@ -2636,6 +2737,8 @@ _tkinter_tkapp_createtimerhandler_impl(TkappObject *self, int milliseconds, CHECK_TCL_APPARTMENT; + TRACE(self, ("((siO))", "after", milliseconds, func)); + v = Tktt_New(func); if (v) { v->token = Tcl_CreateTimerHandler(milliseconds, TimerHandler, @@ -2803,6 +2906,47 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) Py_RETURN_NONE; } +/*[clinic input] +_tkinter.tkapp.settrace + + func: object + / + +Set the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_settrace(TkappObject *self, PyObject *func) +/*[clinic end generated code: output=847f6ebdf46e84fa input=31b260d46d3d018a]*/ +{ + if (func == Py_None) { + func = NULL; + } + else { + Py_INCREF(func); + } + Py_XSETREF(self->trace, func); + Py_RETURN_NONE; +} + +/*[clinic input] +_tkinter.tkapp.gettrace + +Get the tracing function. +[clinic start generated code]*/ + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self) +/*[clinic end generated code: output=d4e2ba7d63e77bb5 input=ac2aea5be74e8c4c]*/ +{ + PyObject *func = self->trace; + if (!func) { + func = Py_None; + } + Py_INCREF(func); + return func; +} + /*[clinic input] _tkinter.tkapp.willdispatch @@ -3038,6 +3182,8 @@ static PyMethodDef Tkapp_methods[] = { _TKINTER_TKAPP_WILLDISPATCH_METHODDEF {"wantobjects", Tkapp_WantObjects, METH_VARARGS}, + _TKINTER_TKAPP_SETTRACE_METHODDEF + _TKINTER_TKAPP_GETTRACE_METHODDEF {"call", Tkapp_Call, METH_VARARGS}, _TKINTER_TKAPP_EVAL_METHODDEF _TKINTER_TKAPP_EVALFILE_METHODDEF diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 188bcc773cfc41c..192c49dba216e25 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -622,6 +622,33 @@ _tkinter_tkapp_loadtk(TkappObject *self, PyObject *Py_UNUSED(ignored)) return _tkinter_tkapp_loadtk_impl(self); } +PyDoc_STRVAR(_tkinter_tkapp_settrace__doc__, +"settrace($self, func, /)\n" +"--\n" +"\n" +"Set the tracing function."); + +#define _TKINTER_TKAPP_SETTRACE_METHODDEF \ + {"settrace", (PyCFunction)_tkinter_tkapp_settrace, METH_O, _tkinter_tkapp_settrace__doc__}, + +PyDoc_STRVAR(_tkinter_tkapp_gettrace__doc__, +"gettrace($self, /)\n" +"--\n" +"\n" +"Get the tracing function."); + +#define _TKINTER_TKAPP_GETTRACE_METHODDEF \ + {"gettrace", (PyCFunction)_tkinter_tkapp_gettrace, METH_NOARGS, _tkinter_tkapp_gettrace__doc__}, + +static PyObject * +_tkinter_tkapp_gettrace_impl(TkappObject *self); + +static PyObject * +_tkinter_tkapp_gettrace(TkappObject *self, PyObject *Py_UNUSED(ignored)) +{ + return _tkinter_tkapp_gettrace_impl(self); +} + PyDoc_STRVAR(_tkinter_tkapp_willdispatch__doc__, "willdispatch($self, /)\n" "--\n" @@ -861,4 +888,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=d447501ec5aa9447 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=86a515890d48a2ce input=a9049054013a1b77]*/ From e5c699280deac076cddfef37c8af917a550f6ac3 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Mon, 6 May 2024 18:13:15 +0100 Subject: [PATCH 159/220] GH-117714: implement athrow().close() and asend().close() using throw (GH-117906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * GH-117714: replace athrow().close() and asend().close() stubs with implimentations * test athrow().close() and asend().close() raises RuntimeError * 📜🤖 Added by blurb_it. * Update Objects/genobject.c Co-authored-by: Petr Viktorin --------- Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Petr Viktorin --- Lib/test/test_asyncgen.py | 48 +++++++++++++++++++ ...-05-01-07-06-48.gh-issue-117714.Ip_dm5.rst | 1 + Objects/genobject.c | 42 ++++++++++++++-- 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-01-07-06-48.gh-issue-117714.Ip_dm5.rst diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 1985ede656e7a05..4f2278bb263681a 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -571,6 +571,54 @@ async def gen(): self.assertTrue(inspect.isawaitable(aclose)) aclose.close() + def test_async_gen_asend_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + async def agenfn(): + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + return + yield + + agen = agenfn() + gen = agen.asend(None) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + + def test_async_gen_athrow_close_runtime_error(self): + import types + + @types.coroutine + def _async_yield(v): + return (yield v) + + class MyExc(Exception): + pass + + async def agenfn(): + try: + yield + except MyExc: + try: + await _async_yield(None) + except GeneratorExit: + await _async_yield(None) + + agen = agenfn() + with self.assertRaises(StopIteration): + agen.asend(None).send(None) + gen = agen.athrow(MyExc) + gen.send(None) + with self.assertRaisesRegex(RuntimeError, "coroutine ignored GeneratorExit"): + gen.close() + class AsyncGenAsyncioTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-01-07-06-48.gh-issue-117714.Ip_dm5.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-07-06-48.gh-issue-117714.Ip_dm5.rst new file mode 100644 index 000000000000000..d8ec242fcb65b91 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-07-06-48.gh-issue-117714.Ip_dm5.rst @@ -0,0 +1 @@ +update ``async_generator.athrow().close()`` and ``async_generator.asend().close()`` to close their section of the underlying async generator diff --git a/Objects/genobject.c b/Objects/genobject.c index 89bb21a8674b9f1..acdcf579a940efd 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -1846,8 +1846,25 @@ async_gen_asend_throw(PyAsyncGenASend *o, PyObject *const *args, Py_ssize_t narg static PyObject * async_gen_asend_close(PyAsyncGenASend *o, PyObject *args) { - o->ags_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyObject *result; + if (o->ags_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + result = async_gen_asend_throw(o, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } static void @@ -2291,8 +2308,25 @@ async_gen_athrow_iternext(PyAsyncGenAThrow *o) static PyObject * async_gen_athrow_close(PyAsyncGenAThrow *o, PyObject *args) { - o->agt_state = AWAITABLE_STATE_CLOSED; - Py_RETURN_NONE; + PyObject *result; + if (o->agt_state == AWAITABLE_STATE_CLOSED) { + Py_RETURN_NONE; + } + result = async_gen_athrow_throw(o, &PyExc_GeneratorExit, 1); + if (result == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration) || + PyErr_ExceptionMatches(PyExc_StopAsyncIteration) || + PyErr_ExceptionMatches(PyExc_GeneratorExit)) + { + PyErr_Clear(); + Py_RETURN_NONE; + } + return result; + } else { + Py_DECREF(result); + PyErr_SetString(PyExc_RuntimeError, "coroutine ignored GeneratorExit"); + return NULL; + } } From e6b213ee3ffb05f067d30cb8bb45681887212444 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Mon, 6 May 2024 18:20:42 +0100 Subject: [PATCH 160/220] gh-118518: Ensure that the code padding it's applied (#118654) --- Python/perf_jit_trampoline.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 23478c6d469c2b3..2a29318b1054a56 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -292,7 +292,7 @@ static void* perf_map_jit_init(void) { } perf_jit_map_state.code_id = 0; - // trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + trampoline_api.code_padding = PERF_JIT_CODE_PADDING; return &perf_jit_map_state; } From 5a1618a2c8c108b8c73aa9459b63f0dbd66b60f6 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 6 May 2024 10:50:35 -0700 Subject: [PATCH 161/220] gh-118362: Fix thread safety around lookups from the type cache in the face of concurrent mutators (#118454) Add _PyType_LookupRef and use incref before setting attribute on type Makes setting an attribute on a class and signaling type modified atomic Avoid adding re-entrancy exposing the type cache in an inconsistent state by decrefing after type is updated --- Include/cpython/object.h | 1 + Include/cpython/pyatomic.h | 3 + Include/cpython/pyatomic_gcc.h | 4 + Include/cpython/pyatomic_msc.h | 13 + Include/cpython/pyatomic_std.h | 8 + Include/internal/pycore_dict.h | 3 + Include/internal/pycore_object.h | 1 + Lib/test/test_class.py | 19 ++ Lib/test/test_free_threading/test_type.py | 112 ++++++++ ...-05-01-14-20-28.gh-issue-118492.VUsSfn.rst | 2 + Modules/_collectionsmodule.c | 6 +- Modules/_ctypes/_ctypes.c | 2 +- Modules/_lsprof.c | 3 +- Modules/_testcapi/gc.c | 3 +- Objects/classobject.c | 22 +- Objects/dictobject.c | 50 +++- Objects/object.c | 40 ++- Objects/typeobject.c | 269 +++++++++++++----- 18 files changed, 437 insertions(+), 124 deletions(-) create mode 100644 Lib/test/test_free_threading/test_type.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-01-14-20-28.gh-issue-118492.VUsSfn.rst diff --git a/Include/cpython/object.h b/Include/cpython/object.h index c2830b75e66fbe5..e624326693d8e7d 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -275,6 +275,7 @@ typedef struct _heaptypeobject { PyAPI_FUNC(const char *) _PyType_Name(PyTypeObject *); PyAPI_FUNC(PyObject *) _PyType_Lookup(PyTypeObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyType_LookupRef(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) PyType_GetDict(PyTypeObject *); PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 69083f1d9dd0c23..55a139bb9158db9 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -484,6 +484,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value); diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index af78a94c736545f..c0f3747be457585 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -516,6 +516,10 @@ static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 212cd7817d01c52..f32995c1f578ac1 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -989,6 +989,19 @@ _Py_atomic_load_int_acquire(const int *obj) #endif } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(uint32_t volatile *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint32_release" +#endif +} + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 6a77eae536d8dde..0cdce4e6dd39f0f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -911,6 +911,14 @@ _Py_atomic_load_int_acquire(const int *obj) memory_order_acquire); } +static inline void +_Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(uint32_t)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_uint64_release(uint64_t *obj, uint64_t value) { diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index f33026dbd6be586..3ba8ee74b4df870 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -106,6 +106,9 @@ PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObjec /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); +extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); +extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); +extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 3b0222b05cbd70d..f15c332a7b0811e 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -658,6 +658,7 @@ extern PyObject *_PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); extern PyObject* _PyType_GetDocFromInternalDoc(const char *, const char *); extern PyObject* _PyType_GetTextSignatureFromInternalDoc(const char *, const char *, int); +extern int _PyObject_SetAttributeErrorContext(PyObject *v, PyObject* name); void _PyObject_InitInlineValues(PyObject *obj, PyTypeObject *tp); extern int _PyObject_StoreInstanceAttribute(PyObject *obj, diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 5885db84b66b016..655d53b8d5bb6a6 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -882,6 +882,25 @@ class Foo: f.a = 3 self.assertEqual(f.a, 3) + def test_store_attr_type_cache(self): + """Verifies that the type cache doesn't provide a value which is + inconsistent from the dict.""" + class X: + def __del__(inner_self): + v = C.a + self.assertEqual(v, C.__dict__['a']) + + class C: + a = X() + + # prime the cache + C.a + C.a + + # destructor shouldn't be able to see inconsisent state + C.a = X() + C.a = X() + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py new file mode 100644 index 000000000000000..76208c42fc8aeae --- /dev/null +++ b/Lib/test/test_free_threading/test_type.py @@ -0,0 +1,112 @@ +import unittest + +from threading import Thread +from multiprocessing.dummy import Pool +from unittest import TestCase + +from test.support import is_wasi + + +NTHREADS = 6 +BOTTOM = 0 +TOP = 1000 +ITERS = 100 + +class A: + attr = 1 + +@unittest.skipIf(is_wasi, "WASI has no threads.") +class TestType(TestCase): + def test_attr_cache(self): + def read(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + A.attr + + def write(id0): + for _ in range(ITERS): + for _ in range(BOTTOM, TOP): + # Make _PyType_Lookup cache hot first + A.attr + A.attr + x = A.attr + x += 1 + A.attr = x + + + with Pool(NTHREADS) as pool: + pool.apply_async(read, (1,)) + pool.apply_async(write, (1,)) + pool.close() + pool.join() + + def test_attr_cache_consistency(self): + class C: + x = 0 + + DONE = False + def writer_func(): + for i in range(3000): + C.x + C.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = C.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def test_attr_cache_consistency_subclass(self): + class C: + x = 0 + + class D(C): + pass + + DONE = False + def writer_func(): + for i in range(3000): + D.x + D.x + C.x += 1 + nonlocal DONE + DONE = True + + def reader_func(): + while True: + # We should always see a greater value read from the type than the + # dictionary + a = C.__dict__['x'] + b = D.x + self.assertGreaterEqual(b, a) + + if DONE: + break + + self.run_one(writer_func, reader_func) + + def run_one(self, writer_func, reader_func): + writer = Thread(target=writer_func) + readers = [] + for x in range(30): + reader = Thread(target=reader_func) + readers.append(reader) + reader.start() + + writer.start() + writer.join() + for reader in readers: + reader.join() + +if __name__ == "__main__": + unittest.main() diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-01-14-20-28.gh-issue-118492.VUsSfn.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-14-20-28.gh-issue-118492.VUsSfn.rst new file mode 100644 index 000000000000000..57d0f1730b828f8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-14-20-28.gh-issue-118492.VUsSfn.rst @@ -0,0 +1,2 @@ +Fix an issue where the type cache can expose a previously accessed attribute +when a finalizer is run. diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index b865351c93d2d8b..644a90a8c71099d 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -2511,9 +2511,9 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, /* Only take the fast path when get() and __setitem__() * have not been overridden. */ - mapping_get = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(get)); + mapping_get = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(get)); dict_get = _PyType_Lookup(&PyDict_Type, &_Py_ID(get)); - mapping_setitem = _PyType_Lookup(Py_TYPE(mapping), &_Py_ID(__setitem__)); + mapping_setitem = _PyType_LookupRef(Py_TYPE(mapping), &_Py_ID(__setitem__)); dict_setitem = _PyType_Lookup(&PyDict_Type, &_Py_ID(__setitem__)); if (mapping_get != NULL && mapping_get == dict_get && @@ -2587,6 +2587,8 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, } done: + Py_XDECREF(mapping_get); + Py_XDECREF(mapping_setitem); Py_DECREF(it); Py_XDECREF(key); Py_XDECREF(newval); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1b1a0ea549f1e16..574cb8014493c44 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1096,7 +1096,7 @@ static int UnionType_setattro(PyObject *self, PyObject *key, PyObject *value) { /* XXX Should we disallow deleting _fields_? */ - if (-1 == PyObject_GenericSetAttr(self, key, value)) + if (-1 == PyType_Type.tp_setattro(self, key, value)) return -1; if (PyUnicode_Check(key) && diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 7a9041ac079245e..5cf9eba243bd207 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -177,8 +177,7 @@ normalizeUserObj(PyObject *obj) PyObject *modname = fn->m_module; if (name != NULL) { - PyObject *mo = _PyType_Lookup(Py_TYPE(self), name); - Py_XINCREF(mo); + PyObject *mo = _PyType_LookupRef(Py_TYPE(self), name); Py_DECREF(name); if (mo != NULL) { PyObject *res = PyObject_Repr(mo); diff --git a/Modules/_testcapi/gc.c b/Modules/_testcapi/gc.c index f4feaaafbdc6cc9..b472a4185a98af2 100644 --- a/Modules/_testcapi/gc.c +++ b/Modules/_testcapi/gc.c @@ -99,10 +99,11 @@ slot_tp_del(PyObject *self) return; } /* Execute __del__ method, if any. */ - del = _PyType_Lookup(Py_TYPE(self), tp_del); + del = _PyType_LookupRef(Py_TYPE(self), tp_del); Py_DECREF(tp_del); if (del != NULL) { res = PyObject_CallOneArg(del, self); + Py_DECREF(del); if (res == NULL) PyErr_WriteUnraisable(del); else diff --git a/Objects/classobject.c b/Objects/classobject.c index 9cbb9442c6059c5..69a7d5f046e30d0 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -188,15 +188,18 @@ method_getattro(PyObject *obj, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); } if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, obj, (PyObject *)Py_TYPE(obj)); + if (f != NULL) { + PyObject *res = f(descr, obj, (PyObject *)Py_TYPE(obj)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } @@ -410,14 +413,17 @@ instancemethod_getattro(PyObject *self, PyObject *name) if (PyType_Ready(tp) < 0) return NULL; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { descrgetfunc f = TP_DESCR_GET(Py_TYPE(descr)); - if (f != NULL) - return f(descr, self, (PyObject *)Py_TYPE(self)); + if (f != NULL) { + PyObject *res = f(descr, self, (PyObject *)Py_TYPE(self)); + Py_DECREF(descr); + return res; + } else { - return Py_NewRef(descr); + return descr; } } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 1f21f70f149c803..3e662e09ea598e5 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2268,15 +2268,13 @@ _PyDict_GetItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) * exception occurred. */ int -_PyDict_GetItemRef_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash, PyObject **result) +_PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result) { - PyDictObject*mp = (PyDictObject *)op; - PyObject *value; #ifdef Py_GIL_DISABLED - Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup_threadsafe(op, key, hash, &value); #else - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); #endif assert(ix >= 0 || value == NULL); if (ix == DKIX_ERROR) { @@ -2314,9 +2312,38 @@ PyDict_GetItemRef(PyObject *op, PyObject *key, PyObject **result) } } - return _PyDict_GetItemRef_KnownHash(op, key, hash, result); + return _PyDict_GetItemRef_KnownHash((PyDictObject *)op, key, hash, result); } +int +_PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result) +{ + ASSERT_DICT_LOCKED(op); + assert(PyUnicode_CheckExact(key)); + + Py_hash_t hash; + if ((hash = unicode_get_hash(key)) == -1) { + hash = PyObject_Hash(key); + if (hash == -1) { + *result = NULL; + return -1; + } + } + + PyObject *value; + Py_ssize_t ix = _Py_dict_lookup(op, key, hash, &value); + assert(ix >= 0 || value == NULL); + if (ix == DKIX_ERROR) { + *result = NULL; + return -1; + } + if (value == NULL) { + *result = NULL; + return 0; // missing key + } + *result = Py_NewRef(value); + return 1; // key is present +} /* Variant of PyDict_GetItem() that doesn't suppress exceptions. This returns NULL *with* an exception set if an exception occurred. @@ -6704,8 +6731,8 @@ _PyObject_MaterializeManagedDict(PyObject *obj) return dict; } -static int -set_or_del_lock_held(PyDictObject *dict, PyObject *name, PyObject *value) +int +_PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) { if (value == NULL) { Py_hash_t hash; @@ -6767,7 +6794,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, // so that no one else will see it. dict = make_dict_from_instance_attributes(PyInterpreterState_Get(), keys, values); if (dict == NULL || - set_or_del_lock_held(dict, name, value) < 0) { + _PyDict_SetItem_LockHeld(dict, name, value) < 0) { Py_XDECREF(dict); return -1; } @@ -6779,7 +6806,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(dict); - res = set_or_del_lock_held (dict, name, value); + res = _PyDict_SetItem_LockHeld(dict, name, value); return res; } @@ -6822,7 +6849,7 @@ store_instance_attr_dict(PyObject *obj, PyDictObject *dict, PyObject *name, PyOb res = store_instance_attr_lock_held(obj, values, name, value); } else { - res = set_or_del_lock_held(dict, name, value); + res = _PyDict_SetItem_LockHeld(dict, name, value); } Py_END_CRITICAL_SECTION(); return res; @@ -7253,6 +7280,7 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, res = PyDict_SetItem(dict, key, value); } } + ASSERT_CONSISTENT(dict); return res; } diff --git a/Objects/object.c b/Objects/object.c index 1bf0e65ec60ce4b..effbd51991eaa5a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1132,8 +1132,8 @@ _PyObject_SetAttrId(PyObject *v, _Py_Identifier *name, PyObject *w) return result; } -static inline int -set_attribute_error_context(PyObject* v, PyObject* name) +int +_PyObject_SetAttributeErrorContext(PyObject* v, PyObject* name) { assert(PyErr_Occurred()); if (!PyErr_ExceptionMatches(PyExc_AttributeError)){ @@ -1188,7 +1188,7 @@ PyObject_GetAttr(PyObject *v, PyObject *name) } if (result == NULL) { - set_attribute_error_context(v, name); + _PyObject_SetAttributeErrorContext(v, name); } return result; } @@ -1466,13 +1466,13 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } - PyObject *descr = _PyType_Lookup(tp, name); + PyObject *descr = _PyType_LookupRef(tp, name); descrgetfunc f = NULL; if (descr != NULL) { - Py_INCREF(descr); if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { meth_found = 1; - } else { + } + else { f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { *method = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1535,7 +1535,7 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); return 0; } @@ -1569,11 +1569,10 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, goto done; } - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); f = NULL; if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_get; if (f != NULL && PyDescr_IsData(descr)) { res = f(descr, obj, (PyObject *)Py_TYPE(obj)); @@ -1647,7 +1646,7 @@ _PyObject_GenericGetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); @@ -1670,6 +1669,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, descrsetfunc f; int res = -1; + assert(!PyType_IsSubtype(tp, &PyType_Type)); if (!PyUnicode_Check(name)){ PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", @@ -1683,10 +1683,9 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, Py_INCREF(name); Py_INCREF(tp); - descr = _PyType_Lookup(tp, name); + descr = _PyType_LookupRef(tp, name); if (descr != NULL) { - Py_INCREF(descr); f = Py_TYPE(descr)->tp_descr_set; if (f != NULL) { res = f(descr, obj, value); @@ -1722,7 +1721,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, "'%.100s' object has no attribute '%U'", tp->tp_name, name); } - set_attribute_error_context(obj, name); + _PyObject_SetAttributeErrorContext(obj, name); } else { PyErr_Format(PyExc_AttributeError, @@ -1745,17 +1744,10 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } error_check: if (res < 0 && PyErr_ExceptionMatches(PyExc_KeyError)) { - if (PyType_IsSubtype(tp, &PyType_Type)) { - PyErr_Format(PyExc_AttributeError, - "type object '%.50s' has no attribute '%U'", - ((PyTypeObject*)obj)->tp_name, name); - } - else { - PyErr_Format(PyExc_AttributeError, - "'%.100s' object has no attribute '%U'", - tp->tp_name, name); - } - set_attribute_error_context(obj, name); + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + _PyObject_SetAttributeErrorContext(obj, name); } done: Py_XDECREF(descr); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index ec19a3d461f6239..4b144fab5de8f18 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -71,6 +71,16 @@ class object "PyObject *" "&PyBaseObject_Type" _PyCriticalSection_End(&_cs); \ } +#define BEGIN_TYPE_DICT_LOCK(d) \ + { \ + _PyCriticalSection2 _cs; \ + _PyCriticalSection2_Begin(&_cs, TYPE_LOCK, \ + &_PyObject_CAST(d)->ob_mutex); \ + +#define END_TYPE_DICT_LOCK() \ + _PyCriticalSection2_End(&_cs); \ + } + #define ASSERT_TYPE_LOCK_HELD() \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) @@ -78,6 +88,8 @@ class object "PyObject *" "&PyBaseObject_Type" #define BEGIN_TYPE_LOCK() #define END_TYPE_LOCK() +#define BEGIN_TYPE_DICT_LOCK(d) +#define END_TYPE_DICT_LOCK() #define ASSERT_TYPE_LOCK_HELD() #endif @@ -728,7 +740,7 @@ _PyType_InitCache(PyInterpreterState *interp) assert(entry->name == NULL); entry->version = 0; - // Set to None so _PyType_Lookup() can use Py_SETREF(), + // Set to None so _PyType_LookupRef() can use Py_SETREF(), // rather than using slower Py_XSETREF(). entry->name = Py_None; entry->value = NULL; @@ -740,7 +752,7 @@ static unsigned int _PyType_ClearCache(PyInterpreterState *interp) { struct type_cache *cache = &interp->types.type_cache; - // Set to None, rather than NULL, so _PyType_Lookup() can + // Set to None, rather than NULL, so _PyType_LookupRef() can // use Py_SETREF() rather than using slower Py_XSETREF(). type_cache_clear(cache, Py_None); @@ -849,6 +861,44 @@ PyType_Unwatch(int watcher_id, PyObject* obj) return 0; } +#ifdef Py_GIL_DISABLED + +static void +type_modification_starting_unlocked(PyTypeObject *type) +{ + ASSERT_TYPE_LOCK_HELD(); + + /* Clear version tags on all types, but leave the valid + version tag intact. This prepares for a modification so that + any concurrent readers of the type cache will not see invalid + values. + */ + if (!_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)) { + return; + } + + PyObject *subclasses = lookup_tp_subclasses(type); + if (subclasses != NULL) { + assert(PyDict_CheckExact(subclasses)); + + Py_ssize_t i = 0; + PyObject *ref; + while (PyDict_Next(subclasses, &i, NULL, &ref)) { + PyTypeObject *subclass = type_from_ref(ref); + if (subclass == NULL) { + continue; + } + type_modification_starting_unlocked(subclass); + Py_DECREF(subclass); + } + } + + /* 0 is not a valid version tag */ + _Py_atomic_store_uint32_release(&type->tp_version_tag, 0); +} + +#endif + static void type_modified_unlocked(PyTypeObject *type) { @@ -882,7 +932,7 @@ type_modified_unlocked(PyTypeObject *type) if (subclass == NULL) { continue; } - PyType_Modified(subclass); + type_modified_unlocked(subclass); Py_DECREF(subclass); } } @@ -2352,7 +2402,7 @@ PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) Variants: - _PyObject_LookupSpecial() returns NULL without raising an exception - when the _PyType_Lookup() call fails; + when the _PyType_LookupRef() call fails; - lookup_maybe_method() and lookup_method() are internal routines similar to _PyObject_LookupSpecial(), but can return unbound PyFunction @@ -2365,13 +2415,12 @@ _PyObject_LookupSpecial(PyObject *self, PyObject *attr) { PyObject *res; - res = _PyType_Lookup(Py_TYPE(self), attr); + res = _PyType_LookupRef(Py_TYPE(self), attr); if (res != NULL) { descrgetfunc f; - if ((f = Py_TYPE(res)->tp_descr_get) == NULL) - Py_INCREF(res); - else - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if ((f = Py_TYPE(res)->tp_descr_get) != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); + } } return res; } @@ -2388,7 +2437,7 @@ _PyObject_LookupSpecialId(PyObject *self, _Py_Identifier *attrid) static PyObject * lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) { - PyObject *res = _PyType_Lookup(Py_TYPE(self), attr); + PyObject *res = _PyType_LookupRef(Py_TYPE(self), attr); if (res == NULL) { return NULL; } @@ -2396,16 +2445,12 @@ lookup_maybe_method(PyObject *self, PyObject *attr, int *unbound) if (_PyType_HasFeature(Py_TYPE(res), Py_TPFLAGS_METHOD_DESCRIPTOR)) { /* Avoid temporary PyMethodObject */ *unbound = 1; - Py_INCREF(res); } else { *unbound = 0; descrgetfunc f = Py_TYPE(res)->tp_descr_get; - if (f == NULL) { - Py_INCREF(res); - } - else { - res = f(res, self, (PyObject *)(Py_TYPE(self))); + if (f != NULL) { + Py_SETREF(res, f(res, self, (PyObject *)(Py_TYPE(self)))); } } return res; @@ -4981,14 +5026,13 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - res = _PyDict_GetItem_KnownHash(dict, name, hash); - if (res != NULL) { - break; - } - if (PyErr_Occurred()) { + if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { *error = -1; goto done; } + if (res != NULL) { + break; + } } *error = 0; done: @@ -5030,8 +5074,6 @@ update_cache(struct type_cache_entry *entry, PyObject *name, unsigned int versio #if Py_GIL_DISABLED -#define TYPE_CACHE_IS_UPDATING(sequence) (sequence & 0x01) - static void update_cache_gil_disabled(struct type_cache_entry *entry, PyObject *name, unsigned int version_tag, PyObject *value) @@ -5075,7 +5117,7 @@ _PyTypes_AfterFork(void) /* Internal API to look for a name through the MRO. This returns a borrowed reference, and doesn't set an exception! */ PyObject * -_PyType_Lookup(PyTypeObject *type, PyObject *name) +_PyType_LookupRef(PyTypeObject *type, PyObject *name) { PyObject *res; int error; @@ -5089,19 +5131,25 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) while (1) { int sequence = _PySeqLock_BeginRead(&entry->sequence); uint32_t entry_version = _Py_atomic_load_uint32_relaxed(&entry->version); - uint32_t type_version = _Py_atomic_load_uint32_relaxed(&type->tp_version_tag); + uint32_t type_version = _Py_atomic_load_uint32_acquire(&type->tp_version_tag); if (entry_version == type_version && _Py_atomic_load_ptr_relaxed(&entry->name) == name) { - assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); PyObject *value = _Py_atomic_load_ptr_relaxed(&entry->value); - // If the sequence is still valid then we're done - if (_PySeqLock_EndRead(&entry->sequence, sequence)) { - return value; + if (value == NULL || _Py_TryIncref(value)) { + if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + return value; + } + Py_XDECREF(value); + } + else { + // If we can't incref the object we need to fallback to locking + break; } - } else { + } + else { // cache miss break; } @@ -5112,6 +5160,7 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) assert(_PyType_HasFeature(type, Py_TPFLAGS_VALID_VERSION_TAG)); OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); + Py_XINCREF(entry->value); return entry->value; } #endif @@ -5161,6 +5210,14 @@ _PyType_Lookup(PyTypeObject *type, PyObject *name) return res; } +PyObject * +_PyType_Lookup(PyTypeObject *type, PyObject *name) +{ + PyObject *res = _PyType_LookupRef(type, name); + Py_XDECREF(res); + return res; +} + PyObject * _PyType_LookupId(PyTypeObject *type, _Py_Identifier *name) { @@ -5218,7 +5275,7 @@ _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. The argument suppress_missing_attribute is used to provide a fast path for hasattr. The possible values are: @@ -5254,10 +5311,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin meta_get = NULL; /* Look for the attribute in the metatype */ - meta_attribute = _PyType_Lookup(metatype, name); + meta_attribute = _PyType_LookupRef(metatype, name); if (meta_attribute != NULL) { - Py_INCREF(meta_attribute); meta_get = Py_TYPE(meta_attribute)->tp_descr_get; if (meta_get != NULL && PyDescr_IsData(meta_attribute)) { @@ -5274,10 +5330,9 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin /* No data descriptor found on metatype. Look in tp_dict of this * type and its bases */ - attribute = _PyType_Lookup(type, name); + attribute = _PyType_LookupRef(type, name); if (attribute != NULL) { /* Implement descriptor functionality, if any */ - Py_INCREF(attribute); descrgetfunc local_get = Py_TYPE(attribute)->tp_descr_get; Py_XDECREF(meta_attribute); @@ -5322,7 +5377,7 @@ _Py_type_getattro_impl(PyTypeObject *type, PyObject *name, int * suppress_missin } /* This is similar to PyObject_GenericGetAttr(), - but uses _PyType_Lookup() instead of just looking in type->tp_dict. */ + but uses _PyType_LookupRef() instead of just looking in type->tp_dict. */ PyObject * _Py_type_getattro(PyObject *type, PyObject *name) { @@ -5341,49 +5396,110 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) name, type->tp_name); return -1; } - if (PyUnicode_Check(name)) { - if (PyUnicode_CheckExact(name)) { - Py_INCREF(name); + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "attribute name must be string, not '%.200s'", + Py_TYPE(name)->tp_name); + return -1; + } + + if (PyUnicode_CheckExact(name)) { + Py_INCREF(name); + } + else { + name = _PyUnicode_Copy(name); + if (name == NULL) + return -1; + } + /* bpo-40521: Interned strings are shared by all subinterpreters */ + if (!PyUnicode_CHECK_INTERNED(name)) { + PyUnicode_InternInPlace(&name); + if (!PyUnicode_CHECK_INTERNED(name)) { + PyErr_SetString(PyExc_MemoryError, + "Out of memory interning an attribute name"); + Py_DECREF(name); + return -1; } - else { - name = _PyUnicode_Copy(name); - if (name == NULL) - return -1; + } + + PyTypeObject *metatype = Py_TYPE(type); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); + assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); + + PyObject *old_value; + PyObject *descr = _PyType_LookupRef(metatype, name); + if (descr != NULL) { + descrsetfunc f = Py_TYPE(descr)->tp_descr_set; + if (f != NULL) { + old_value = NULL; + res = f(descr, (PyObject *)type, value); + goto done; } - /* bpo-40521: Interned strings are shared by all subinterpreters */ - if (!PyUnicode_CHECK_INTERNED(name)) { - PyUnicode_InternInPlace(&name); - if (!PyUnicode_CHECK_INTERNED(name)) { - PyErr_SetString(PyExc_MemoryError, - "Out of memory interning an attribute name"); - Py_DECREF(name); - return -1; - } + } + + PyObject *dict = type->tp_dict; + if (dict == NULL) { + // We don't just do PyType_Ready because we could already be readying + BEGIN_TYPE_LOCK(); + dict = type->tp_dict; + if (dict == NULL) { + dict = type->tp_dict = PyDict_New(); + } + END_TYPE_LOCK(); + if (dict == NULL) { + return -1; } } - else { - /* Will fail in _PyObject_GenericSetAttrWithDict. */ - Py_INCREF(name); + + // We don't want any re-entrancy between when we update the dict + // and call type_modified_unlocked, including running the destructor + // of the current value as it can observe the cache in an inconsistent + // state. Because we have an exact unicode and our dict has exact + // unicodes we know that this will all complete without releasing + // the locks. + BEGIN_TYPE_DICT_LOCK(dict); + + if (_PyDict_GetItemRef_Unicode_LockHeld((PyDictObject *)dict, name, &old_value) < 0) { + return -1; } - BEGIN_TYPE_LOCK() - res = _PyObject_GenericSetAttrWithDict((PyObject *)type, name, value, NULL); - if (res == 0) { - /* Clear the VALID_VERSION flag of 'type' and all its - subclasses. This could possibly be unified with the - update_subclasses() recursion in update_slot(), but carefully: - they each have their own conditions on which to stop - recursing into subclasses. */ - type_modified_unlocked(type); +#ifdef Py_GIL_DISABLED + // In free-threaded builds readers can race with the lock-free portion + // of the type cache and the assignment into the dict. We clear all of the + // versions initially so no readers will succeed in the lock-free case. + // They'll then block on the type lock until the update below is done. + type_modification_starting_unlocked(type); +#endif + + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, name, value); + /* Clear the VALID_VERSION flag of 'type' and all its + subclasses. This could possibly be unified with the + update_subclasses() recursion in update_slot(), but carefully: + they each have their own conditions on which to stop + recursing into subclasses. */ + type_modified_unlocked(type); + + if (res == 0) { if (is_dunder_name(name)) { res = update_slot(type, name); } - assert(_PyType_CheckConsistency(type)); } - END_TYPE_LOCK() + else if (PyErr_ExceptionMatches(PyExc_KeyError)) { + PyErr_Format(PyExc_AttributeError, + "type object '%.50s' has no attribute '%U'", + ((PyTypeObject*)type)->tp_name, name); + + _PyObject_SetAttributeErrorContext((PyObject *)type, name); + } + + assert(_PyType_CheckConsistency(type)); + END_TYPE_DICT_LOCK(); +done: Py_DECREF(name); + Py_XDECREF(descr); + Py_XDECREF(old_value); return res; } @@ -9343,33 +9459,33 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when the attribute is present. So we use - _PyType_Lookup and create the method only when needed, with + _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattr = _PyType_Lookup(tp, &_Py_ID(__getattr__)); + getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ tp->tp_getattro = _Py_slot_tp_getattro; return _Py_slot_tp_getattro(self, name); } - Py_INCREF(getattr); /* speed hack: we could use lookup_maybe, but that would resolve the method fully for each attribute lookup for classes with __getattr__, even when self has the default __getattribute__ - method. So we use _PyType_Lookup and create the method only when + method. So we use _PyType_LookupRef and create the method only when needed, with call_attribute. */ - getattribute = _PyType_Lookup(tp, &_Py_ID(__getattribute__)); + getattribute = _PyType_LookupRef(tp, &_Py_ID(__getattribute__)); if (getattribute == NULL || (Py_IS_TYPE(getattribute, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)getattribute)->d_wrapped == (void *)PyObject_GenericGetAttr)) { + Py_XDECREF(getattribute); res = _PyObject_GenericGetAttrWithDict(self, name, NULL, 1); /* if res == NULL with no exception set, then it must be an AttributeError suppressed by us. */ if (res == NULL && !PyErr_Occurred()) { res = call_attribute(self, getattr, name); } - } else { - Py_INCREF(getattribute); + } + else { res = call_attribute(self, getattribute, name); Py_DECREF(getattribute); if (res == NULL && PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -9476,7 +9592,7 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyTypeObject *tp = Py_TYPE(self); PyObject *get; - get = _PyType_Lookup(tp, &_Py_ID(__get__)); + get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) @@ -9488,7 +9604,9 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) if (type == NULL) type = Py_None; PyObject *stack[3] = {self, obj, type}; - return PyObject_Vectorcall(get, stack, 3, NULL); + PyObject *res = PyObject_Vectorcall(get, stack, 3, NULL); + Py_DECREF(get); + return res; } static int @@ -10383,6 +10501,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type->tp_flags &= ~Py_TPFLAGS_HAVE_VECTORCALL; } } + Py_DECREF(descr); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; From e5353d49dc53632e694a5df485fafd47f6b98c91 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 6 May 2024 11:34:13 -0700 Subject: [PATCH 162/220] GH-83151: Add closure support to pdb (GH-111094) --- Lib/pdb.py | 90 ++++++++++++++++++- Lib/test/test_pdb.py | 65 +++++++++++++- ...3-10-20-03-50-17.gh-issue-83151.bcsD40.rst | 3 + 3 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index b2bd2d79384e3b0..e507a9bb896611b 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -77,10 +77,12 @@ import code import glob import token +import types import codeop import pprint import signal import inspect +import textwrap import tokenize import traceback import linecache @@ -624,11 +626,96 @@ def _disable_command_completion(self): self.completenames = completenames return + def _exec_in_closure(self, source, globals, locals): + """ Run source code in closure so code object created within source + can find variables in locals correctly + + returns True if the source is executed, False otherwise + """ + + # Determine if the source should be executed in closure. Only when the + # source compiled to multiple code objects, we should use this feature. + # Otherwise, we can just raise an exception and normal exec will be used. + + code = compile(source, "", "exec") + if not any(isinstance(const, CodeType) for const in code.co_consts): + return False + + # locals could be a proxy which does not support pop + # copy it first to avoid modifying the original locals + locals_copy = dict(locals) + + locals_copy["__pdb_eval__"] = { + "result": None, + "write_back": {} + } + + # If the source is an expression, we need to print its value + try: + compile(source, "", "eval") + except SyntaxError: + pass + else: + source = "__pdb_eval__['result'] = " + source + + # Add write-back to update the locals + source = ("try:\n" + + textwrap.indent(source, " ") + "\n" + + "finally:\n" + + " __pdb_eval__['write_back'] = locals()") + + # Build a closure source code with freevars from locals like: + # def __pdb_outer(): + # var = None + # def __pdb_scope(): # This is the code object we want to execute + # nonlocal var + # + # return __pdb_scope.__code__ + source_with_closure = ("def __pdb_outer():\n" + + "\n".join(f" {var} = None" for var in locals_copy) + "\n" + + " def __pdb_scope():\n" + + "\n".join(f" nonlocal {var}" for var in locals_copy) + "\n" + + textwrap.indent(source, " ") + "\n" + + " return __pdb_scope.__code__" + ) + + # Get the code object of __pdb_scope() + # The exec fills locals_copy with the __pdb_outer() function and we can call + # that to get the code object of __pdb_scope() + ns = {} + try: + exec(source_with_closure, {}, ns) + except Exception: + return False + code = ns["__pdb_outer"]() + + cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars) + + try: + exec(code, globals, locals_copy, closure=cells) + except Exception: + return False + + # get the data we need from the statement + pdb_eval = locals_copy["__pdb_eval__"] + + # __pdb_eval__ should not be updated back to locals + pdb_eval["write_back"].pop("__pdb_eval__") + + # Write all local variables back to locals + locals.update(pdb_eval["write_back"]) + eval_result = pdb_eval["result"] + if eval_result is not None: + print(repr(eval_result)) + + return True + def default(self, line): if line[:1] == '!': line = line[1:].strip() locals = self.curframe_locals globals = self.curframe.f_globals try: + buffer = line if (code := codeop.compile_command(line + '\n', '', 'single')) is None: # Multi-line mode with self._disable_command_completion(): @@ -661,7 +748,8 @@ def default(self, line): sys.stdin = self.stdin sys.stdout = self.stdout sys.displayhook = self.displayhook - exec(code, globals, locals) + if not self._exec_in_closure(buffer, globals, locals): + exec(code, globals, locals) finally: sys.stdout = save_stdout sys.stdin = save_stdin diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 82f3fbc26b90d95..f47466410082ef5 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2224,8 +2224,71 @@ def test_pdb_multiline_statement(): (Pdb) c """ +def test_pdb_closure(): + """Test for all expressions/statements that involve closure + + >>> k = 0 + >>> g = 1 + >>> def test_function(): + ... x = 2 + ... g = 3 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'k', + ... 'g', + ... 'y = y', + ... 'global g; g', + ... 'global g; (lambda: g)()', + ... '(lambda: x)()', + ... '(lambda: g)()', + ... 'lst = [n for n in range(10) if (n % x) == 0]', + ... 'lst', + ... 'sum(n for n in lst if n > x)', + ... 'x = 1; raise Exception()', + ... 'x', + ... 'def f():', + ... ' return x', + ... '', + ... 'f()', + ... 'c' + ... ]): + ... test_function() + > (4)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) k + 0 + (Pdb) g + 3 + (Pdb) y = y + *** NameError: name 'y' is not defined + (Pdb) global g; g + 1 + (Pdb) global g; (lambda: g)() + 1 + (Pdb) (lambda: x)() + 2 + (Pdb) (lambda: g)() + 3 + (Pdb) lst = [n for n in range(10) if (n % x) == 0] + (Pdb) lst + [0, 2, 4, 6, 8] + (Pdb) sum(n for n in lst if n > x) + 18 + (Pdb) x = 1; raise Exception() + *** Exception + (Pdb) x + 1 + (Pdb) def f(): + ... return x + ... + (Pdb) f() + 1 + (Pdb) c + """ + def test_pdb_show_attribute_and_item(): - """Test for multiline statement + """Test for expressions with command prefix >>> def test_function(): ... n = lambda x: x diff --git a/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst b/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst new file mode 100644 index 000000000000000..aaefbb9e2a6d3df --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-20-03-50-17.gh-issue-83151.bcsD40.rst @@ -0,0 +1,3 @@ +Enabled arbitrary statements and evaluations in :mod:`pdb` shell to access the +local variables of the current frame, which made it possible for multi-scope +code like generators or nested function to work. From 325a1da18d672171e9b9f794c1bfa7b79a7e5e59 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Mon, 6 May 2024 21:16:08 +0200 Subject: [PATCH 163/220] gh-115119: Default to --with-system-libmpdec=yes (#118539) Co-authored-by: Zachary Ware --- .github/workflows/posix-deps-apt.sh | 1 + Doc/using/configure.rst | 11 ++- Doc/whatsnew/3.13.rst | 5 ++ ...-05-06-00-39-06.gh-issue-115119.LT27pF.rst | 2 + configure | 84 +++++++++++++++---- configure.ac | 33 +++++++- 6 files changed, 116 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-05-06-00-39-06.gh-issue-115119.LT27pF.rst diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 0800401f4cd1139..fb485bd4f82bd2f 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -15,6 +15,7 @@ apt-get -yq install \ libgdbm-dev \ libgdbm-compat-dev \ liblzma-dev \ + libmpdec-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e662c0dafdb8de1..d30356d2058effb 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -804,11 +804,18 @@ Libraries options .. option:: --with-system-libmpdec - Build the ``_decimal`` extension module using an installed ``mpdec`` - library, see the :mod:`decimal` module (default is no). + Build the ``_decimal`` extension module using an installed ``mpdecimal`` + library, see the :mod:`decimal` module (default is yes). .. versionadded:: 3.3 + .. versionchanged:: 3.13 + Default to using the installed ``mpdecimal`` library. + + .. deprecated-removed:: 3.13 3.15 + A copy of the ``mpdecimal`` library sources will no longer be distributed + with Python 3.15. + .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. .. option:: --with-readline=readline|editline diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d573d8109edef86..43934baeb33b8a6 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1807,6 +1807,10 @@ Changes in the Python API Build Changes ============= +* The :file:`configure` option :option:`--with-system-libmpdec` now defaults + to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python + 3.15. + * Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate the :file:`configure` script. (Contributed by Christian Heimes in :gh:`89886`.) @@ -2336,6 +2340,7 @@ Pending Removal in Python 3.14 Pending Removal in Python 3.15 ------------------------------ +* The bundled copy of ``libmpdecimal``. * :c:func:`PyImport_ImportModuleNoBlock`: use :c:func:`PyImport_ImportModule`. * :c:func:`PyWeakref_GET_OBJECT`: use :c:func:`PyWeakref_GetRef` instead. * :c:func:`PyWeakref_GetObject`: use :c:func:`PyWeakref_GetRef` instead. diff --git a/Misc/NEWS.d/next/Build/2024-05-06-00-39-06.gh-issue-115119.LT27pF.rst b/Misc/NEWS.d/next/Build/2024-05-06-00-39-06.gh-issue-115119.LT27pF.rst new file mode 100644 index 000000000000000..a3e25388416a2f6 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-06-00-39-06.gh-issue-115119.LT27pF.rst @@ -0,0 +1,2 @@ +The :file:`configure` option :option:`--with-system-libmpdec` now defaults to ``yes``. +The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. diff --git a/configure b/configure index 0cd137198d0bb71..de426e6b686e68a 100755 --- a/configure +++ b/configure @@ -1892,8 +1892,9 @@ Optional Packages: --with-libs='lib1 ...' link against additional libs (default is no) --with-system-expat build pyexpat module using an installed expat library, see Doc/library/pyexpat.rst (default is no) - --with-system-libmpdec build _decimal module using an installed libmpdec - library, see Doc/library/decimal.rst (default is no) + --with-system-libmpdec build _decimal module using an installed mpdecimal + library, see Doc/library/decimal.rst (default is + yes) --with-decimal-contextvar build _decimal module using a coroutine-local rather than a thread-local context (default is yes) @@ -14611,7 +14612,7 @@ if test ${with_system_libmpdec+y} then : withval=$with_system_libmpdec; else $as_nop - with_system_libmpdec="no" + with_system_libmpdec="yes" fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $with_system_libmpdec" >&5 @@ -14621,19 +14622,19 @@ if test "x$with_system_libmpdec" = xyes then : pkg_failed=no -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec" >&5 -printf %s "checking for libmpdec... " >&6; } +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for libmpdec >= 2.5.0" >&5 +printf %s "checking for libmpdec >= 2.5.0... " >&6; } if test -n "$LIBMPDEC_CFLAGS"; then pkg_cv_LIBMPDEC_CFLAGS="$LIBMPDEC_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libmpdec") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec" 2>/dev/null` + pkg_cv_LIBMPDEC_CFLAGS=`$PKG_CONFIG --cflags "libmpdec >= 2.5.0" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -14645,12 +14646,12 @@ if test -n "$LIBMPDEC_LIBS"; then pkg_cv_LIBMPDEC_LIBS="$LIBMPDEC_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec\""; } >&5 - ($PKG_CONFIG --exists --print-errors "libmpdec") 2>&5 + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"libmpdec >= 2.5.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "libmpdec >= 2.5.0") 2>&5 ac_status=$? printf "%s\n" "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec" 2>/dev/null` + pkg_cv_LIBMPDEC_LIBS=`$PKG_CONFIG --libs "libmpdec >= 2.5.0" 2>/dev/null` test "x$?" != "x0" && pkg_failed=yes else pkg_failed=yes @@ -14671,9 +14672,9 @@ else _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec" 2>&1` + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` else - LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec" 2>&1` + LIBMPDEC_PKG_ERRORS=`$PKG_CONFIG --print-errors --cflags --libs "libmpdec >= 2.5.0" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$LIBMPDEC_PKG_ERRORS" >&5 @@ -14700,6 +14701,61 @@ else $as_nop LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" fi +if test "x$with_system_libmpdec" = xyes +then : + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + +int +main (void) +{ +const char *x = mpd_version(); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + have_mpdec=yes +else $as_nop + have_mpdec=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + +else $as_nop + have_mpdec=yes + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} +fi + +if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; unable to build _decimal" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; unable to build _decimal" >&2;} +fi + # Disable forced inlining in debug builds, see GH-94847 if test "x$with_pydebug" = xyes then : @@ -30426,7 +30482,7 @@ then : if true then : - if true + if test "$have_mpdec" = "yes" then : py_cv_module__decimal=yes else $as_nop diff --git a/configure.ac b/configure.ac index ae594517df5365b..8eb967674875927 100644 --- a/configure.ac +++ b/configure.ac @@ -3974,16 +3974,16 @@ AC_ARG_WITH( [system_libmpdec], [AS_HELP_STRING( [--with-system-libmpdec], - [build _decimal module using an installed libmpdec library, see Doc/library/decimal.rst (default is no)] + [build _decimal module using an installed mpdecimal library, see Doc/library/decimal.rst (default is yes)] )], [], - [with_system_libmpdec="no"]) + [with_system_libmpdec="yes"]) AC_MSG_RESULT([$with_system_libmpdec]) AS_VAR_IF( [with_system_libmpdec], [yes], [PKG_CHECK_MODULES( - [LIBMPDEC], [libmpdec], [], + [LIBMPDEC], [libmpdec >= 2.5.0], [], [LIBMPDEC_CFLAGS=${LIBMPDEC_CFLAGS-""} LIBMPDEC_LIBS=${LIBMPDEC_LIBS-"-lmpdec -lm"} LIBMPDEC_INTERNAL=])], @@ -3991,6 +3991,29 @@ AS_VAR_IF( LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)"]) +AS_VAR_IF([with_system_libmpdec], [yes], + [WITH_SAVE_ENV([ + CPPFLAGS="$LIBMPDEC_CFLAGS $CPPFLAGS" + LIBS="$LIBMPDEC_LIBS $LIBS" + + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([ + #include + #if MPD_VERSION_HEX < 0x02050000 + # error "mpdecimal 2.5.0 or higher required" + #endif + ], [const char *x = mpd_version();])], + [have_mpdec=yes], + [have_mpdec=no]) + ])], + [AS_VAR_SET([have_mpdec], [yes]) + AC_MSG_WARN([m4_normalize([ + the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; + consider using a system installed mpdecimal library.])])]) + +AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], + [AC_MSG_WARN([no system libmpdecimal found; unable to build _decimal])]) + # Disable forced inlining in debug builds, see GH-94847 AS_VAR_IF( [with_pydebug], [yes], @@ -7668,7 +7691,9 @@ PY_STDLIB_MOD([_curses_panel], [], [test "$have_panel" != "no"], [$PANEL_CFLAGS $CURSES_CFLAGS], [$PANEL_LIBS $CURSES_LIBS] ) -PY_STDLIB_MOD([_decimal], [], [], [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) +PY_STDLIB_MOD([_decimal], + [], [test "$have_mpdec" = "yes"], + [$LIBMPDEC_CFLAGS], [$LIBMPDEC_LIBS]) PY_STDLIB_MOD([_dbm], [test -n "$with_dbmliborder"], [test "$have_dbm" != "no"], [$DBM_CFLAGS], [$DBM_LIBS]) From 67ce820d51d7f4bb7068ed560f4f2e2d83dd30d5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 6 May 2024 22:25:20 +0300 Subject: [PATCH 164/220] gh-71592: Fix a leak in tkinter.Tk destructor when _debug is true (GH-118664) --- Modules/_tkinter.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 163449a927cf188..fc8af244adf4ed4 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2972,6 +2972,7 @@ Tkapp_Dealloc(PyObject *self) ENTER_TCL Tcl_DeleteInterp(Tkapp_Interp(self)); LEAVE_TCL + Py_XDECREF(((TkappObject *)self)->trace); PyObject_Free(self); Py_DECREF(tp); DisableEventHook(); From 9bf00322ba09e2439a78fcec77dc6917e1391e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Mon, 6 May 2024 21:35:22 +0200 Subject: [PATCH 165/220] gh-118628: Don't display pyrepl warning on Windows (#118665) --- Lib/_pyrepl/__main__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Lib/_pyrepl/__main__.py b/Lib/_pyrepl/__main__.py index 417ee17adc83d3b..c598019e7cd4ade 100644 --- a/Lib/_pyrepl/__main__.py +++ b/Lib/_pyrepl/__main__.py @@ -1,7 +1,8 @@ import os import sys -CAN_USE_PYREPL = True +CAN_USE_PYREPL = sys.platform != "win32" + def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): global CAN_USE_PYREPL @@ -21,7 +22,7 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): sys.ps1 = ">>> " if not hasattr(sys, "ps2"): sys.ps2 = "... " - # + run_interactive = None try: import errno @@ -33,7 +34,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): from .simple_interact import run_multiline_interactive_console run_interactive = run_multiline_interactive_console except Exception as e: - print(f"warning: can't use pyrepl: {e}", file=sys.stderr) + from .trace import trace + msg = f"warning: can't use pyrepl: {e}" + trace(msg) + print(msg, file=sys.stderr) CAN_USE_PYREPL = False if run_interactive is None: return sys._baserepl() From 00d913c6718aa365027c6dcf850e8f40731e54fc Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 6 May 2024 13:06:09 -0700 Subject: [PATCH 166/220] gh-118415: Fix issues with local tracing being enabled/disabled on a function (#118496) --- .../test_free_threading/test_monitoring.py | 39 +++++--- Python/instrumentation.c | 91 +++++++++---------- 2 files changed, 71 insertions(+), 59 deletions(-) diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index e170840ca3cb277..fa588046bf9923d 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -8,20 +8,14 @@ from sys import monitoring from test.support import is_wasi -from threading import Thread +from threading import Thread, _PyRLock from unittest import TestCase class InstrumentationMultiThreadedMixin: - if not hasattr(sys, "gettotalrefcount"): - thread_count = 50 - func_count = 1000 - fib = 15 - else: - # Run a little faster in debug builds... - thread_count = 25 - func_count = 500 - fib = 15 + thread_count = 10 + func_count = 200 + fib = 12 def after_threads(self): """Runs once after all the threads have started""" @@ -175,9 +169,6 @@ def during_threads(self): @unittest.skipIf(is_wasi, "WASI has no threads.") class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" - thread_count = 25 - func_count = 200 - fib = 15 def setUp(self): self.set = False @@ -227,6 +218,28 @@ def test_register_callback(self): for ref in self.refs: self.assertEqual(ref(), None) + def test_set_local_trace_opcodes(self): + def trace(frame, event, arg): + frame.f_trace_opcodes = True + return trace + + sys.settrace(trace) + try: + l = _PyRLock() + + def f(): + for i in range(3000): + with l: + pass + + t = Thread(target=f) + t.start() + for i in range(3000): + with l: + pass + t.join() + finally: + sys.settrace(None) if __name__ == "__main__": unittest.main() diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 72c9d2af5b3202b..77d3489afcfc722 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -459,7 +459,6 @@ dump_instrumentation_data(PyCodeObject *code, int star, FILE*out) } #define CHECK(test) do { \ - ASSERT_WORLD_STOPPED_OR_LOCKED(code); \ if (!(test)) { \ dump_instrumentation_data(code, i, stderr); \ } \ @@ -516,10 +515,6 @@ sanity_check_instrumentation(PyCodeObject *code) if (!is_instrumented(opcode)) { CHECK(_PyOpcode_Deopt[opcode] == opcode); } - if (data->per_instruction_tools) { - uint8_t tools = active_monitors.tools[PY_MONITORING_EVENT_INSTRUCTION]; - CHECK((tools & data->per_instruction_tools[i]) == data->per_instruction_tools[i]); - } } if (opcode == INSTRUMENTED_LINE) { CHECK(data->lines); @@ -677,8 +672,6 @@ de_instrument_per_instruction(PyCodeObject *code, int i) } assert(*opcode_ptr != INSTRUMENTED_INSTRUCTION); assert(instr->op.code != INSTRUMENTED_INSTRUCTION); - /* Keep things clean for sanity check */ - code->_co_monitoring->per_instruction_opcodes[i] = 0; } @@ -992,6 +985,7 @@ set_global_version(PyThreadState *tstate, uint32_t version) static bool is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); return global_version(interp) == code->_co_instrumentation_version; } @@ -999,11 +993,24 @@ is_version_up_to_date(PyCodeObject *code, PyInterpreterState *interp) static bool instrumentation_cross_checks(PyInterpreterState *interp, PyCodeObject *code) { + ASSERT_WORLD_STOPPED_OR_LOCKED(code); _Py_LocalMonitors expected = local_union( interp->monitors, code->_co_monitoring->local_monitors); return monitors_equals(code->_co_monitoring->active_monitors, expected); } + +static int +debug_check_sanity(PyInterpreterState *interp, PyCodeObject *code) +{ + int res; + LOCK_CODE(code); + res = is_version_up_to_date(code, interp) && + instrumentation_cross_checks(interp, code); + UNLOCK_CODE(); + return res; +} + #endif static inline uint8_t @@ -1018,8 +1025,7 @@ get_tools_for_instruction(PyCodeObject *code, PyInterpreterState *interp, int i, event = PY_MONITORING_EVENT_CALL; } if (PY_MONITORING_IS_INSTRUMENTED_EVENT(event)) { - CHECK(is_version_up_to_date(code, interp)); - CHECK(instrumentation_cross_checks(interp, code)); + CHECK(debug_check_sanity(interp, code)); if (code->_co_monitoring->tools) { tools = code->_co_monitoring->tools[i]; } @@ -1218,8 +1224,7 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, { PyCodeObject *code = _PyFrame_GetCode(frame); assert(tstate->tracing == 0); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); + assert(debug_check_sanity(tstate->interp, code)); int i = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *monitoring = code->_co_monitoring; @@ -1339,8 +1344,7 @@ int _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr) { PyCodeObject *code = _PyFrame_GetCode(frame); - assert(is_version_up_to_date(code, tstate->interp)); - assert(instrumentation_cross_checks(tstate->interp, code)); + assert(debug_check_sanity(tstate->interp, code)); int offset = (int)(instr - _PyCode_CODE(code)); _PyCoMonitoringData *instrumentation_data = code->_co_monitoring; assert(instrumentation_data->per_instruction_opcodes); @@ -1671,9 +1675,11 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ + // Initialize all of the instructions so if local events change while another thread is executing + // we know what the original opcode was. for (int i = 0; i < code_len; i++) { - code->_co_monitoring->per_instruction_opcodes[i] = 0; + int opcode = _PyCode_CODE(code)[i].op.code; + code->_co_monitoring->per_instruction_opcodes[i] = _PyOpcode_Deopt[opcode]; } } if (multitools && code->_co_monitoring->per_instruction_tools == NULL) { @@ -1682,7 +1688,6 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) PyErr_NoMemory(); return -1; } - /* This may not be necessary, as we can initialize this memory lazily, but it helps catch errors. */ for (int i = 0; i < code_len; i++) { code->_co_monitoring->per_instruction_tools[i] = 0; } @@ -1692,17 +1697,10 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) } static int -instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) +force_instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - if (is_version_up_to_date(code, interp)) { - assert( - interp->ceval.instrumentation_version == 0 || - instrumentation_cross_checks(interp, code) - ); - return 0; - } #ifdef _Py_TIER2 if (code->co_executors != NULL) { _PyCode_Clear_Executors(code); @@ -1769,7 +1767,6 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) // GH-103845: We need to remove both the line and instruction instrumentation before // adding new ones, otherwise we may remove the newly added instrumentation. - uint8_t removed_line_tools = removed_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t removed_per_instruction_tools = removed_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; @@ -1777,9 +1774,7 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { - if (removed_line_tools) { - remove_line_tools(code, i, removed_line_tools); - } + remove_line_tools(code, i, removed_line_tools); } i += _PyInstruction_GetLength(code, i); } @@ -1791,15 +1786,14 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); continue; } - if (removed_per_instruction_tools) { - remove_per_instruction_tools(code, i, removed_per_instruction_tools); - } + remove_per_instruction_tools(code, i, removed_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } #ifdef INSTRUMENT_DEBUG sanity_check_instrumentation(code); #endif + uint8_t new_line_tools = new_events.tools[PY_MONITORING_EVENT_LINE]; uint8_t new_per_instruction_tools = new_events.tools[PY_MONITORING_EVENT_INSTRUCTION]; @@ -1807,9 +1801,7 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; for (int i = code->_co_firsttraceable; i < code_len;) { if (line_data[i].original_opcode) { - if (new_line_tools) { - add_line_tools(code, i, new_line_tools); - } + add_line_tools(code, i, new_line_tools); } i += _PyInstruction_GetLength(code, i); } @@ -1821,12 +1813,11 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) i += _PyInstruction_GetLength(code, i); continue; } - if (new_per_instruction_tools) { - add_per_instruction_tools(code, i, new_per_instruction_tools); - } + add_per_instruction_tools(code, i, new_per_instruction_tools); i += _PyInstruction_GetLength(code, i); } } + done: FT_ATOMIC_STORE_UINTPTR_RELEASE(code->_co_instrumentation_version, global_version(interp)); @@ -1837,6 +1828,22 @@ instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) return 0; } +static int +instrument_lock_held(PyCodeObject *code, PyInterpreterState *interp) +{ + ASSERT_WORLD_STOPPED_OR_LOCKED(code); + + if (is_version_up_to_date(code, interp)) { + assert( + interp->ceval.instrumentation_version == 0 || + instrumentation_cross_checks(interp, code) + ); + return 0; + } + + return force_instrument_lock_held(code, interp); +} + int _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) { @@ -1983,16 +1990,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent goto done; } set_local_events(local, tool_id, events); - if (is_version_up_to_date(code, interp)) { - /* Force instrumentation update */ - code->_co_instrumentation_version -= MONITORING_VERSION_INCREMENT; - } - -#ifdef _Py_TIER2 - _Py_Executors_InvalidateDependency(interp, code, 1); -#endif - res = instrument_lock_held(code, interp); + res = force_instrument_lock_held(code, interp); done: UNLOCK_CODE(); From 616b745b89a52a1d27123107718f85e65918afdc Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Mon, 6 May 2024 22:21:06 +0100 Subject: [PATCH 167/220] GH-115709: Invalidate executors when a local variable is changed via frame.f_locals (#118639) Also fix unrelated assert in debug Tier2/JIT builds. --- Include/cpython/optimizer.h | 12 +++++++++--- Lib/test/test_capi/test_opt.py | 13 +++++++++++++ Objects/frameobject.c | 3 ++- Python/bytecodes.c | 3 +++ Python/generated_cases.c.h | 3 +++ 5 files changed, 30 insertions(+), 4 deletions(-) diff --git a/Include/cpython/optimizer.h b/Include/cpython/optimizer.h index 744a272251e75c1..5f218d75b346a02 100644 --- a/Include/cpython/optimizer.h +++ b/Include/cpython/optimizer.h @@ -141,9 +141,6 @@ void _Py_ExecutorDetach(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); -PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); -PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); - /* For testing */ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewCounter(void); PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); @@ -151,6 +148,15 @@ PyAPI_FUNC(PyObject *)PyUnstable_Optimizer_NewUOpOptimizer(void); #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 +#ifdef _Py_TIER2 +PyAPI_FUNC(void) _Py_Executors_InvalidateDependency(PyInterpreterState *interp, void *obj, int is_invalidation); +PyAPI_FUNC(void) _Py_Executors_InvalidateAll(PyInterpreterState *interp, int is_invalidation); +#else +# define _Py_Executors_InvalidateDependency(A, B, C) ((void)0) +# define _Py_Executors_InvalidateAll(A, B) ((void)0) +#endif + + #ifdef __cplusplus } #endif diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 6e5b626e93291ae..0491ff9b84d486c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1321,5 +1321,18 @@ def testfunc(n): self.assertIsNotNone(ex) self.assertIn("_FOR_ITER_GEN_FRAME", get_opnames(ex)) + def test_modified_local_is_seen_by_optimized_code(self): + l = sys._getframe().f_locals + a = 1 + s = 0 + for j in range(1 << 10): + a + a + l["xa"[j >> 9]] = 1.0 + s += a + self.assertIs(type(a), float) + self.assertIs(type(s), float) + self.assertEqual(s, 1024.0) + + if __name__ == "__main__": unittest.main() diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 869cdec8aa70627..26a04cbeea90bf7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -148,8 +148,9 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) if (PyUnicode_CheckExact(key)) { int i = framelocalsproxy_getkeyindex(frame, key, false); if (i >= 0) { - _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); + _Py_Executors_InvalidateDependency(PyInterpreterState_Get(), co, 1); + _PyLocals_Kind kind = _PyLocals_GetKind(co->co_localspluskinds, i); PyObject *oldvalue = fast[i]; PyObject *cell = NULL; if (kind == CO_FAST_FREE) { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b2a0dc030e20cc7..b2ddec98e682f27 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2424,6 +2424,9 @@ dummy_func( opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } DISPATCH_GOTO(); } tstate->previous_executor = Py_None; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 87098b0506522f0..d3126b0c9807238 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2673,6 +2673,9 @@ opcode = executor->vm_data.opcode; oparg = (oparg & ~255) | executor->vm_data.oparg; next_instr = this_instr; + if (_PyOpcode_Caches[_PyOpcode_Deopt[opcode]]) { + PAUSE_ADAPTIVE_COUNTER(this_instr[1].counter); + } DISPATCH_GOTO(); } tstate->previous_executor = Py_None; From 2ba2c142a615abbd8138d253edfe02426c386961 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 6 May 2024 17:24:14 -0400 Subject: [PATCH 168/220] gh-118527: Intern code name and filename on default build (#118576) Interned and non-interned strings are treated differently by `marshal`, so be consistent between the default and free-threaded build. --- Objects/codeobject.c | 2 -- Programs/test_frozenmain.h | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 15d3960fc2df512..2ce5f7dca2eb5cf 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -390,11 +390,9 @@ init_code(PyCodeObject *co, struct _PyCodeConstructor *con) co->co_filename = Py_NewRef(con->filename); co->co_name = Py_NewRef(con->name); co->co_qualname = Py_NewRef(con->qualname); -#ifdef Py_GIL_DISABLED PyUnicode_InternInPlace(&co->co_filename); PyUnicode_InternInPlace(&co->co_name); PyUnicode_InternInPlace(&co->co_qualname); -#endif co->co_flags = con->flags; co->co_firstlineno = con->firstlineno; diff --git a/Programs/test_frozenmain.h b/Programs/test_frozenmain.h index 657e9345cf5ab7d..3e7ab7bffb158ee 100644 --- a/Programs/test_frozenmain.h +++ b/Programs/test_frozenmain.h @@ -25,8 +25,8 @@ unsigned char M_test_frozenmain[] = { 108,99,97,112,105,218,5,112,114,105,110,116,218,4,97,114, 103,118,218,11,103,101,116,95,99,111,110,102,105,103,115,114, 3,0,0,0,218,3,107,101,121,169,0,243,0,0,0,0, - 250,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, - 110,46,112,121,250,8,60,109,111,100,117,108,101,62,114,18, + 218,18,116,101,115,116,95,102,114,111,122,101,110,109,97,105, + 110,46,112,121,218,8,60,109,111,100,117,108,101,62,114,18, 0,0,0,1,0,0,0,115,99,0,0,0,240,3,1,1, 1,243,8,0,1,11,219,0,24,225,0,5,208,6,26,212, 0,27,217,0,5,128,106,144,35,151,40,145,40,212,0,27, From 9fd33af5ac57f649971ffd8091bd898a96f85b55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 7 May 2024 00:08:17 +0200 Subject: [PATCH 169/220] Test premium Mac builders (#118672) --- .github/workflows/build.yml | 2 +- .github/workflows/reusable-macos.yml | 2 +- Lib/test/test_pyrepl.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aac395d5638750e..7d9e70a68005e3a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -208,7 +208,7 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} # macos-14 is M1, macos-13 is Intel - os-matrix: '["macos-14", "macos-13"]' + os-matrix: '["macos-14-xlarge", "macos-13-large"]' build_macos_free_threading: name: 'macOS (free-threading)' diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index e3319f141bd7f5b..d06a718d199c96d 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -50,7 +50,7 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - run: make -j4 + run: make -j8 - name: Display build info run: make pythoninfo - name: Tests diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 2b217abdf1543b4..3df76e02b231dc2 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -15,9 +15,9 @@ # Optionally test pyrepl. This currently requires that the # 'curses' resource be given on the regrtest command line using the -u # option. Additionally, we need to attempt to import curses and readline. -requires('curses') -curses = import_module('curses') -readline = import_module('readline') +requires("curses") +curses = import_module("curses") +readline = import_module("readline") from _pyrepl.console import Console, Event from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig From 8419f01673858d5002747c9393d4ed0ec22fdb47 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 6 May 2024 15:35:06 -0700 Subject: [PATCH 170/220] gh-118647: Add defaults to typing.Generator and typing.AsyncGenerator (#118648) Co-authored-by: Alex Waygood Co-authored-by: Nikita Sobolev --- Doc/library/typing.rst | 26 ++++++++++++++++--- Lib/test/test_typing.py | 11 ++++++++ Lib/typing.py | 21 +++++++++++---- ...-05-06-08-23-01.gh-issue-118648.OVA3jJ.rst | 2 ++ 4 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-06-08-23-01.gh-issue-118648.OVA3jJ.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index e06287250641e66..652a3f1f70519c4 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3648,8 +3648,14 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` is no ``ReturnType`` type parameter. As with :class:`Generator`, the ``SendType`` behaves contravariantly. - If your generator will only yield values, set the ``SendType`` to - ``None``:: + The ``SendType`` defaults to :const:`!None`:: + + async def infinite_stream(start: int) -> AsyncGenerator[int]: + while True: + yield start + start = await increment(start) + + It is also possible to set this type explicitly:: async def infinite_stream(start: int) -> AsyncGenerator[int, None]: while True: @@ -3671,6 +3677,9 @@ Aliases to asynchronous ABCs in :mod:`collections.abc` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + The ``SendType`` parameter now has a default. + .. class:: AsyncIterable(Generic[T_co]) Deprecated alias to :class:`collections.abc.AsyncIterable`. @@ -3754,8 +3763,14 @@ Aliases to other ABCs in :mod:`collections.abc` of :class:`Generator` behaves contravariantly, not covariantly or invariantly. - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: + The ``SendType`` and ``ReturnType`` parameters default to :const:`!None`:: + + def infinite_stream(start: int) -> Generator[int]: + while True: + yield start + start += 1 + + It is also possible to set these types explicitly:: def infinite_stream(start: int) -> Generator[int, None, None]: while True: @@ -3774,6 +3789,9 @@ Aliases to other ABCs in :mod:`collections.abc` :class:`collections.abc.Generator` now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Default values for the send and return types were added. + .. class:: Hashable Deprecated alias to :class:`collections.abc.Hashable`. diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 112db03ae878875..8f0be1fd3f55e54 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7289,6 +7289,17 @@ def foo(): g = foo() self.assertIsSubclass(type(g), typing.Generator) + def test_generator_default(self): + g1 = typing.Generator[int] + g2 = typing.Generator[int, None, None] + self.assertEqual(get_args(g1), (int, type(None), type(None))) + self.assertEqual(get_args(g1), get_args(g2)) + + g3 = typing.Generator[int, float] + g4 = typing.Generator[int, float, None] + self.assertEqual(get_args(g3), (int, float, type(None))) + self.assertEqual(get_args(g3), get_args(g4)) + def test_no_generator_instantiation(self): with self.assertRaises(TypeError): typing.Generator() diff --git a/Lib/typing.py b/Lib/typing.py index ff0e9b811a58670..c159fcfda68ee8f 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -1328,7 +1328,7 @@ def __getattr__(self, attr): raise AttributeError(attr) def __setattr__(self, attr, val): - if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams'}: + if _is_dunder(attr) or attr in {'_name', '_inst', '_nparams', '_defaults'}: super().__setattr__(attr, val) else: setattr(self.__origin__, attr, val) @@ -1578,11 +1578,12 @@ def __iter__(self): # parameters are accepted (needs custom __getitem__). class _SpecialGenericAlias(_NotIterable, _BaseGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None): + def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): if name is None: name = origin.__name__ super().__init__(origin, inst=inst, name=name) self._nparams = nparams + self._defaults = defaults if origin.__module__ == 'builtins': self.__doc__ = f'A generic version of {origin.__qualname__}.' else: @@ -1594,12 +1595,22 @@ def __getitem__(self, params): params = (params,) msg = "Parameters to generic types must be types." params = tuple(_type_check(p, msg) for p in params) + if (self._defaults + and len(params) < self._nparams + and len(params) + len(self._defaults) >= self._nparams + ): + params = (*params, *self._defaults[len(params) - self._nparams:]) actual_len = len(params) + if actual_len != self._nparams: + if self._defaults: + expected = f"at least {self._nparams - len(self._defaults)}" + else: + expected = str(self._nparams) if not self._nparams: raise TypeError(f"{self} is not a generic class") raise TypeError(f"Too {'many' if actual_len > self._nparams else 'few'} arguments for {self};" - f" actual {actual_len}, expected {self._nparams}") + f" actual {actual_len}, expected {expected}") return self.copy_with(params) def copy_with(self, params): @@ -2813,8 +2824,8 @@ class Other(Leaf): # Error reported by type checker OrderedDict = _alias(collections.OrderedDict, 2) Counter = _alias(collections.Counter, 1) ChainMap = _alias(collections.ChainMap, 2) -Generator = _alias(collections.abc.Generator, 3) -AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2) +Generator = _alias(collections.abc.Generator, 3, defaults=(types.NoneType, types.NoneType)) +AsyncGenerator = _alias(collections.abc.AsyncGenerator, 2, defaults=(types.NoneType,)) Type = _alias(type, 1, inst=False, name='Type') Type.__doc__ = \ """Deprecated alias to builtins.type. diff --git a/Misc/NEWS.d/next/Library/2024-05-06-08-23-01.gh-issue-118648.OVA3jJ.rst b/Misc/NEWS.d/next/Library/2024-05-06-08-23-01.gh-issue-118648.OVA3jJ.rst new file mode 100644 index 000000000000000..7695fb0ba20df81 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-06-08-23-01.gh-issue-118648.OVA3jJ.rst @@ -0,0 +1,2 @@ +Add type parameter defaults to :class:`typing.Generator` and +:class:`typing.AsyncGenerator`. From 5a9eeafa055a8724b1741316cfbd9612825220aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Tue, 7 May 2024 00:49:45 +0200 Subject: [PATCH 171/220] Use Intel with 12 cores for free-threading tests for maximum speedup (#118677) --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7d9e70a68005e3a..8f966591c302351 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -218,8 +218,8 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: true - # macos-14 is M1 - os-matrix: '["macos-14"]' + # macos-14-large is Intel with 12 cores (most parallelism) + os-matrix: '["macos-14-large"]' build_ubuntu: name: 'Ubuntu' From 040571f258d13a807f5c8e4ce0a182d5f9a2e81b Mon Sep 17 00:00:00 2001 From: denballakh <47365157+denballakh@users.noreply.github.com> Date: Tue, 7 May 2024 01:56:28 +0300 Subject: [PATCH 172/220] fix typo in `_pyrepl.pager`: `plainpager` -> `plain_pager` (#118675) --- Lib/_pyrepl/pager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/_pyrepl/pager.py b/Lib/_pyrepl/pager.py index ecf5ddc79a5efa3..af0409c4523bc29 100644 --- a/Lib/_pyrepl/pager.py +++ b/Lib/_pyrepl/pager.py @@ -23,7 +23,7 @@ def get_pager() -> Pager: if not sys.stdin.isatty() or not sys.stdout.isatty(): return plain_pager if sys.platform == "emscripten": - return plainpager + return plain_pager use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER') if use_pager: if sys.platform == 'win32': # pipes completely broken in Windows From e0422198fb4de0a5d81edd3de0d0ed32c119e9bb Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Mon, 6 May 2024 15:57:27 -0700 Subject: [PATCH 173/220] gh-117486: Improve behavior for user-defined AST subclasses (#118212) Now, such classes will no longer require changes in Python 3.13 in the normal case. The test suite for robotframework passes with no DeprecationWarnings under this PR. I also added a new DeprecationWarning for the case where `_field_types` exists but is incomplete, since that seems likely to indicate a user mistake. --- Doc/library/ast.rst | 14 ++++++- Doc/whatsnew/3.13.rst | 6 +++ Lib/test/test_ast.py | 41 +++++++++++++++++-- ...-04-23-21-17-00.gh-issue-117486.ea3KYD.rst | 4 ++ Parser/asdl_c.py | 31 +++++++------- Python/Python-ast.c | 31 +++++++------- 6 files changed, 94 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index e954c38c7c55b59..02dc7c86082502d 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -61,7 +61,7 @@ Node classes .. attribute:: _fields - Each concrete class has an attribute :attr:`_fields` which gives the names + Each concrete class has an attribute :attr:`!_fields` which gives the names of all child nodes. Each instance of a concrete class has one attribute for each child node, @@ -74,6 +74,18 @@ Node classes as Python lists. All possible attributes must be present and have valid values when compiling an AST with :func:`compile`. + .. attribute:: _field_types + + The :attr:`!_field_types` attribute on each concrete class is a dictionary + mapping field names (as also listed in :attr:`_fields`) to their types. + + .. doctest:: + + >>> ast.TypeVar._field_types + {'name': , 'bound': ast.expr | None, 'default_value': ast.expr | None} + + .. versionadded:: 3.13 + .. attribute:: lineno col_offset end_lineno diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 43934baeb33b8a6..c82d8bdb7342f41 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -384,6 +384,12 @@ ast argument that does not map to a field on the AST node is now deprecated, and will raise an exception in Python 3.15. + These changes do not apply to user-defined subclasses of :class:`ast.AST`, + unless the class opts in to the new behavior by setting the attribute + :attr:`ast.AST._field_types`. + + (Contributed by Jelle Zijlstra in :gh:`105858` and :gh:`117486`.) + * :func:`ast.parse` now accepts an optional argument *optimize* which is passed on to the :func:`compile` built-in. This makes it possible to obtain an optimized AST. diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 6d05c8f8f47c50d..f6e22d44406d9ee 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -3036,7 +3036,7 @@ def test_FunctionDef(self): self.assertEqual(node.name, 'foo') self.assertEqual(node.decorator_list, []) - def test_custom_subclass(self): + def test_custom_subclass_with_no_fields(self): class NoInit(ast.AST): pass @@ -3044,17 +3044,17 @@ class NoInit(ast.AST): self.assertIsInstance(obj, NoInit) self.assertEqual(obj.__dict__, {}) + def test_fields_but_no_field_types(self): class Fields(ast.AST): _fields = ('a',) - with self.assertWarnsRegex(DeprecationWarning, - r"Fields provides _fields but not _field_types."): - obj = Fields() + obj = Fields() with self.assertRaises(AttributeError): obj.a obj = Fields(a=1) self.assertEqual(obj.a, 1) + def test_fields_and_types(self): class FieldsAndTypes(ast.AST): _fields = ('a',) _field_types = {'a': int | None} @@ -3065,6 +3065,7 @@ class FieldsAndTypes(ast.AST): obj = FieldsAndTypes(a=1) self.assertEqual(obj.a, 1) + def test_fields_and_types_no_default(self): class FieldsAndTypesNoDefault(ast.AST): _fields = ('a',) _field_types = {'a': int} @@ -3077,6 +3078,38 @@ class FieldsAndTypesNoDefault(ast.AST): obj = FieldsAndTypesNoDefault(a=1) self.assertEqual(obj.a, 1) + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types" + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ('a', 'b') + _field_types = {'a': int | None, 'b': list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + @support.cpython_only class ModuleStateTests(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst b/Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst new file mode 100644 index 000000000000000..f02d895161cfdbb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-23-21-17-00.gh-issue-117486.ea3KYD.rst @@ -0,0 +1,4 @@ +Improve the behavior of user-defined subclasses of :class:`ast.AST`. Such +classes will now require no changes in the usual case to conform with the +behavior changes of the :mod:`ast` module in Python 3.13. Patch by Jelle +Zijlstra. diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 1f0be456655b25b..11d59faeb0d42ce 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -979,14 +979,9 @@ def visitModule(self, mod): goto cleanup; } if (field_types == NULL) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s provides _fields but not _field_types. " - "This will become an error in Python 3.15.", - Py_TYPE(self)->tp_name - ) < 0) { - res = -1; - } + // Probably a user-defined subclass of AST that lacks _field_types. + // This will continue to work as it did before 3.13; i.e., attributes + // that are not passed in simply do not exist on the instance. goto cleanup; } remaining_list = PySequence_List(remaining_fields); @@ -997,12 +992,21 @@ def visitModule(self, mod): PyObject *name = PyList_GET_ITEM(remaining_list, i); PyObject *type = PyDict_GetItemWithError(field_types, name); if (!type) { - if (!PyErr_Occurred()) { - PyErr_SetObject(PyExc_KeyError, name); + if (PyErr_Occurred()) { + goto set_remaining_cleanup; + } + else { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Field '%U' is missing from %.400s._field_types. " + "This will become an error in Python 3.15.", + name, Py_TYPE(self)->tp_name + ) < 0) { + goto set_remaining_cleanup; + } } - goto set_remaining_cleanup; } - if (_PyUnion_Check(type)) { + else if (_PyUnion_Check(type)) { // optional field // do nothing, we'll have set a None default on the class } @@ -1026,8 +1030,7 @@ def visitModule(self, mod): "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { - res = -1; - goto cleanup; + goto set_remaining_cleanup; } } } diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 1953142f6def44b..4956d04f719de99 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5178,14 +5178,9 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) goto cleanup; } if (field_types == NULL) { - if (PyErr_WarnFormat( - PyExc_DeprecationWarning, 1, - "%.400s provides _fields but not _field_types. " - "This will become an error in Python 3.15.", - Py_TYPE(self)->tp_name - ) < 0) { - res = -1; - } + // Probably a user-defined subclass of AST that lacks _field_types. + // This will continue to work as it did before 3.13; i.e., attributes + // that are not passed in simply do not exist on the instance. goto cleanup; } remaining_list = PySequence_List(remaining_fields); @@ -5196,12 +5191,21 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) PyObject *name = PyList_GET_ITEM(remaining_list, i); PyObject *type = PyDict_GetItemWithError(field_types, name); if (!type) { - if (!PyErr_Occurred()) { - PyErr_SetObject(PyExc_KeyError, name); + if (PyErr_Occurred()) { + goto set_remaining_cleanup; + } + else { + if (PyErr_WarnFormat( + PyExc_DeprecationWarning, 1, + "Field '%U' is missing from %.400s._field_types. " + "This will become an error in Python 3.15.", + name, Py_TYPE(self)->tp_name + ) < 0) { + goto set_remaining_cleanup; + } } - goto set_remaining_cleanup; } - if (_PyUnion_Check(type)) { + else if (_PyUnion_Check(type)) { // optional field // do nothing, we'll have set a None default on the class } @@ -5225,8 +5229,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { - res = -1; - goto cleanup; + goto set_remaining_cleanup; } } } From 430945db4c5589197c6d18462e89ca5ae9e38ba6 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 6 May 2024 16:21:39 -0700 Subject: [PATCH 174/220] GH-118251: Bump the JIT CI timeout to 90 minutes (#118661) --- .github/workflows/jit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index f68567c8a3153e8..7152cde8f4607c8 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -23,7 +23,7 @@ jobs: jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) runs-on: ${{ matrix.runner }} - timeout-minutes: 75 + timeout-minutes: 90 strategy: fail-fast: false matrix: From 636b8d94c91324c7e49a7cfe914f162690adf488 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 6 May 2024 16:31:09 -0700 Subject: [PATCH 175/220] gh-112075: Fix race in constructing dict for instance (#118499) --- Include/internal/pycore_dict.h | 2 +- Lib/test/test_free_threading/test_dict.py | 141 ++++++++++++++++++++++ Objects/dictobject.c | 140 ++++++++++----------- Objects/object.c | 4 +- 4 files changed, 216 insertions(+), 71 deletions(-) create mode 100644 Lib/test/test_free_threading/test_dict.py diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 3ba8ee74b4df870..cb7d4c3219a9afa 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -105,10 +105,10 @@ PyAPI_FUNC(PyObject *)_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObjec /* Consumes references to key and value */ PyAPI_FUNC(int) _PyDict_SetItem_Take2(PyDictObject *op, PyObject *key, PyObject *value); -extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value); extern int _PyDict_GetItemRef_Unicode_LockHeld(PyDictObject *op, PyObject *key, PyObject **result); extern int _PyDict_GetItemRef_KnownHash(PyDictObject *op, PyObject *key, Py_hash_t hash, PyObject **result); +extern int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *name, PyObject *value); extern int _PyDict_Pop_KnownHash( PyDictObject *dict, diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py new file mode 100644 index 000000000000000..6a909dd3ee025f8 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict.py @@ -0,0 +1,141 @@ +import gc +import time +import unittest +import weakref + +from ast import Or +from functools import partial +from threading import Thread +from unittest import TestCase + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestDict(TestCase): + def test_racing_creation_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with shared keys""" + class C(int): + pass + + self.racing_creation(C) + + def test_racing_creation_no_shared_keys(self): + """Verify that creating dictionaries is thread safe when we + have a type with an ordinary dict""" + self.racing_creation(Or) + + def test_racing_creation_inline_values_invalid(self): + """Verify that re-creating a dict after we have invalid inline values + is thread safe""" + class C: + pass + + def make_obj(): + a = C() + # Make object, make inline values invalid, and then delete dict + a.__dict__ = {} + del a.__dict__ + return a + + self.racing_creation(make_obj) + + def test_racing_creation_nonmanaged_dict(self): + """Verify that explicit creation of an unmanaged dict is thread safe + outside of the normal attribute setting code path""" + def make_obj(): + def f(): pass + return f + + def set(func, name, val): + # Force creation of the dict via PyObject_GenericGetDict + func.__dict__[name] = val + + self.racing_creation(make_obj, set) + + def racing_creation(self, cls, set=setattr): + objects = [] + processed = [] + + OBJECT_COUNT = 100 + THREAD_COUNT = 10 + CUR = 0 + + for i in range(OBJECT_COUNT): + objects.append(cls()) + + def writer_func(name): + last = -1 + while True: + if CUR == last: + continue + elif CUR == OBJECT_COUNT: + break + + obj = objects[CUR] + set(obj, name, name) + last = CUR + processed.append(name) + + writers = [] + for x in range(THREAD_COUNT): + writer = Thread(target=partial(writer_func, f"a{x:02}")) + writers.append(writer) + writer.start() + + for i in range(OBJECT_COUNT): + CUR = i + while len(processed) != THREAD_COUNT: + time.sleep(0.001) + processed.clear() + + CUR = OBJECT_COUNT + + for writer in writers: + writer.join() + + for obj_idx, obj in enumerate(objects): + assert ( + len(obj.__dict__) == THREAD_COUNT + ), f"{len(obj.__dict__)} {obj.__dict__!r} {obj_idx}" + for i in range(THREAD_COUNT): + assert f"a{i:02}" in obj.__dict__, f"a{i:02} missing at {obj_idx}" + + def test_racing_set_dict(self): + """Races assigning to __dict__ should be thread safe""" + + def f(): pass + l = [] + THREAD_COUNT = 10 + class MyDict(dict): pass + + def writer_func(l): + for i in range(1000): + d = MyDict() + l.append(weakref.ref(d)) + f.__dict__ = d + + lists = [] + writers = [] + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + f.__dict__ = {} + gc.collect() + + for thread_list in lists: + for ref in thread_list: + self.assertIsNone(ref()) + +if __name__ == "__main__": + unittest.main() diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 3e662e09ea598e5..b0fce09d7940e0a 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -924,16 +924,15 @@ new_dict(PyInterpreterState *interp, return (PyObject *)mp; } -/* Consumes a reference to the keys object */ static PyObject * new_dict_with_shared_keys(PyInterpreterState *interp, PyDictKeysObject *keys) { size_t size = shared_keys_usable_size(keys); PyDictValues *values = new_values(size); if (values == NULL) { - dictkeys_decref(interp, keys, false); return PyErr_NoMemory(); } + dictkeys_incref(keys); for (size_t i = 0; i < size; i++) { values->values[i] = NULL; } @@ -6693,8 +6692,6 @@ materialize_managed_dict_lock_held(PyObject *obj) { _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(obj); - OBJECT_STAT_INC(dict_materialized_on_request); - PyDictValues *values = _PyObject_InlineValues(obj); PyInterpreterState *interp = _PyInterpreterState_GET(); PyDictKeysObject *keys = CACHED_KEYS(Py_TYPE(obj)); @@ -7186,35 +7183,77 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) return 0; } -PyObject * -PyObject_GenericGetDict(PyObject *obj, void *context) +static inline PyObject * +ensure_managed_dict(PyObject *obj) { - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyTypeObject *tp = Py_TYPE(obj); - PyDictObject *dict; - if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { - dict = _PyObject_GetManagedDict(obj); - if (dict == NULL && - (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + PyDictObject *dict = _PyObject_GetManagedDict(obj); + if (dict == NULL) { + PyTypeObject *tp = Py_TYPE(obj); + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && FT_ATOMIC_LOAD_UINT8(_PyObject_InlineValues(obj)->valid)) { dict = _PyObject_MaterializeManagedDict(obj); } - else if (dict == NULL) { - Py_BEGIN_CRITICAL_SECTION(obj); - + else { +#ifdef Py_GIL_DISABLED // Check again that we're not racing with someone else creating the dict + Py_BEGIN_CRITICAL_SECTION(obj); dict = _PyObject_GetManagedDict(obj); - if (dict == NULL) { - OBJECT_STAT_INC(dict_materialized_on_request); - dictkeys_incref(CACHED_KEYS(tp)); - dict = (PyDictObject *)new_dict_with_shared_keys(interp, CACHED_KEYS(tp)); - FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, - (PyDictObject *)dict); + if (dict != NULL) { + goto done; } +#endif + dict = (PyDictObject *)new_dict_with_shared_keys(_PyInterpreterState_GET(), + CACHED_KEYS(tp)); + FT_ATOMIC_STORE_PTR_RELEASE(_PyObject_ManagedDictPointer(obj)->dict, + (PyDictObject *)dict); +#ifdef Py_GIL_DISABLED +done: Py_END_CRITICAL_SECTION(); +#endif } - return Py_XNewRef((PyObject *)dict); + } + return (PyObject *)dict; +} + +static inline PyObject * +ensure_nonmanaged_dict(PyObject *obj, PyObject **dictptr) +{ + PyDictKeysObject *cached; + + PyObject *dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + if (dict == NULL) { +#ifdef Py_GIL_DISABLED + Py_BEGIN_CRITICAL_SECTION(obj); + dict = *dictptr; + if (dict != NULL) { + goto done; + } +#endif + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); + dict = new_dict_with_shared_keys(interp, cached); + } + else { + dict = PyDict_New(); + } + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, dict); +#ifdef Py_GIL_DISABLED +done: + Py_END_CRITICAL_SECTION(); +#endif + } + return dict; +} + +PyObject * +PyObject_GenericGetDict(PyObject *obj, void *context) +{ + PyTypeObject *tp = Py_TYPE(obj); + if (_PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT)) { + return Py_XNewRef(ensure_managed_dict(obj)); } else { PyObject **dictptr = _PyObject_ComputedDictPointer(obj); @@ -7223,65 +7262,28 @@ PyObject_GenericGetDict(PyObject *obj, void *context) "This object has no __dict__"); return NULL; } - PyObject *dict = *dictptr; - if (dict == NULL) { - PyTypeObject *tp = Py_TYPE(obj); - if (_PyType_HasFeature(tp, Py_TPFLAGS_HEAPTYPE) && CACHED_KEYS(tp)) { - dictkeys_incref(CACHED_KEYS(tp)); - *dictptr = dict = new_dict_with_shared_keys( - interp, CACHED_KEYS(tp)); - } - else { - *dictptr = dict = PyDict_New(); - } - } - return Py_XNewRef(dict); + + return Py_XNewRef(ensure_nonmanaged_dict(obj, dictptr)); } } int -_PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, +_PyObjectDict_SetItem(PyTypeObject *tp, PyObject *obj, PyObject **dictptr, PyObject *key, PyObject *value) { PyObject *dict; int res; - PyDictKeysObject *cached; - PyInterpreterState *interp = _PyInterpreterState_GET(); assert(dictptr != NULL); - if ((tp->tp_flags & Py_TPFLAGS_HEAPTYPE) && (cached = CACHED_KEYS(tp))) { - assert(dictptr != NULL); - dict = *dictptr; - if (dict == NULL) { - assert(!_PyType_HasFeature(tp, Py_TPFLAGS_INLINE_VALUES)); - dictkeys_incref(cached); - dict = new_dict_with_shared_keys(interp, cached); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } - else { - res = PyDict_SetItem(dict, key, value); - } - } else { - dict = *dictptr; - if (dict == NULL) { - dict = PyDict_New(); - if (dict == NULL) - return -1; - *dictptr = dict; - } - if (value == NULL) { - res = PyDict_DelItem(dict, key); - } else { - res = PyDict_SetItem(dict, key, value); - } + dict = ensure_nonmanaged_dict(obj, dictptr); + if (dict == NULL) { + return -1; } + Py_BEGIN_CRITICAL_SECTION(dict); + res = _PyDict_SetItem_LockHeld((PyDictObject *)dict, key, value); ASSERT_CONSISTENT(dict); + Py_END_CRITICAL_SECTION(); return res; } diff --git a/Objects/object.c b/Objects/object.c index effbd51991eaa5a..8ad0389cbc7626c 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1731,7 +1731,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, goto done; } else { - res = _PyObjectDict_SetItem(tp, dictptr, name, value); + res = _PyObjectDict_SetItem(tp, obj, dictptr, name, value); } } else { @@ -1789,7 +1789,9 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) "not a '%.200s'", Py_TYPE(value)->tp_name); return -1; } + Py_BEGIN_CRITICAL_SECTION(obj); Py_XSETREF(*dictptr, Py_NewRef(value)); + Py_END_CRITICAL_SECTION(); return 0; } From e272195b3eff3a78e334a601a637d198b8de2319 Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 6 May 2024 16:45:04 -0700 Subject: [PATCH 176/220] gh-118362: Skip tests when threading isn't available (#118666) * Skip tests when threads aren't available * Use ThreadPoolExecutor --- Lib/test/test_free_threading/test_list.py | 4 ++-- Lib/test/test_free_threading/test_monitoring.py | 12 ++++++------ Lib/test/test_free_threading/test_type.py | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py index 79cb0a930923657..6ad806d67a80edd 100644 --- a/Lib/test/test_free_threading/test_list.py +++ b/Lib/test/test_free_threading/test_list.py @@ -3,7 +3,7 @@ from threading import Thread from unittest import TestCase -from test.support import is_wasi +from test.support import threading_helper class C: @@ -11,7 +11,7 @@ def __init__(self, v): self.v = v -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class TestList(TestCase): def test_racing_iter_append(self): diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index fa588046bf9923d..3a3f1ba3b605d67 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -7,7 +7,7 @@ import weakref from sys import monitoring -from test.support import is_wasi +from test.support import threading_helper from threading import Thread, _PyRLock from unittest import TestCase @@ -87,7 +87,7 @@ def tearDown(self): monitoring.free_tool_id(self.tool_id) -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class SetPreTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Sets tracing one time after the threads have started""" @@ -106,7 +106,7 @@ def after_threads(self): sys.settrace(self.trace_func) -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class MonitoringMultiThreaded( MonitoringTestMixin, InstrumentationMultiThreadedMixin, TestCase ): @@ -140,7 +140,7 @@ def during_threads(self): self.set = not self.set -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Uses sys.settrace and repeatedly toggles instrumentation on and off""" @@ -166,7 +166,7 @@ def during_threads(self): self.set = not self.set -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): """Uses sys.setprofile and repeatedly toggles instrumentation on and off""" @@ -192,7 +192,7 @@ def during_threads(self): self.set = not self.set -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class MonitoringMisc(MonitoringTestMixin, TestCase): def register_callback(self): def callback(*args): diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 76208c42fc8aeae..6eead198deed462 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -1,10 +1,11 @@ import unittest +from concurrent.futures import ThreadPoolExecutor from threading import Thread -from multiprocessing.dummy import Pool from unittest import TestCase -from test.support import is_wasi +from test.support import threading_helper, import_helper + NTHREADS = 6 @@ -15,7 +16,7 @@ class A: attr = 1 -@unittest.skipIf(is_wasi, "WASI has no threads.") +@threading_helper.requires_working_threading() class TestType(TestCase): def test_attr_cache(self): def read(id0): @@ -34,11 +35,10 @@ def write(id0): A.attr = x - with Pool(NTHREADS) as pool: - pool.apply_async(read, (1,)) - pool.apply_async(write, (1,)) - pool.close() - pool.join() + with ThreadPoolExecutor(NTHREADS) as pool: + pool.submit(read, (1,)) + pool.submit(write, (1,)) + pool.shutdown(wait=True) def test_attr_cache_consistency(self): class C: From 8d8275b0cf43f0e20c72a9641cbddc5044cdae04 Mon Sep 17 00:00:00 2001 From: "Jeong, YunWon" <69878+youknowone@users.noreply.github.com> Date: Tue, 7 May 2024 09:02:52 +0900 Subject: [PATCH 177/220] gh-118473: Fix set_asyncgen_hooks not to be partially set when arguments are invalid (#118474) --- Lib/test/test_sys.py | 16 ++++++++++- ...-05-01-22-43-54.gh-issue-118473.QIvq9R.rst | 1 + Python/sysmodule.c | 28 +++++++++++++------ 3 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-01-22-43-54.gh-issue-118473.QIvq9R.rst diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 944e84e88c8a35d..ee3bd0092f9bf3b 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1788,6 +1788,21 @@ def test_asyncgen_hooks(self): self.assertIsNone(old.finalizer) firstiter = lambda *a: None + finalizer = lambda *a: None + + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter=firstiter, finalizer="invalid") + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + + # gh-118473 + with self.assertRaises(TypeError): + sys.set_asyncgen_hooks(firstiter="invalid", finalizer=finalizer) + cur = sys.get_asyncgen_hooks() + self.assertIsNone(cur.firstiter) + self.assertIsNone(cur.finalizer) + sys.set_asyncgen_hooks(firstiter=firstiter) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) @@ -1795,7 +1810,6 @@ def test_asyncgen_hooks(self): self.assertIs(hooks.finalizer, None) self.assertIs(hooks[1], None) - finalizer = lambda *a: None sys.set_asyncgen_hooks(finalizer=finalizer) hooks = sys.get_asyncgen_hooks() self.assertIs(hooks.firstiter, firstiter) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-01-22-43-54.gh-issue-118473.QIvq9R.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-22-43-54.gh-issue-118473.QIvq9R.rst new file mode 100644 index 000000000000000..9d65e3c403ff882 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-01-22-43-54.gh-issue-118473.QIvq9R.rst @@ -0,0 +1 @@ +Fix :func:`sys.set_asyncgen_hooks` not to be partially set when raising :exc:`TypeError`. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 417040c891f8031..2851247628c32a5 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1399,12 +1399,6 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(finalizer)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { - return NULL; - } - } - else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { - return NULL; } if (firstiter && firstiter != Py_None) { @@ -1414,15 +1408,33 @@ sys_set_asyncgen_hooks(PyObject *self, PyObject *args, PyObject *kw) Py_TYPE(firstiter)->tp_name); return NULL; } - if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + } + + PyObject *cur_finalizer = _PyEval_GetAsyncGenFinalizer(); + + if (finalizer && finalizer != Py_None) { + if (_PyEval_SetAsyncGenFinalizer(finalizer) < 0) { return NULL; } } - else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + else if (finalizer == Py_None && _PyEval_SetAsyncGenFinalizer(NULL) < 0) { return NULL; } + if (firstiter && firstiter != Py_None) { + if (_PyEval_SetAsyncGenFirstiter(firstiter) < 0) { + goto error; + } + } + else if (firstiter == Py_None && _PyEval_SetAsyncGenFirstiter(NULL) < 0) { + goto error; + } + Py_RETURN_NONE; + +error: + _PyEval_SetAsyncGenFinalizer(cur_finalizer); + return NULL; } PyDoc_STRVAR(set_asyncgen_hooks_doc, From 723d4d2fe8e77b398f0ccffcfa541149caaac6a1 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 6 May 2024 20:12:39 -0400 Subject: [PATCH 178/220] gh-118527: Intern code consts in free-threaded build (#118667) We already intern and immortalize most string constants. In the free-threaded build, other constants can be a source of reference count contention because they are shared by all threads running the same code objects. --- Include/internal/pycore_code.h | 10 + Include/internal/pycore_interp.h | 1 + Include/internal/pycore_setobject.h | 3 + Lib/test/support/__init__.py | 9 + Lib/test/test_code.py | 23 +- Lib/test/test_ctypes/test_internals.py | 2 +- Lib/test/test_ctypes/test_python_api.py | 2 +- Lib/test/test_memoryio.py | 2 +- Modules/_testinternalcapi.c | 12 + Objects/codeobject.c | 302 ++++++++++++++++++++++-- Objects/setobject.c | 6 + Python/bltinmodule.c | 13 + Python/pylifecycle.c | 7 + Python/pystate.c | 1 + 14 files changed, 375 insertions(+), 18 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index 8d832c59e36874c..bcbaf60f226c77c 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -8,6 +8,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_lock.h" // PyMutex + // We hide some of the newer PyCodeObject fields behind macros. // This helps with backporting certain changes to 3.12. @@ -16,6 +18,14 @@ extern "C" { #define _PyCode_HAS_INSTRUMENTATION(CODE) \ (CODE->_co_instrumentation_version > 0) +struct _py_code_state { + PyMutex mutex; + // Interned constants from code objects. Used by the free-threaded build. + struct _Py_hashtable_t *constants; +}; + +extern PyStatus _PyCode_Init(PyInterpreterState *interp); +extern void _PyCode_Fini(PyInterpreterState *interp); #define CODE_MAX_WATCHERS 8 diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index a2aec6dd4b72411..86dada5061e7b56 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -245,6 +245,7 @@ struct _is { struct _Py_long_state long_state; struct _dtoa_state dtoa; struct _py_func_state func_state; + struct _py_code_state code_state; struct _Py_dict_state dict_state; struct _Py_exc_state exc_state; diff --git a/Include/internal/pycore_setobject.h b/Include/internal/pycore_setobject.h index 41b351ead25dd18..0494c07fe1869d3 100644 --- a/Include/internal/pycore_setobject.h +++ b/Include/internal/pycore_setobject.h @@ -30,6 +30,9 @@ PyAPI_DATA(PyObject *) _PySet_Dummy; PyAPI_FUNC(int) _PySet_Contains(PySetObject *so, PyObject *key); +// Clears the set without acquiring locks. Used by _PyCode_Fini. +extern void _PySet_ClearInternal(PySetObject *so); + #ifdef __cplusplus } #endif diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 999fffb03ed59a1..e2a854663ddb7d2 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -535,6 +535,15 @@ def suppress_immortalization(suppress=True): finally: _testinternalcapi.set_immortalize_deferred(*old_values) +def skip_if_suppress_immortalization(): + try: + import _testinternalcapi + except ImportError: + return + return unittest.skipUnless(_testinternalcapi.get_immortalize_deferred(), + "requires immortalization of deferred objects") + + MS_WINDOWS = (sys.platform == 'win32') # Is not actually used in tests, but is kept for compatibility. diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index aa793f562253930..059fab1e2ecc08d 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -142,7 +142,8 @@ from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, gc_collect, Py_GIL_DISABLED, - suppress_immortalization) + suppress_immortalization, + skip_if_suppress_immortalization) from test.support.script_helper import assert_python_ok from test.support import threading_helper, import_helper from test.support.bytecode_helper import instructions_with_positions @@ -570,11 +571,31 @@ def f(a='str_value'): self.assertIsInterned(f()) @cpython_only + @unittest.skipIf(Py_GIL_DISABLED, "free-threaded build interns all string constants") def test_interned_string_with_null(self): co = compile(r'res = "str\0value!"', '?', 'exec') v = self.find_const(co.co_consts, 'str\0value!') self.assertIsNotInterned(v) + @cpython_only + @unittest.skipUnless(Py_GIL_DISABLED, "does not intern all constants") + @skip_if_suppress_immortalization() + def test_interned_constants(self): + # compile separately to avoid compile time de-duping + + globals = {} + exec(textwrap.dedent(""" + def func1(): + return (0.0, (1, 2, "hello")) + """), globals) + + exec(textwrap.dedent(""" + def func2(): + return (0.0, (1, 2, "hello")) + """), globals) + + self.assertTrue(globals["func1"]() is globals["func2"]()) + class CodeWeakRefTest(unittest.TestCase): diff --git a/Lib/test/test_ctypes/test_internals.py b/Lib/test/test_ctypes/test_internals.py index 94c9a86c2d06dfa..778da6573da9751 100644 --- a/Lib/test/test_ctypes/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -28,7 +28,7 @@ def test_ints(self): self.assertEqual(ci._objects, None) def test_c_char_p(self): - s = b"Hello, World" + s = "Hello, World".encode("ascii") refcnt = sys.getrefcount(s) cs = c_char_p(s) self.assertEqual(refcnt + 1, sys.getrefcount(s)) diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 77da35855928a44..1072a1098332611 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -47,7 +47,7 @@ def test_PyLong_Long(self): @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" + s = object() ref = sys.getrefcount(s) # id(python-object) is the address pyobj = _ctypes.PyObj_FromPtr(id(s)) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 8192502a40791b9..95629ed862d6eb8 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -801,7 +801,7 @@ def test_sizeof(self): def _test_cow_mutation(self, mutation): # Common code for all BytesIO copy-on-write mutation tests. - imm = b' ' * 1024 + imm = (' ' * 1024).encode("ascii") old_rc = sys.getrefcount(imm) memio = self.ioclass(imm) self.assertEqual(sys.getrefcount(imm), old_rc + 1) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index de98af32b5dff73..e209c7e05264f23 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1985,6 +1985,17 @@ set_immortalize_deferred(PyObject *self, PyObject *value) #endif } +static PyObject * +get_immortalize_deferred(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = PyInterpreterState_Get(); + return PyBool_FromLong(interp->gc.immortalize.enable_on_thread_created); +#else + Py_RETURN_FALSE; +#endif +} + static PyObject * has_inline_values(PyObject *self, PyObject *obj) { @@ -2081,6 +2092,7 @@ static PyMethodDef module_functions[] = { {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif {"set_immortalize_deferred", set_immortalize_deferred, METH_VARARGS}, + {"get_immortalize_deferred", get_immortalize_deferred, METH_NOARGS}, #ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, #endif diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 2ce5f7dca2eb5cf..3d804f73a540880 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -5,6 +5,8 @@ #include "pycore_code.h" // _PyCodeConstructor #include "pycore_frame.h" // FRAME_SPECIALS_SIZE +#include "pycore_hashtable.h" // _Py_hashtable_t +#include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.co_extra_freefuncs #include "pycore_object.h" // _PyObject_SetDeferredRefcount #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches @@ -100,10 +102,20 @@ PyCode_ClearWatcher(int watcher_id) * generic helpers ******************/ -/* all_name_chars(s): true iff s matches [a-zA-Z0-9_]* */ static int -all_name_chars(PyObject *o) +should_intern_string(PyObject *o) { +#ifdef Py_GIL_DISABLED + // The free-threaded build interns (and immortalizes) all string constants + // unless we've disabled immortalizing objects that use deferred reference + // counting. + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->gc.immortalize.enable_on_thread_created) { + return 1; + } +#endif + + // compute if s matches [a-zA-Z0-9_] const unsigned char *s, *e; if (!PyUnicode_IS_ASCII(o)) @@ -118,6 +130,10 @@ all_name_chars(PyObject *o) return 1; } +#ifdef Py_GIL_DISABLED +static PyObject *intern_one_constant(PyObject *op); +#endif + static int intern_strings(PyObject *tuple) { @@ -135,14 +151,16 @@ intern_strings(PyObject *tuple) return 0; } -/* Intern selected string constants */ +/* Intern constants. In the default build, this interns selected string + constants. In the free-threaded build, this also interns non-string + constants. */ static int -intern_string_constants(PyObject *tuple, int *modified) +intern_constants(PyObject *tuple, int *modified) { for (Py_ssize_t i = PyTuple_GET_SIZE(tuple); --i >= 0; ) { PyObject *v = PyTuple_GET_ITEM(tuple, i); if (PyUnicode_CheckExact(v)) { - if (all_name_chars(v)) { + if (should_intern_string(v)) { PyObject *w = v; PyUnicode_InternInPlace(&v); if (w != v) { @@ -154,7 +172,7 @@ intern_string_constants(PyObject *tuple, int *modified) } } else if (PyTuple_CheckExact(v)) { - if (intern_string_constants(v, NULL) < 0) { + if (intern_constants(v, NULL) < 0) { return -1; } } @@ -165,7 +183,7 @@ intern_string_constants(PyObject *tuple, int *modified) return -1; } int tmp_modified = 0; - if (intern_string_constants(tmp, &tmp_modified) < 0) { + if (intern_constants(tmp, &tmp_modified) < 0) { Py_DECREF(tmp); return -1; } @@ -184,6 +202,59 @@ intern_string_constants(PyObject *tuple, int *modified) } Py_DECREF(tmp); } +#ifdef Py_GIL_DISABLED + else if (PySlice_Check(v)) { + PySliceObject *slice = (PySliceObject *)v; + PyObject *tmp = PyTuple_New(3); + if (tmp == NULL) { + return -1; + } + PyTuple_SET_ITEM(tmp, 0, Py_NewRef(slice->start)); + PyTuple_SET_ITEM(tmp, 1, Py_NewRef(slice->stop)); + PyTuple_SET_ITEM(tmp, 2, Py_NewRef(slice->step)); + int tmp_modified = 0; + if (intern_constants(tmp, &tmp_modified) < 0) { + Py_DECREF(tmp); + return -1; + } + if (tmp_modified) { + v = PySlice_New(PyTuple_GET_ITEM(tmp, 0), + PyTuple_GET_ITEM(tmp, 1), + PyTuple_GET_ITEM(tmp, 2)); + if (v == NULL) { + Py_DECREF(tmp); + return -1; + } + PyTuple_SET_ITEM(tuple, i, v); + Py_DECREF(slice); + if (modified) { + *modified = 1; + } + } + Py_DECREF(tmp); + } + + // Intern non-string consants in the free-threaded build, but only if + // we are also immortalizing objects that use deferred reference + // counting. + PyThreadState *tstate = PyThreadState_GET(); + if (!_Py_IsImmortal(v) && !PyCode_Check(v) && + !PyUnicode_CheckExact(v) && + tstate->interp->gc.immortalize.enable_on_thread_created) + { + PyObject *interned = intern_one_constant(v); + if (interned == NULL) { + return -1; + } + else if (interned != v) { + PyTuple_SET_ITEM(tuple, i, interned); + Py_SETREF(v, interned); + if (modified) { + *modified = 1; + } + } + } +#endif } return 0; } @@ -540,18 +611,41 @@ remove_column_info(PyObject *locations) return res; } -/* The caller is responsible for ensuring that the given data is valid. */ - -PyCodeObject * -_PyCode_New(struct _PyCodeConstructor *con) +static int +intern_code_constants(struct _PyCodeConstructor *con) { +#ifdef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct _py_code_state *state = &interp->code_state; + PyMutex_Lock(&state->mutex); +#endif if (intern_strings(con->names) < 0) { - return NULL; + goto error; } - if (intern_string_constants(con->consts, NULL) < 0) { - return NULL; + if (intern_constants(con->consts, NULL) < 0) { + goto error; } if (intern_strings(con->localsplusnames) < 0) { + goto error; + } +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return 0; + +error: +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&state->mutex); +#endif + return -1; +} + +/* The caller is responsible for ensuring that the given data is valid. */ + +PyCodeObject * +_PyCode_New(struct _PyCodeConstructor *con) +{ + if (intern_code_constants(con) < 0) { return NULL; } @@ -2397,3 +2491,183 @@ _PyCode_ConstantKey(PyObject *op) } return key; } + +#ifdef Py_GIL_DISABLED +static PyObject * +intern_one_constant(PyObject *op) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _Py_hashtable_t *consts = interp->code_state.constants; + + assert(!PyUnicode_CheckExact(op)); // strings are interned separately + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts, op); + if (entry == NULL) { + if (_Py_hashtable_set(consts, op, op) != 0) { + return NULL; + } + +#ifdef Py_REF_DEBUG + Py_ssize_t refcnt = Py_REFCNT(op); + if (refcnt != 1) { + // Adjust the reftotal to account for the fact that we only + // restore a single reference in _PyCode_Fini. + _Py_AddRefTotal(_PyThreadState_GET(), -(refcnt - 1)); + } +#endif + + _Py_SetImmortal(op); + return op; + } + + assert(_Py_IsImmortal(entry->value)); + return (PyObject *)entry->value; +} + +static int +compare_constants(const void *key1, const void *key2) { + PyObject *op1 = (PyObject *)key1; + PyObject *op2 = (PyObject *)key2; + if (op1 == op2) { + return 1; + } + if (Py_TYPE(op1) != Py_TYPE(op2)) { + return 0; + } + // We compare container contents by identity because we have already + // internalized the items. + if (PyTuple_CheckExact(op1)) { + Py_ssize_t size = PyTuple_GET_SIZE(op1); + if (size != PyTuple_GET_SIZE(op2)) { + return 0; + } + for (Py_ssize_t i = 0; i < size; i++) { + if (PyTuple_GET_ITEM(op1, i) != PyTuple_GET_ITEM(op2, i)) { + return 0; + } + } + return 1; + } + else if (PyFrozenSet_CheckExact(op1)) { + if (PySet_GET_SIZE(op1) != PySet_GET_SIZE(op2)) { + return 0; + } + Py_ssize_t pos1 = 0, pos2 = 0; + PyObject *obj1, *obj2; + Py_hash_t hash1, hash2; + while ((_PySet_NextEntry(op1, &pos1, &obj1, &hash1)) && + (_PySet_NextEntry(op2, &pos2, &obj2, &hash2))) + { + if (obj1 != obj2) { + return 0; + } + } + return 1; + } + else if (PySlice_Check(op1)) { + PySliceObject *s1 = (PySliceObject *)op1; + PySliceObject *s2 = (PySliceObject *)op2; + return (s1->start == s2->start && + s1->stop == s2->stop && + s1->step == s2->step); + } + else if (PyBytes_CheckExact(op1) || PyLong_CheckExact(op1)) { + return PyObject_RichCompareBool(op1, op2, Py_EQ); + } + else if (PyFloat_CheckExact(op1)) { + // Ensure that, for example, +0.0 and -0.0 are distinct + double f1 = PyFloat_AS_DOUBLE(op1); + double f2 = PyFloat_AS_DOUBLE(op2); + return memcmp(&f1, &f2, sizeof(double)) == 0; + } + else if (PyComplex_CheckExact(op1)) { + Py_complex c1 = ((PyComplexObject *)op1)->cval; + Py_complex c2 = ((PyComplexObject *)op2)->cval; + return memcmp(&c1, &c2, sizeof(Py_complex)) == 0; + } + _Py_FatalErrorFormat("unexpected type in compare_constants: %s", + Py_TYPE(op1)->tp_name); + return 0; +} + +static Py_uhash_t +hash_const(const void *key) +{ + PyObject *op = (PyObject *)key; + if (PySlice_Check(op)) { + PySliceObject *s = (PySliceObject *)op; + PyObject *data[3] = { s->start, s->stop, s->step }; + return _Py_HashBytes(&data, sizeof(data)); + } + else if (PyTuple_CheckExact(op)) { + Py_ssize_t size = PyTuple_GET_SIZE(op); + PyObject **data = _PyTuple_ITEMS(op); + return _Py_HashBytes(data, sizeof(PyObject *) * size); + } + Py_hash_t h = PyObject_Hash(op); + if (h == -1) { + // This should never happen: all the constants we support have + // infallible hash functions. + Py_FatalError("code: hash failed"); + } + return (Py_uhash_t)h; +} + +static int +clear_containers(_Py_hashtable_t *ht, const void *key, const void *value, + void *user_data) +{ + // First clear containers to avoid recursive deallocation later on in + // destroy_key. + PyObject *op = (PyObject *)key; + if (PyTuple_CheckExact(op)) { + for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(op); i++) { + Py_CLEAR(_PyTuple_ITEMS(op)[i]); + } + } + else if (PySlice_Check(op)) { + PySliceObject *slice = (PySliceObject *)op; + Py_SETREF(slice->start, Py_None); + Py_SETREF(slice->stop, Py_None); + Py_SETREF(slice->step, Py_None); + } + else if (PyFrozenSet_CheckExact(op)) { + _PySet_ClearInternal((PySetObject *)op); + } + return 0; +} + +static void +destroy_key(void *key) +{ + _Py_ClearImmortal(key); +} +#endif + +PyStatus +_PyCode_Init(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + state->constants = _Py_hashtable_new_full(&hash_const, &compare_constants, + &destroy_key, NULL, NULL); + if (state->constants == NULL) { + return _PyStatus_NO_MEMORY(); + } +#endif + return _PyStatus_OK(); +} + +void +_PyCode_Fini(PyInterpreterState *interp) +{ +#ifdef Py_GIL_DISABLED + // Free interned constants + struct _py_code_state *state = &interp->code_state; + if (state->constants) { + _Py_hashtable_foreach(state->constants, &clear_containers, NULL); + _Py_hashtable_destroy(state->constants); + state->constants = NULL; + } +#endif +} diff --git a/Objects/setobject.c b/Objects/setobject.c index 19975e3d4d18e25..68986bb6a6b557e 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2621,6 +2621,12 @@ PySet_Clear(PyObject *set) return 0; } +void +_PySet_ClearInternal(PySetObject *so) +{ + (void)set_clear_internal(so); +} + int PySet_Contains(PyObject *anyset, PyObject *key) { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index e7b60ebca5e2d7e..d192d5be751cfc9 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -866,8 +866,21 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (str == NULL) goto error; +#ifdef Py_GIL_DISABLED + // gh-118527: Disable immortalization of code constants for explicit + // compile() calls to get consistent frozen outputs between the default + // and free-threaded builds. + PyInterpreterState *interp = _PyInterpreterState_GET(); + int old_value = interp->gc.immortalize.enable_on_thread_created; + interp->gc.immortalize.enable_on_thread_created = 0; +#endif + result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); +#ifdef Py_GIL_DISABLED + interp->gc.immortalize.enable_on_thread_created = old_value; +#endif + Py_XDECREF(source_copy); goto finally; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index f24b0482c2bc38b..67bbbd01ca0c482 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -854,6 +854,11 @@ pycore_interp_init(PyThreadState *tstate) return status; } + status = _PyCode_Init(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + status = _PyDtoa_Init(interp); if (_PyStatus_EXCEPTION(status)) { return status; @@ -1827,6 +1832,8 @@ finalize_interp_types(PyInterpreterState *interp) _PyTypes_Fini(interp); + _PyCode_Fini(interp); + // Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses // a dict internally. _PyUnicode_ClearInterned(interp); diff --git a/Python/pystate.c b/Python/pystate.c index f442d87ba3150e2..7c75263b7e526a7 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -404,6 +404,7 @@ _Py_COMP_DIAG_POP &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ + &(runtime)->_main_interpreter.code_state.mutex, \ } static void From ff6cbb2503a8fe3fceeadd889e34fc9a8f308ecd Mon Sep 17 00:00:00 2001 From: Dino Viehland Date: Mon, 6 May 2024 17:22:26 -0700 Subject: [PATCH 179/220] gh-112075: use per-thread dict version pool (#118676) use thread state set of dict versions --- Include/cpython/pystate.h | 1 + Include/internal/pycore_dict.h | 21 +++++++++++-- Lib/test/test_free_threading/test_dict.py | 36 +++++++++++++++++++++++ Modules/_testcapi/dict.c | 14 ++++++++- Python/pystate.c | 1 + 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 0611e299403031e..2df9ecd6d520840 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -188,6 +188,7 @@ struct _ts { PyObject *previous_executor; + uint64_t dict_global_version; }; #ifdef Py_DEBUG diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index cb7d4c3219a9afa..8d8d3748edaea83 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -221,8 +221,25 @@ static inline PyDictUnicodeEntry* DK_UNICODE_ENTRIES(PyDictKeysObject *dk) { #define DICT_WATCHER_AND_MODIFICATION_MASK ((1 << (DICT_MAX_WATCHERS + DICT_WATCHED_MUTATION_BITS)) - 1) #ifdef Py_GIL_DISABLED -#define DICT_NEXT_VERSION(INTERP) \ - (_Py_atomic_add_uint64(&(INTERP)->dict_state.global_version, DICT_VERSION_INCREMENT) + DICT_VERSION_INCREMENT) + +#define THREAD_LOCAL_DICT_VERSION_COUNT 256 +#define THREAD_LOCAL_DICT_VERSION_BATCH THREAD_LOCAL_DICT_VERSION_COUNT * DICT_VERSION_INCREMENT + +static inline uint64_t +dict_next_version(PyInterpreterState *interp) +{ + PyThreadState *tstate = PyThreadState_GET(); + uint64_t cur_progress = (tstate->dict_global_version & + (THREAD_LOCAL_DICT_VERSION_BATCH - 1)); + if (cur_progress == 0) { + uint64_t next = _Py_atomic_add_uint64(&interp->dict_state.global_version, + THREAD_LOCAL_DICT_VERSION_BATCH); + tstate->dict_global_version = next; + } + return tstate->dict_global_version += DICT_VERSION_INCREMENT; +} + +#define DICT_NEXT_VERSION(INTERP) dict_next_version(INTERP) #else #define DICT_NEXT_VERSION(INTERP) \ diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 6a909dd3ee025f8..f877582e6b565c9 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -8,6 +8,8 @@ from threading import Thread from unittest import TestCase +from _testcapi import dict_version + from test.support import threading_helper @@ -137,5 +139,39 @@ def writer_func(l): for ref in thread_list: self.assertIsNone(ref()) + def test_dict_version(self): + THREAD_COUNT = 10 + DICT_COUNT = 10000 + lists = [] + writers = [] + + def writer_func(thread_list): + for i in range(DICT_COUNT): + thread_list.append(dict_version({})) + + for x in range(THREAD_COUNT): + thread_list = [] + lists.append(thread_list) + writer = Thread(target=partial(writer_func, thread_list)) + writers.append(writer) + + for writer in writers: + writer.start() + + for writer in writers: + writer.join() + + total_len = 0 + values = set() + for thread_list in lists: + for v in thread_list: + if v in values: + print('dup', v, (v/4096)%256) + values.add(v) + total_len += len(thread_list) + versions = set(dict_version for thread_list in lists for dict_version in thread_list) + self.assertEqual(len(versions), THREAD_COUNT*DICT_COUNT) + + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapi/dict.c b/Modules/_testcapi/dict.c index 4319906dc4fee0a..e80d898118daa51 100644 --- a/Modules/_testcapi/dict.c +++ b/Modules/_testcapi/dict.c @@ -1,7 +1,6 @@ #include "parts.h" #include "util.h" - static PyObject * dict_containsstring(PyObject *self, PyObject *args) { @@ -182,6 +181,18 @@ dict_popstring_null(PyObject *self, PyObject *args) RETURN_INT(PyDict_PopString(dict, key, NULL)); } +static PyObject * +dict_version(PyObject *self, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "expected dict"); + return NULL; + } +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + return PyLong_FromUnsignedLongLong(((PyDictObject *)dict)->ma_version_tag); +_Py_COMP_DIAG_POP +} static PyMethodDef test_methods[] = { {"dict_containsstring", dict_containsstring, METH_VARARGS}, @@ -193,6 +204,7 @@ static PyMethodDef test_methods[] = { {"dict_pop_null", dict_pop_null, METH_VARARGS}, {"dict_popstring", dict_popstring, METH_VARARGS}, {"dict_popstring_null", dict_popstring_null, METH_VARARGS}, + {"dict_version", dict_version, METH_O}, {NULL}, }; diff --git a/Python/pystate.c b/Python/pystate.c index 7c75263b7e526a7..2f1521edd8bf0ab 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1488,6 +1488,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_limit = NULL; tstate->what_event = -1; tstate->previous_executor = NULL; + tstate->dict_global_version = 0; tstate->delete_later = NULL; From b4bdf83cc67434235d9630c92c84a5261992b235 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Tue, 7 May 2024 01:32:48 +0100 Subject: [PATCH 180/220] GH-116380: Revert move of pathlib globbing code to `pathlib._glob` (#118678) The previous change made the `glob` module slower to import, because it imported `pathlib._glob` and hence the rest of `pathlib`. Reverts a40f557d7b7a355a55bb90c068e3e9202fd9c8f2. --- Lib/glob.py | 327 ++++++++++++++++++++++++++++++++++++++++- Lib/pathlib/_abc.py | 7 +- Lib/pathlib/_glob.py | 330 ------------------------------------------ Lib/pathlib/_local.py | 4 +- 4 files changed, 331 insertions(+), 337 deletions(-) delete mode 100644 Lib/pathlib/_glob.py diff --git a/Lib/glob.py b/Lib/glob.py index 9fe0e9f98ac5a7d..6088de00a67a99c 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -2,12 +2,14 @@ import contextlib import os +import re import fnmatch +import functools import itertools +import operator import stat import sys -from pathlib._glob import translate, magic_check, magic_check_bytes __all__ = ["glob", "iglob", "escape", "translate"] @@ -225,6 +227,9 @@ def _join(dirname, basename): return dirname or basename return os.path.join(dirname, basename) +magic_check = re.compile('([*?[])') +magic_check_bytes = re.compile(b'([*?[])') + def has_magic(s): if isinstance(s, bytes): match = magic_check_bytes.search(s) @@ -254,4 +259,324 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class _Globber: + """Class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Low-level methods + + lstat = operator.methodcaller('lstat') + add_slash = operator.methodcaller('joinpath', '') + + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + return contextlib.nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path. + """ + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir(). + """ + return entry + + # High-level methods + + def compile(self, pat): + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.parse_entry(entry) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.parse_entry(entry) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + else: + try: + self.lstat(path) + yield path + except OSError: + pass + + @classmethod + def walk(cls, root, top_down, on_error, follow_symlinks): + """Walk the directory tree from the given root, similar to os.walk(). + """ + paths = [root] + while paths: + path = paths.pop() + if isinstance(path, tuple): + yield path + continue + try: + with cls.scandir(path) as scandir_it: + dirnames = [] + filenames = [] + if not top_down: + paths.append((path, dirnames, filenames)) + for entry in scandir_it: + name = entry.name + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + if not top_down: + paths.append(cls.parse_entry(entry)) + dirnames.append(name) + else: + filenames.append(name) + except OSError: + filenames.append(name) + except OSError as error: + if on_error is not None: + on_error(error) + else: + if top_down: + yield path, dirnames, filenames + if dirnames: + prefix = cls.add_slash(path) + paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] + + +class _StringGlobber(_Globber): + lstat = staticmethod(os.lstat) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 9ef016a060f1d72..06c10e8e4612cad 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -12,11 +12,10 @@ """ import functools +from glob import _Globber, _no_recurse_symlinks from errno import ENOENT, ENOTDIR, EBADF, ELOOP, EINVAL from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from ._glob import Globber, no_recurse_symlinks - __all__ = ["UnsupportedOperation"] @@ -119,7 +118,7 @@ class PurePathBase: '_resolving', ) parser = ParserBase() - _globber = Globber + _globber = _Globber def __init__(self, path, *paths): self._raw_path = self.parser.join(path, *paths) if paths else path @@ -671,7 +670,7 @@ def _glob_selector(self, parts, case_sensitive, recurse_symlinks): # know the case sensitivity of the underlying filesystem, so we # must use scandir() for everything, including non-wildcard parts. case_pedantic = True - recursive = True if recurse_symlinks else no_recurse_symlinks + recursive = True if recurse_symlinks else _no_recurse_symlinks globber = self._globber(self.parser.sep, case_sensitive, case_pedantic, recursive) return globber.selector(parts) diff --git a/Lib/pathlib/_glob.py b/Lib/pathlib/_glob.py deleted file mode 100644 index 49a3851e80a038d..000000000000000 --- a/Lib/pathlib/_glob.py +++ /dev/null @@ -1,330 +0,0 @@ -import os -import re -import fnmatch -import functools -import operator - - -special_parts = ('', '.', '..') -magic_check = re.compile('([*?[])') -magic_check_bytes = re.compile(b'([*?[])') -no_recurse_symlinks = object() - - -def translate(pat, *, recursive=False, include_hidden=False, seps=None): - """Translate a pathname with shell wildcards to a regular expression. - - If `recursive` is true, the pattern segment '**' will match any number of - path segments. - - If `include_hidden` is true, wildcards can match path segments beginning - with a dot ('.'). - - If a sequence of separator characters is given to `seps`, they will be - used to split the pattern into segments and match path separators. If not - given, os.path.sep and os.path.altsep (where available) are used. - """ - if not seps: - if os.path.altsep: - seps = (os.path.sep, os.path.altsep) - else: - seps = os.path.sep - escaped_seps = ''.join(map(re.escape, seps)) - any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps - not_sep = f'[^{escaped_seps}]' - if include_hidden: - one_last_segment = f'{not_sep}+' - one_segment = f'{one_last_segment}{any_sep}' - any_segments = f'(?:.+{any_sep})?' - any_last_segments = '.*' - else: - one_last_segment = f'[^{escaped_seps}.]{not_sep}*' - one_segment = f'{one_last_segment}{any_sep}' - any_segments = f'(?:{one_segment})*' - any_last_segments = f'{any_segments}(?:{one_last_segment})?' - - results = [] - parts = re.split(any_sep, pat) - last_part_idx = len(parts) - 1 - for idx, part in enumerate(parts): - if part == '*': - results.append(one_segment if idx < last_part_idx else one_last_segment) - elif recursive and part == '**': - if idx < last_part_idx: - if parts[idx + 1] != '**': - results.append(any_segments) - else: - results.append(any_last_segments) - else: - if part: - if not include_hidden and part[0] in '*?': - results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) - if idx < last_part_idx: - results.append(any_sep) - res = ''.join(results) - return fr'(?s:{res})\Z' - - -@functools.lru_cache(maxsize=512) -def compile_pattern(pat, sep, case_sensitive, recursive=True): - """Compile given glob pattern to a re.Pattern object (observing case - sensitivity).""" - flags = re.NOFLAG if case_sensitive else re.IGNORECASE - regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) - return re.compile(regex, flags=flags).match - - -class Globber: - """Class providing shell-style pattern matching and globbing. - """ - - def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): - self.sep = sep - self.case_sensitive = case_sensitive - self.case_pedantic = case_pedantic - self.recursive = recursive - - # Low-level methods - - lstat = operator.methodcaller('lstat') - add_slash = operator.methodcaller('joinpath', '') - - @staticmethod - def scandir(path): - """Emulates os.scandir(), which returns an object that can be used as - a context manager. This method is called by walk() and glob(). - """ - from contextlib import nullcontext - return nullcontext(path.iterdir()) - - @staticmethod - def concat_path(path, text): - """Appends text to the given path. - """ - return path.with_segments(path._raw_path + text) - - @staticmethod - def parse_entry(entry): - """Returns the path of an entry yielded from scandir(). - """ - return entry - - # High-level methods - - def compile(self, pat): - return compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) - - def selector(self, parts): - """Returns a function that selects from a given path, walking and - filtering according to the glob-style pattern parts in *parts*. - """ - if not parts: - return self.select_exists - part = parts.pop() - if self.recursive and part == '**': - selector = self.recursive_selector - elif part in special_parts: - selector = self.special_selector - elif not self.case_pedantic and magic_check.search(part) is None: - selector = self.literal_selector - else: - selector = self.wildcard_selector - return selector(part, parts) - - def special_selector(self, part, parts): - """Returns a function that selects special children of the given path. - """ - select_next = self.selector(parts) - - def select_special(path, exists=False): - path = self.concat_path(self.add_slash(path), part) - return select_next(path, exists) - return select_special - - def literal_selector(self, part, parts): - """Returns a function that selects a literal descendant of a path. - """ - - # Optimization: consume and join any subsequent literal parts here, - # rather than leaving them for the next selector. This reduces the - # number of string concatenation operations and calls to add_slash(). - while parts and magic_check.search(parts[-1]) is None: - part += self.sep + parts.pop() - - select_next = self.selector(parts) - - def select_literal(path, exists=False): - path = self.concat_path(self.add_slash(path), part) - return select_next(path, exists=False) - return select_literal - - def wildcard_selector(self, part, parts): - """Returns a function that selects direct children of a given path, - filtering by pattern. - """ - - match = None if part == '*' else self.compile(part) - dir_only = bool(parts) - if dir_only: - select_next = self.selector(parts) - - def select_wildcard(path, exists=False): - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - if match is None or match(entry.name): - if dir_only: - try: - if not entry.is_dir(): - continue - except OSError: - continue - entry_path = self.parse_entry(entry) - if dir_only: - yield from select_next(entry_path, exists=True) - else: - yield entry_path - return select_wildcard - - def recursive_selector(self, part, parts): - """Returns a function that selects a given path and all its children, - recursively, filtering by pattern. - """ - # Optimization: consume following '**' parts, which have no effect. - while parts and parts[-1] == '**': - parts.pop() - - # Optimization: consume and join any following non-special parts here, - # rather than leaving them for the next selector. They're used to - # build a regular expression, which we use to filter the results of - # the recursive walk. As a result, non-special pattern segments - # following a '**' wildcard don't require additional filesystem access - # to expand. - follow_symlinks = self.recursive is not no_recurse_symlinks - if follow_symlinks: - while parts and parts[-1] not in special_parts: - part += self.sep + parts.pop() - - match = None if part == '**' else self.compile(part) - dir_only = bool(parts) - select_next = self.selector(parts) - - def select_recursive(path, exists=False): - path = self.add_slash(path) - match_pos = len(str(path)) - if match is None or match(str(path), match_pos): - yield from select_next(path, exists) - stack = [path] - while stack: - yield from select_recursive_step(stack, match_pos) - - def select_recursive_step(stack, match_pos): - path = stack.pop() - try: - # We must close the scandir() object before proceeding to - # avoid exhausting file descriptors when globbing deep trees. - with self.scandir(path) as scandir_it: - entries = list(scandir_it) - except OSError: - pass - else: - for entry in entries: - is_dir = False - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - is_dir = True - except OSError: - pass - - if is_dir or not dir_only: - entry_path = self.parse_entry(entry) - if match is None or match(str(entry_path), match_pos): - if dir_only: - yield from select_next(entry_path, exists=True) - else: - # Optimization: directly yield the path if this is - # last pattern part. - yield entry_path - if is_dir: - stack.append(entry_path) - - return select_recursive - - def select_exists(self, path, exists=False): - """Yields the given path, if it exists. - """ - if exists: - # Optimization: this path is already known to exist, e.g. because - # it was returned from os.scandir(), so we skip calling lstat(). - yield path - else: - try: - self.lstat(path) - yield path - except OSError: - pass - - @classmethod - def walk(cls, root, top_down, on_error, follow_symlinks): - """Walk the directory tree from the given root, similar to os.walk(). - """ - paths = [root] - while paths: - path = paths.pop() - if isinstance(path, tuple): - yield path - continue - try: - with cls.scandir(path) as scandir_it: - dirnames = [] - filenames = [] - if not top_down: - paths.append((path, dirnames, filenames)) - for entry in scandir_it: - name = entry.name - try: - if entry.is_dir(follow_symlinks=follow_symlinks): - if not top_down: - paths.append(cls.parse_entry(entry)) - dirnames.append(name) - else: - filenames.append(name) - except OSError: - filenames.append(name) - except OSError as error: - if on_error is not None: - on_error(error) - else: - if top_down: - yield path, dirnames, filenames - if dirnames: - prefix = cls.add_slash(path) - paths += [cls.concat_path(prefix, d) for d in reversed(dirnames)] - - -class StringGlobber(Globber): - lstat = staticmethod(os.lstat) - scandir = staticmethod(os.scandir) - parse_entry = operator.attrgetter('path') - concat_path = operator.add - - if os.name == 'nt': - @staticmethod - def add_slash(pathname): - tail = os.path.splitroot(pathname)[2] - if not tail or tail[-1] in '\\/': - return pathname - return f'{pathname}\\' - else: - @staticmethod - def add_slash(pathname): - if not pathname or pathname[-1] == '/': - return pathname - return f'{pathname}/' diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index e708191213a8b47..b1e678aceb9ce8d 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -5,6 +5,7 @@ import posixpath import sys import warnings +from glob import _StringGlobber from itertools import chain from _collections_abc import Sequence @@ -18,7 +19,6 @@ grp = None from ._abc import UnsupportedOperation, PurePathBase, PathBase -from ._glob import StringGlobber __all__ = [ @@ -102,7 +102,7 @@ class PurePath(PurePathBase): '_hash', ) parser = os.path - _globber = StringGlobber + _globber = _StringGlobber def __new__(cls, *args, **kwargs): """Construct a PurePath from one or several strings and or existing From e7dafdc2240a8e4e45f53782c47120eb3fe37712 Mon Sep 17 00:00:00 2001 From: James Hilton-Balfe Date: Tue, 7 May 2024 02:41:32 +0100 Subject: [PATCH 181/220] gh-110209: Add __class_getitem__ for generator and coroutine (#110212) --- Lib/test/test_genericalias.py | 3 ++- .../Library/2023-10-02-10-35-58.gh-issue-110209.b5zfIz.rst | 1 + Objects/genobject.c | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2023-10-02-10-35-58.gh-issue-110209.b5zfIz.rst diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 04cb810d9babbf3..04122fbdd0ae80b 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -49,7 +49,7 @@ ShareableList = None from os import DirEntry from re import Pattern, Match -from types import GenericAlias, MappingProxyType, AsyncGeneratorType +from types import GenericAlias, MappingProxyType, AsyncGeneratorType, CoroutineType, GeneratorType from tempfile import TemporaryDirectory, SpooledTemporaryFile from urllib.parse import SplitResult, ParseResult from unittest.case import _AssertRaisesContext @@ -120,6 +120,7 @@ class BaseTest(unittest.TestCase): KeysView, ItemsView, ValuesView, Sequence, MutableSequence, MappingProxyType, AsyncGeneratorType, + GeneratorType, CoroutineType, DirEntry, chain, LoggerAdapter, StreamHandler, diff --git a/Misc/NEWS.d/next/Library/2023-10-02-10-35-58.gh-issue-110209.b5zfIz.rst b/Misc/NEWS.d/next/Library/2023-10-02-10-35-58.gh-issue-110209.b5zfIz.rst new file mode 100644 index 000000000000000..b88e80d038d9b18 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-10-02-10-35-58.gh-issue-110209.b5zfIz.rst @@ -0,0 +1 @@ +Add :meth:`~object.__class_getitem__` to :class:`types.GeneratorType` and :class:`types.CoroutineType` for type hinting purposes. Patch by James Hilton-Balfe. diff --git a/Objects/genobject.c b/Objects/genobject.c index acdcf579a940efd..92cd8c61e7e9ca2 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -799,6 +799,7 @@ static PyMethodDef gen_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; @@ -1151,6 +1152,7 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close",(PyCFunction)gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", (PyCFunction)gen_sizeof, METH_NOARGS, sizeof__doc__}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL} /* Sentinel */ }; From 60bd1118447e00755320cf8a0f439c827d498b51 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Mon, 6 May 2024 21:48:28 -0400 Subject: [PATCH 182/220] Docs: add whatsnew for free-threading (#118679) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Doc/whatsnew/3.13.rst | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index c82d8bdb7342f41..df38a6154701c34 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -99,6 +99,12 @@ New typing features: * :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive type narrowing behavior. +Free-threading: + +* :pep:`703`: CPython 3.13 has experimental support for running with the + :term:`global interpreter lock` disabled when built with ``--disable-gil``. + See :ref:`Free-threaded CPython ` for more details. + New Features ============ @@ -1052,6 +1058,46 @@ See :pep:`744` for more details. Tier 2 IR by Mark Shannon and Guido van Rossum. Tier 2 optimizer by Ken Jin.) +.. _free-threaded-cpython: + +Free-threaded CPython +===================== + +CPython will run with the :term:`global interpreter lock` (GIL) disabled when +configured using the ``--disable-gil`` option at build time. This is an +experimental feature and therefore isn't used by default. Users need to +either compile their own interpreter, or install one of the experimental +builds that are marked as *free-threaded*. + +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multicore hardware. + +Work is still ongoing: expect some bugs and a substantial single-threaded +performance hit. + +The free-threaded build still supports optionally running with GIL enabled at +runtime using the environment variable :envvar:`PYTHON_GIL` or the command line +option :option:`-X gil`. + +* Use :func:`!sys._is_gil_enabled` to determine if the :term:`GIL` is enabled. + +* Use ``sysconfig.get_config_var("Py_GIL_DISABLED")`` to identify CPython + builds configured with ``--disable-gil``. + +C-API extensions need to be built specifically for the free-threaded build. + +* Extensions that support running with the :term:`GIL` disabled should use + the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should use + :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support running + with the GIL disabled. Importing C extensions that don't use these mechanisms + will cause the GIL to be enabled unless the GIL was explicitly disabled with + the :envvar:`PYTHON_GIL` environment variable or the :option:`-X gil=0` + option. + +* pip 24.1b1 or newer is required to install packages with C extensions in the + free-threaded build. Deprecated From 853163d3b5fa56d54a47817642c804b5d0c9f8e5 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Mon, 6 May 2024 20:07:23 -0700 Subject: [PATCH 183/220] gh-116322: Enable the GIL while loading C extension modules (#118560) Add the ability to enable/disable the GIL at runtime, and use that in the C module loading code. We can't know before running a module init function if it supports free-threading, so the GIL is temporarily enabled before doing so. If the module declares support for running without the GIL, the GIL is later disabled. Otherwise, the GIL is permanently enabled, and will never be disabled again for the life of the current interpreter. --- Include/internal/pycore_ceval.h | 47 +++++- Include/internal/pycore_gil.h | 16 +- Include/internal/pycore_import.h | 13 ++ ...-05-03-17-49-37.gh-issue-116322.Gy6M4j.rst | 1 + PC/_wmimodule.cpp | 12 +- Python/ceval_gil.c | 158 ++++++++++++++++-- Python/import.c | 101 ++++++++++- Python/pystate.c | 33 +++- Python/sysmodule.c | 3 +- 9 files changed, 352 insertions(+), 32 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-03-17-49-37.gh-issue-116322.Gy6M4j.rst diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 904fbd1a389f41e..48ad0678995904d 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -131,9 +131,54 @@ extern int _PyEval_ThreadsInitialized(void); extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); -extern void _PyEval_AcquireLock(PyThreadState *tstate); +// Acquire the GIL and return 1. In free-threaded builds, this function may +// return 0 to indicate that the GIL was disabled and therefore not acquired. +extern int _PyEval_AcquireLock(PyThreadState *tstate); + extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); +#ifdef Py_GIL_DISABLED +// Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or +// enabled, respectively. +// +// The enabled state of the GIL will not change while one or more threads are +// attached. +static inline int +_PyEval_IsGILEnabled(PyThreadState *tstate) +{ + return tstate->interp->ceval.gil->enabled != 0; +} + +// Enable or disable the GIL used by the interpreter that owns tstate, which +// must be the current thread. This may affect other interpreters, if the GIL +// is shared. All three functions will be no-ops (and return 0) if the +// interpreter's `enable_gil' config is not _PyConfig_GIL_DEFAULT. +// +// Every call to _PyEval_EnableGILTransient() must be paired with exactly one +// call to either _PyEval_EnableGILPermanent() or +// _PyEval_DisableGIL(). _PyEval_EnableGILPermanent() and _PyEval_DisableGIL() +// must only be called while the GIL is enabled from a call to +// _PyEval_EnableGILTransient(). +// +// _PyEval_EnableGILTransient() returns 1 if it enabled the GIL, or 0 if the +// GIL was already enabled, whether transiently or permanently. The caller will +// hold the GIL upon return. +// +// _PyEval_EnableGILPermanent() returns 1 if it permanently enabled the GIL +// (which must already be enabled), or 0 if it was already permanently +// enabled. Once _PyEval_EnableGILPermanent() has been called once, all +// subsequent calls to any of the three functions will be no-ops. +// +// _PyEval_DisableGIL() returns 1 if it disabled the GIL, or 0 if the GIL was +// kept enabled because of another request, whether transient or permanent. +// +// All three functions must be called by an attached thread (this implies that +// if the GIL is enabled, the current thread must hold it). +extern int _PyEval_EnableGILTransient(PyThreadState *tstate); +extern int _PyEval_EnableGILPermanent(PyThreadState *tstate); +extern int _PyEval_DisableGIL(PyThreadState *state); +#endif + extern void _PyEval_DeactivateOpCache(void); diff --git a/Include/internal/pycore_gil.h b/Include/internal/pycore_gil.h index d36b4c0db010b2e..a2de5077371ebae 100644 --- a/Include/internal/pycore_gil.h +++ b/Include/internal/pycore_gil.h @@ -21,8 +21,20 @@ extern "C" { struct _gil_runtime_state { #ifdef Py_GIL_DISABLED - /* Whether or not this GIL is being used. Can change from 0 to 1 at runtime - if, for example, a module that requires the GIL is loaded. */ + /* If this GIL is disabled, enabled == 0. + + If this GIL is enabled transiently (most likely to initialize a module + of unknown safety), enabled indicates the number of active transient + requests. + + If this GIL is enabled permanently, enabled == INT_MAX. + + It must not be modified directly; use _PyEval_EnableGILTransiently(), + _PyEval_EnableGILPermanently(), and _PyEval_DisableGIL() + + It is always read and written atomically, but a thread can assume its + value will be stable as long as that thread is attached or knows that no + other threads are attached (e.g., during a stop-the-world.). */ int enabled; #endif /* microseconds (the Python API uses seconds, though) */ diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index b02769903a6f9bb..bd40707fed21a8e 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -206,6 +206,19 @@ extern int _PyImport_CheckSubinterpIncompatibleExtensionAllowed( // Export for '_testinternalcapi' shared extension PyAPI_FUNC(int) _PyImport_ClearExtension(PyObject *name, PyObject *filename); +#ifdef Py_GIL_DISABLED +// Assuming that the GIL is enabled from a call to +// _PyEval_EnableGILTransient(), resolve the transient request depending on the +// state of the module argument: +// - If module is NULL or a PyModuleObject with md_gil == Py_MOD_GIL_NOT_USED, +// call _PyEval_DisableGIL(). +// - Otherwise, call _PyEval_EnableGILPermanent(). If the GIL was not already +// enabled permanently, issue a warning referencing the module's name. +// +// This function may raise an exception. +extern int _PyImport_CheckGILForModule(PyObject *module, PyObject *module_name); +#endif + #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-03-17-49-37.gh-issue-116322.Gy6M4j.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-03-17-49-37.gh-issue-116322.Gy6M4j.rst new file mode 100644 index 000000000000000..d3ea5e0d0f3596c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-03-17-49-37.gh-issue-116322.Gy6M4j.rst @@ -0,0 +1 @@ +In ``--disable-gil`` builds, the GIL will be enabled while loading C extension modules. If the module indicates that it supports running without the GIL, the GIL will be disabled once loading is complete. Otherwise, the GIL will remain enabled for the remainder of the interpreter's lifetime. This behavior does not apply if the GIL has been explicitly enabled or disabled with ``PYTHON_GIL`` or ``-Xgil``. diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index 22ed05276e6f071..48863b90f4cc278 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -362,12 +362,18 @@ static PyMethodDef wmi_functions[] = { { NULL, NULL, 0, NULL } }; +static PyModuleDef_Slot wmi_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + static PyModuleDef wmi_def = { PyModuleDef_HEAD_INIT, "_wmi", - NULL, // doc - 0, // m_size - wmi_functions + NULL, // doc + 0, // m_size + wmi_functions, // m_methods + wmi_slots, // m_slots }; extern "C" { diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index b456a9d80455e4f..7a54c185303cd1d 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -205,6 +205,16 @@ static void recreate_gil(struct _gil_runtime_state *gil) } #endif +static void +drop_gil_impl(struct _gil_runtime_state *gil) +{ + MUTEX_LOCK(gil->mutex); + _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); + _Py_atomic_store_int_relaxed(&gil->locked, 0); + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); +} + static void drop_gil(PyInterpreterState *interp, PyThreadState *tstate) { @@ -220,7 +230,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) struct _gil_runtime_state *gil = ceval->gil; #ifdef Py_GIL_DISABLED - if (!gil->enabled) { + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { return; } #endif @@ -236,11 +246,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); } - MUTEX_LOCK(gil->mutex); - _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); - _Py_atomic_store_int_relaxed(&gil->locked, 0); - COND_SIGNAL(gil->cond); - MUTEX_UNLOCK(gil->mutex); + drop_gil_impl(gil); #ifdef FORCE_SWITCHING /* We check tstate first in case we might be releasing the GIL for @@ -275,8 +281,10 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) The function saves errno at entry and restores its value at exit. - tstate must be non-NULL. */ -static void + tstate must be non-NULL. + + Returns 1 if the GIL was acquired, or 0 if not. */ +static int take_gil(PyThreadState *tstate) { int err = errno; @@ -300,8 +308,8 @@ take_gil(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; struct _gil_runtime_state *gil = interp->ceval.gil; #ifdef Py_GIL_DISABLED - if (!gil->enabled) { - return; + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + return 0; } #endif @@ -346,6 +354,17 @@ take_gil(PyThreadState *tstate) } } +#ifdef Py_GIL_DISABLED + if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + // Another thread disabled the GIL between our check above and + // now. Don't take the GIL, signal any other waiting threads, and + // return 0. + COND_SIGNAL(gil->cond); + MUTEX_UNLOCK(gil->mutex); + return 0; + } +#endif + #ifdef FORCE_SWITCHING /* This mutex must be taken before modifying gil->last_holder: see drop_gil(). */ @@ -387,6 +406,7 @@ take_gil(PyThreadState *tstate) MUTEX_UNLOCK(gil->mutex); errno = err; + return 1; } void _PyEval_SetSwitchInterval(unsigned long microseconds) @@ -451,7 +471,8 @@ init_own_gil(PyInterpreterState *interp, struct _gil_runtime_state *gil) { assert(!gil_created(gil)); #ifdef Py_GIL_DISABLED - gil->enabled = _PyInterpreterState_GetConfig(interp)->enable_gil == _PyConfig_GIL_ENABLE; + const PyConfig *config = _PyInterpreterState_GetConfig(interp); + gil->enabled = config->enable_gil == _PyConfig_GIL_ENABLE ? INT_MAX : 0; #endif create_gil(gil); assert(gil_created(gil)); @@ -545,11 +566,11 @@ PyEval_ReleaseLock(void) drop_gil(tstate->interp, tstate); } -void +int _PyEval_AcquireLock(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - take_gil(tstate); + return take_gil(tstate); } void @@ -1011,6 +1032,117 @@ _PyEval_InitState(PyInterpreterState *interp) _gil_initialize(&interp->_gil); } +#ifdef Py_GIL_DISABLED +int +_PyEval_EnableGILTransient(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + // The GIL is already enabled permanently. + return 0; + } + if (enabled == INT_MAX - 1) { + Py_FatalError("Too many transient requests to enable the GIL"); + } + if (enabled > 0) { + // If enabled is nonzero, we know we hold the GIL. This means that no + // other threads are attached, and nobody else can be concurrently + // mutating it. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + return 0; + } + + // Enabling the GIL changes what it means to be an "attached" thread. To + // safely make this transition, we: + // 1. Detach the current thread. + // 2. Stop the world to detach (and suspend) all other threads. + // 3. Enable the GIL, if nobody else did between our check above and when + // our stop-the-world begins. + // 4. Start the world. + // 5. Attach the current thread. Other threads may attach and hold the GIL + // before this thread, which is harmless. + _PyThreadState_Detach(tstate); + + // This could be an interpreter-local stop-the-world in situations where we + // know that this interpreter's GIL is not shared, and that it won't become + // shared before the stop-the-world begins. For now, we always stop all + // interpreters for simplicity. + _PyEval_StopTheWorldAll(&_PyRuntime); + + enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + int this_thread_enabled = enabled == 0; + _Py_atomic_store_int_relaxed(&gil->enabled, enabled + 1); + + _PyEval_StartTheWorldAll(&_PyRuntime); + _PyThreadState_Attach(tstate); + + return this_thread_enabled; +} + +int +_PyEval_EnableGILPermanent(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; + } + + _Py_atomic_store_int_relaxed(&gil->enabled, INT_MAX); + return 1; +} + +int +_PyEval_DisableGIL(PyThreadState *tstate) +{ + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil != _PyConfig_GIL_DEFAULT) { + return 0; + } + + struct _gil_runtime_state *gil = tstate->interp->ceval.gil; + assert(current_thread_holds_gil(gil, tstate)); + + int enabled = _Py_atomic_load_int_relaxed(&gil->enabled); + if (enabled == INT_MAX) { + return 0; + } + + assert(enabled >= 1); + enabled--; + + // Disabling the GIL is much simpler than enabling it, since we know we are + // the only attached thread. Other threads may start free-threading as soon + // as this store is complete, if it sets gil->enabled to 0. + _Py_atomic_store_int_relaxed(&gil->enabled, enabled); + + if (enabled == 0) { + // We're attached, so we know the GIL will remain disabled until at + // least the next time we detach, which must be after this function + // returns. + // + // Drop the GIL, which will wake up any threads waiting in take_gil() + // and let them resume execution without the GIL. + drop_gil_impl(gil); + return 1; + } + return 0; +} +#endif + /* Do periodic things, like check for signals and async I/0. * We need to do reasonably frequently, but not too frequently. diff --git a/Python/import.c b/Python/import.c index 33277073ac543db..0b58567dde1fec5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1,6 +1,7 @@ /* Module definition and import implementation */ #include "Python.h" +#include "pycore_ceval.h" #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -1023,6 +1024,12 @@ struct extensions_cache_value { struct cached_m_dict _m_dict; _Py_ext_module_origin origin; + +#ifdef Py_GIL_DISABLED + /* The module's md_gil slot, for legacy modules that are reinitialized from + m_dict rather than calling their initialization function again. */ + void *md_gil; +#endif }; static struct extensions_cache_value * @@ -1351,7 +1358,7 @@ static struct extensions_cache_value * _extensions_cache_set(PyObject *path, PyObject *name, PyModuleDef *def, PyModInitFunction m_init, Py_ssize_t m_index, PyObject *m_dict, - _Py_ext_module_origin origin) + _Py_ext_module_origin origin, void *md_gil) { struct extensions_cache_value *value = NULL; void *key = NULL; @@ -1401,7 +1408,13 @@ _extensions_cache_set(PyObject *path, PyObject *name, .m_index=m_index, /* m_dict is set by set_cached_m_dict(). */ .origin=origin, +#ifdef Py_GIL_DISABLED + .md_gil=md_gil, +#endif }; +#ifndef Py_GIL_DISABLED + (void)md_gil; +#endif if (init_cached_m_dict(newvalue, m_dict) < 0) { goto finally; } @@ -1526,6 +1539,47 @@ _PyImport_CheckSubinterpIncompatibleExtensionAllowed(const char *name) return 0; } +#ifdef Py_GIL_DISABLED +int +_PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) +{ + PyThreadState *tstate = _PyThreadState_GET(); + if (module == NULL) { + _PyEval_DisableGIL(tstate); + return 0; + } + + if (!PyModule_Check(module) || + ((PyModuleObject *)module)->md_gil == Py_MOD_GIL_USED) { + if (_PyEval_EnableGILPermanent(tstate)) { + int warn_result = PyErr_WarnFormat( + PyExc_RuntimeWarning, + 1, + "The global interpreter lock (GIL) has been enabled to load " + "module '%U', which has not declared that it can run safely " + "without the GIL. To override this behavior and keep the GIL " + "disabled (at your own risk), run with PYTHON_GIL=0 or -Xgil=0.", + module_name + ); + if (warn_result < 0) { + return warn_result; + } + } + + const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); + if (config->enable_gil == _PyConfig_GIL_DEFAULT && config->verbose) { + PySys_FormatStderr("# loading module '%U', which requires the GIL\n", + module_name); + } + } + else { + _PyEval_DisableGIL(tstate); + } + + return 0; +} +#endif + static PyObject * get_core_module_dict(PyInterpreterState *interp, PyObject *name, PyObject *path) @@ -1625,6 +1679,7 @@ struct singlephase_global_update { Py_ssize_t m_index; PyObject *m_dict; _Py_ext_module_origin origin; + void *md_gil; }; static struct extensions_cache_value * @@ -1683,7 +1738,7 @@ update_global_state_for_extension(PyThreadState *tstate, #endif cached = _extensions_cache_set( path, name, def, m_init, singlephase->m_index, m_dict, - singlephase->origin); + singlephase->origin, singlephase->md_gil); if (cached == NULL) { // XXX Ignore this error? Doing so would effectively // mark the module as not loadable. @@ -1768,6 +1823,13 @@ reload_singlephase_extension(PyThreadState *tstate, Py_DECREF(mod); return NULL; } +#ifdef Py_GIL_DISABLED + if (def->m_base.m_copy != NULL) { + // For non-core modules, fetch the GIL slot that was stored by + // import_run_extension(). + ((PyModuleObject *)mod)->md_gil = cached->md_gil; + } +#endif /* We can't set mod->md_def if it's missing, * because _PyImport_ClearModulesByIndex() might break * due to violating interpreter isolation. @@ -1921,6 +1983,9 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, // cache is less reliable than it should be). .m_index=def->m_base.m_index, .origin=info->origin, +#ifdef Py_GIL_DISABLED + .md_gil=((PyModuleObject *)mod)->md_gil, +#endif }; // gh-88216: Extensions and def->m_base.m_copy can be updated // when the extension module doesn't support sub-interpreters. @@ -2039,6 +2104,10 @@ _PyImport_FixupBuiltin(PyThreadState *tstate, PyObject *mod, const char *name, /* We don't want def->m_base.m_copy populated. */ .m_dict=NULL, .origin=_Py_ext_module_origin_CORE, +#ifdef Py_GIL_DISABLED + /* Unused when m_dict == NULL. */ + .md_gil=NULL, +#endif }; cached = update_global_state_for_extension( tstate, nameobj, nameobj, def, &singlephase); @@ -2128,9 +2197,23 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) goto finally; } +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif /* Now load it. */ mod = import_run_extension( tstate, p0, &info, spec, get_modules_dict(tstate, true)); +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; + } +#endif finally: _Py_ext_module_loader_info_clear(&info); @@ -4505,10 +4588,22 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) goto finally; } +#ifdef Py_GIL_DISABLED + // This call (and the corresponding call to _PyImport_CheckGILForModule()) + // would ideally be inside import_run_extension(). They are kept in the + // callers for now because that would complicate the control flow inside + // import_run_extension(). It should be possible to restructure + // import_run_extension() to address this. + _PyEval_EnableGILTransient(tstate); +#endif mod = import_run_extension( tstate, p0, &info, spec, get_modules_dict(tstate, true)); - if (mod == NULL) { +#ifdef Py_GIL_DISABLED + if (_PyImport_CheckGILForModule(mod, info.name) < 0) { + Py_CLEAR(mod); + goto finally; } +#endif // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? if (fp) { diff --git a/Python/pystate.c b/Python/pystate.c index 2f1521edd8bf0ab..b1e085bb8069155 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2057,19 +2057,36 @@ _PyThreadState_Attach(PyThreadState *tstate) Py_FatalError("non-NULL old thread state"); } - _PyEval_AcquireLock(tstate); - // XXX assert(tstate_is_alive(tstate)); - current_fast_set(&_PyRuntime, tstate); - tstate_activate(tstate); + while (1) { + int acquired_gil = _PyEval_AcquireLock(tstate); - if (!tstate_try_attach(tstate)) { - tstate_wait_attach(tstate); - } + // XXX assert(tstate_is_alive(tstate)); + current_fast_set(&_PyRuntime, tstate); + tstate_activate(tstate); + + if (!tstate_try_attach(tstate)) { + tstate_wait_attach(tstate); + } #ifdef Py_GIL_DISABLED - _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); + if (_PyEval_IsGILEnabled(tstate) != acquired_gil) { + // The GIL was enabled between our call to _PyEval_AcquireLock() + // and when we attached (the GIL can't go from enabled to disabled + // here because only a thread holding the GIL can disable + // it). Detach and try again. + assert(!acquired_gil); + tstate_set_detached(tstate, _Py_THREAD_DETACHED); + tstate_deactivate(tstate); + current_fast_clear(&_PyRuntime); + continue; + } + _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); +#else + (void)acquired_gil; #endif + break; + } // Resume previous critical section. This acquires the lock(s) from the // top-most critical section. diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 2851247628c32a5..601d8ae32bb34ba 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2433,8 +2433,7 @@ sys__is_gil_enabled_impl(PyObject *module) /*[clinic end generated code: output=57732cf53f5b9120 input=7e9c47f15a00e809]*/ { #ifdef Py_GIL_DISABLED - PyInterpreterState *interp = _PyInterpreterState_GET(); - return interp->ceval.gil->enabled; + return _PyEval_IsGILEnabled(_PyThreadState_GET()); #else return 1; #endif From 1a23716d4ba6c6dadd76a3662db580f451f660e3 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 May 2024 21:16:22 -0600 Subject: [PATCH 184/220] gh-117953: Imply Single-phase Init if the Init Function Fails (gh-118684) This ensures the kind is always either _Py_ext_module_kind_SINGLEPHASE or _Py_ext_module_kind_MULTIPHASE. --- Python/importdl.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Python/importdl.c b/Python/importdl.c index 38f56db4b4cc96a..7c42d37283c4954 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -426,6 +426,11 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, /* Validate the result (and populate "res". */ if (m == NULL) { + /* The init func for multi-phase init modules is expected + * to return a PyModuleDef after calling PyModuleDef_Init(). + * That function never raises an exception nor returns NULL, + * so at this point it must be a single-phase init modules. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; if (PyErr_Occurred()) { _Py_ext_module_loader_result_set_error( &res, _Py_ext_module_loader_result_EXCEPTION); @@ -436,6 +441,8 @@ _PyImport_RunModInitFunc(PyModInitFunction p0, } goto error; } else if (PyErr_Occurred()) { + /* Likewise, we infer that this is a single-phase init module. */ + res.kind = _Py_ext_module_kind_SINGLEPHASE; _Py_ext_module_loader_result_set_error( &res, _Py_ext_module_loader_result_ERR_UNREPORTED_EXC); /* We would probably be correct to decref m here, From b2cd54a4fb2ecdb7b1d30bda8af3314d3a32031e Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 6 May 2024 22:21:51 -0600 Subject: [PATCH 185/220] gh-117953: Always Run Extension Init Func in Main Interpreter First (gh-118157) This change makes sure all extension/builtin modules have their init function run first by the main interpreter before proceeding with import in the original interpreter (main or otherwise). This means when the import of a single-phase init module fails in an isolated subinterpreter, it won't tie any global state/callbacks to the subinterpreter. --- Lib/test/test_import/__init__.py | 18 +- Lib/test/test_interpreters/__init__.py | 7 +- ...-05-06-10-57-54.gh-issue-117953.DqCzIs.rst | 8 + Python/import.c | 252 ++++++++++++++---- 4 files changed, 216 insertions(+), 69 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-06-10-57-54.gh-issue-117953.DqCzIs.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index b2aae774a122056..64282d0f2d0bcf6 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2454,10 +2454,6 @@ def setUpClass(cls): # Start fresh. cls.clean_up() - @classmethod - def tearDownClass(cls): - restore__testsinglephase() - def tearDown(self): # Clean up the module. self.clean_up() @@ -3014,20 +3010,20 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # * alive in 0 interpreters # * module def in _PyRuntime.imports.extensions # * mod init func ran for the first time (since reset) - # * m_copy is NULL (claered when the interpreter was destroyed) + # * m_copy is still set (owned by main interpreter) # * module's global state was initialized, not reset # Use a subinterpreter that sticks around. loaded_interp1 = self.import_in_subinterp(interpid1) self.check_common(loaded_interp1) - self.check_semi_fresh(loaded_interp1, loaded_main, base) + self.check_copied(loaded_interp1, base) # At this point: # * alive in 1 interpreter (interp1) # * module def still in _PyRuntime.imports.extensions - # * mod init func ran for the second time (since reset) - # * m_copy was copied from interp1 (was NULL) - # * module's global state was updated, not reset + # * mod init func did not run again + # * m_copy was not changed + # * module's global state was not touched # Use a subinterpreter while the previous one is still alive. loaded_interp2 = self.import_in_subinterp(interpid2) @@ -3038,8 +3034,8 @@ def test_basic_multiple_interpreters_deleted_no_reset(self): # * alive in 2 interpreters (interp1, interp2) # * module def still in _PyRuntime.imports.extensions # * mod init func did not run again - # * m_copy was copied from interp2 (was from interp1) - # * module's global state was updated, not reset + # * m_copy was not changed + # * module's global state was not touched @requires_subinterpreters def test_basic_multiple_interpreters_reset_each(self): diff --git a/Lib/test/test_interpreters/__init__.py b/Lib/test/test_interpreters/__init__.py index 4b16ecc31156a51..52ff553f60d0d75 100644 --- a/Lib/test/test_interpreters/__init__.py +++ b/Lib/test/test_interpreters/__init__.py @@ -1,5 +1,6 @@ import os -from test.support import load_package_tests +from test.support import load_package_tests, Py_GIL_DISABLED -def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) +if not Py_GIL_DISABLED: + def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-06-10-57-54.gh-issue-117953.DqCzIs.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-06-10-57-54.gh-issue-117953.DqCzIs.rst new file mode 100644 index 000000000000000..6b69c9311013bb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-06-10-57-54.gh-issue-117953.DqCzIs.rst @@ -0,0 +1,8 @@ +When a builtin or extension module is imported for the first time, while a +subinterpreter is active, the module's init function is now run by the main +interpreter first before import continues in the subinterpreter. +Consequently, single-phase init modules now fail in an isolated +subinterpreter without the init function running under that interpreter, +whereas before it would run under the subinterpreter *before* failing, +potentially leaving behind global state and callbacks and otherwise leaving +the module in an inconsistent state. diff --git a/Python/import.c b/Python/import.c index 0b58567dde1fec5..ba44477318d4731 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1154,9 +1154,6 @@ init_cached_m_dict(struct extensions_cache_value *value, PyObject *m_dict) return -1; } // XXX We may want to make the copy immortal. - // XXX This incref shouldn't be necessary. We are decref'ing - // one to many times somewhere. - Py_INCREF(copied); value->_m_dict = (struct cached_m_dict){ .copied=copied, @@ -1580,6 +1577,26 @@ _PyImport_CheckGILForModule(PyObject* module, PyObject *module_name) } #endif +static PyThreadState * +switch_to_main_interpreter(PyThreadState *tstate) +{ + if (_Py_IsMainInterpreter(tstate->interp)) { + return tstate; + } + PyThreadState *main_tstate = PyThreadState_New(_PyInterpreterState_Main()); + if (main_tstate == NULL) { + return NULL; + } + main_tstate->_whence = _PyThreadState_WHENCE_EXEC; +#ifndef NDEBUG + PyThreadState *old_tstate = PyThreadState_Swap(main_tstate); + assert(old_tstate == tstate); +#else + (void)PyThreadState_Swap(main_tstate); +#endif + return main_tstate; +} + static PyObject * get_core_module_dict(PyInterpreterState *interp, PyObject *name, PyObject *path) @@ -1936,24 +1953,171 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, struct extensions_cache_value *cached = NULL; const char *name_buf = PyBytes_AS_STRING(info->name_encoded); + /* We cannot know if the module is single-phase init or + * multi-phase init until after we call its init function. Even + * in isolated interpreters (that do not support single-phase init), + * the init function will run without restriction. For multi-phase + * init modules that isn't a problem because the init function only + * runs PyModuleDef_Init() on the module's def and then returns it. + * + * However, for single-phase init the module's init function will + * create the module, create other objects (and allocate other + * memory), populate it and its module state, and initialze static + * types. Some modules store other objects and data in global C + * variables and register callbacks with the runtime/stdlib or + * even external libraries (which is part of why we can't just + * dlclose() the module in the error case). That's a problem + * for isolated interpreters since all of the above happens + * and only then * will the import fail. Memory will leak, + * callbacks will still get used, and sometimes there + * will be crashes (memory access violations + * and use-after-free). + * + * To put it another way, if the module is single-phase init + * then the import will probably break interpreter isolation + * and should fail ASAP. However, the module's init function + * will still get run. That means it may still store state + * in the shared-object/DLL address space (which never gets + * closed/cleared), including objects (e.g. static types). + * This is a problem for isolated subinterpreters since each + * has its own object allocator. If the loaded shared-object + * still holds a reference to an object after the corresponding + * interpreter has finalized then either we must let it leak + * or else any later use of that object by another interpreter + * (or across multiple init-fini cycles) will crash the process. + * + * To avoid all of that, we make sure the module's init function + * is always run first with the main interpreter active. If it was + * already the main interpreter then we can continue loading the + * module like normal. Otherwise, right after the init function, + * we take care of some import state bookkeeping, switch back + * to the subinterpreter, check for single-phase init, + * and then continue loading like normal. */ + + bool switched = false; + /* We *could* leave in place a legacy interpreter here + * (one that shares obmalloc/GIL with main interp), + * but there isn't a big advantage, we anticipate + * such interpreters will be increasingly uncommon, + * and the code is a bit simpler if we always switch + * to the main interpreter. */ + PyThreadState *main_tstate = switch_to_main_interpreter(tstate); + if (main_tstate == NULL) { + return NULL; + } + else if (main_tstate != tstate) { + switched = true; + /* In the switched case, we could play it safe + * by getting the main interpreter's import lock here. + * It's unlikely to matter though. */ + } + struct _Py_ext_module_loader_result res; - if (_PyImport_RunModInitFunc(p0, info, &res) < 0) { + int rc = _PyImport_RunModInitFunc(p0, info, &res); + if (rc < 0) { /* We discard res.def. */ assert(res.module == NULL); - _Py_ext_module_loader_result_apply_error(&res, name_buf); - return NULL; } - assert(!PyErr_Occurred()); - assert(res.err == NULL); + else { + assert(!PyErr_Occurred()); + assert(res.err == NULL); + + mod = res.module; + res.module = NULL; + def = res.def; + assert(def != NULL); + + /* Do anything else that should be done + * while still using the main interpreter. */ + if (res.kind == _Py_ext_module_kind_SINGLEPHASE) { + /* Remember the filename as the __file__ attribute */ + if (info->filename != NULL) { + // XXX There's a refleak somewhere with the filename. + // Until we can track it down, we intern it. + PyObject *filename = Py_NewRef(info->filename); + PyUnicode_InternInPlace(&filename); + if (PyModule_AddObjectRef(mod, "__file__", filename) < 0) { + PyErr_Clear(); /* Not important enough to report */ + } + } + + /* Update global import state. */ + assert(def->m_base.m_index != 0); + struct singlephase_global_update singlephase = { + // XXX Modules that share a def should each get their own index, + // whereas currently they share (which means the per-interpreter + // cache is less reliable than it should be). + .m_index=def->m_base.m_index, + .origin=info->origin, +#ifdef Py_GIL_DISABLED + .md_gil=((PyModuleObject *)mod)->md_gil, +#endif + }; + // gh-88216: Extensions and def->m_base.m_copy can be updated + // when the extension module doesn't support sub-interpreters. + if (def->m_size == -1) { + /* We will reload from m_copy. */ + assert(def->m_base.m_init == NULL); + singlephase.m_dict = PyModule_GetDict(mod); + assert(singlephase.m_dict != NULL); + } + else { + /* We will reload via the init function. */ + assert(def->m_size >= 0); + assert(def->m_base.m_copy == NULL); + singlephase.m_init = p0; + } + cached = update_global_state_for_extension( + tstate, info->path, info->name, def, &singlephase); + if (cached == NULL) { + assert(PyErr_Occurred()); + goto main_finally; + } + } + } - mod = res.module; - res.module = NULL; - def = res.def; - assert(def != NULL); +main_finally: + /* Switch back to the subinterpreter. */ + if (switched) { + assert(main_tstate != tstate); + + /* Handle any exceptions, which we cannot propagate directly + * to the subinterpreter. */ + if (PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_MemoryError)) { + /* We trust it will be caught again soon. */ + PyErr_Clear(); + } + else { + /* Printing the exception should be sufficient. */ + PyErr_PrintEx(0); + } + } + + /* Any module we got from the init function will have to be + * reloaded in the subinterpreter. */ + Py_CLEAR(mod); + + PyThreadState_Clear(main_tstate); + (void)PyThreadState_Swap(tstate); + PyThreadState_Delete(main_tstate); + } + + /*****************************************************************/ + /* At this point we are back to the interpreter we started with. */ + /*****************************************************************/ + + /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + goto error; + } if (res.kind == _Py_ext_module_kind_MULTIPHASE) { assert_multiphase_def(def); assert(mod == NULL); + /* Note that we cheat a little by not repeating the calls + * to _PyImport_GetModInitFunc() and _PyImport_RunModInitFunc(). */ mod = PyModule_FromDefAndSpec(def, spec); if (mod == NULL) { goto error; @@ -1962,57 +2126,35 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, else { assert(res.kind == _Py_ext_module_kind_SINGLEPHASE); assert_singlephase_def(def); - assert(PyModule_Check(mod)); if (_PyImport_CheckSubinterpIncompatibleExtensionAllowed(name_buf) < 0) { goto error; } + assert(!PyErr_Occurred()); - /* Remember the filename as the __file__ attribute */ - if (info->filename != NULL) { - if (PyModule_AddObjectRef(mod, "__file__", info->filename) < 0) { - PyErr_Clear(); /* Not important enough to report */ + if (switched) { + /* We switched to the main interpreter to run the init + * function, so now we will "reload" the module from the + * cached data using the original subinterpreter. */ + assert(mod == NULL); + mod = reload_singlephase_extension(tstate, cached, info); + if (mod == NULL) { + goto error; } - } - - /* Update global import state. */ - assert(def->m_base.m_index != 0); - struct singlephase_global_update singlephase = { - // XXX Modules that share a def should each get their own index, - // whereas currently they share (which means the per-interpreter - // cache is less reliable than it should be). - .m_index=def->m_base.m_index, - .origin=info->origin, -#ifdef Py_GIL_DISABLED - .md_gil=((PyModuleObject *)mod)->md_gil, -#endif - }; - // gh-88216: Extensions and def->m_base.m_copy can be updated - // when the extension module doesn't support sub-interpreters. - if (def->m_size == -1) { - /* We will reload from m_copy. */ - assert(def->m_base.m_init == NULL); - singlephase.m_dict = PyModule_GetDict(mod); - assert(singlephase.m_dict != NULL); + assert(!PyErr_Occurred()); + assert(PyModule_Check(mod)); } else { - /* We will reload via the init function. */ - assert(def->m_size >= 0); - assert(def->m_base.m_copy == NULL); - singlephase.m_init = p0; - } - cached = update_global_state_for_extension( - tstate, info->path, info->name, def, &singlephase); - if (cached == NULL) { - goto error; - } - - /* Update per-interpreter import state. */ - PyObject *modules = get_modules_dict(tstate, true); - if (finish_singlephase_extension( - tstate, mod, cached, info->name, modules) < 0) - { - goto error; + assert(mod != NULL); + assert(PyModule_Check(mod)); + + /* Update per-interpreter import state. */ + PyObject *modules = get_modules_dict(tstate, true); + if (finish_singlephase_extension( + tstate, mod, cached, info->name, modules) < 0) + { + goto error; + } } } From 0d9148823dbb6af020945ab1b487d7f183b561a5 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 6 May 2024 21:22:59 -0700 Subject: [PATCH 186/220] gh-118414: Fix assertion in YIELD_VALUE when tracing lines or instrs (#118683) --- Lib/test/test_monitoring.py | 11 +++++++++++ .../2024-05-07-01-39-24.gh-issue-118414.G5GG7l.rst | 1 + Python/bytecodes.c | 4 +++- Python/executor_cases.c.h | 4 +++- Python/generated_cases.c.h | 4 +++- 5 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-07-01-39-24.gh-issue-118414.G5GG7l.rst diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index eeb3f88a081750d..6974bc5517ae5fb 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -656,6 +656,17 @@ def func2(): self.check_lines(func2, [1,2,3,4,5,6]) + def test_generator_with_line(self): + + def f(): + def a(): + yield + def b(): + yield from a() + next(b()) + + self.check_lines(f, [1,3,5,4,2,4]) + class TestDisable(MonitoringTestBase, unittest.TestCase): def gen(self, cond): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-07-01-39-24.gh-issue-118414.G5GG7l.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-01-39-24.gh-issue-118414.G5GG7l.rst new file mode 100644 index 000000000000000..cd545049f926933 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-07-01-39-24.gh-issue-118414.G5GG7l.rst @@ -0,0 +1 @@ +Add instrumented opcodes to YIELD_VALUE assertion for tracing cases. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index b2ddec98e682f27..55eda9711dea1fe 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -1121,7 +1121,9 @@ dummy_func( /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 5f15f67324292b6..347a1e677a08325 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1137,7 +1137,9 @@ /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d3126b0c9807238..8b8112209cc78a9 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -6223,7 +6223,9 @@ /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); #if TIER_ONE - assert(_PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || + assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || + frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || + _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); From 698417f2f677b7b9373f8a2f202b6c18870bf3c2 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 7 May 2024 00:49:07 -0400 Subject: [PATCH 187/220] Update macOS installer screens and README for 3.13.0b1 (GH-118685) --- Mac/BuildScript/README.rst | 46 ++++----------------------- Mac/BuildScript/resources/ReadMe.rtf | 6 ++-- Mac/BuildScript/resources/Welcome.rtf | 4 +-- 3 files changed, 11 insertions(+), 45 deletions(-) diff --git a/Mac/BuildScript/README.rst b/Mac/BuildScript/README.rst index a9fae36ba28ae99..e44e74f3a492341 100644 --- a/Mac/BuildScript/README.rst +++ b/Mac/BuildScript/README.rst @@ -17,7 +17,7 @@ Gatekeeper download quarantine, the final package must be signed with a valid Apple Developer ID certificate using productsign. Starting with macOS 10.15 Catalina, Gatekeeper now also requires that installer packages are submitted to and pass Apple's automated -notarization service using the altool command. To pass notarization, +notarization service using the ``notarytool`` command. To pass notarization, the binaries included in the package must be built with at least the macOS 10.9 SDK, must now be signed with the codesign utility, and executables must opt in to the hardened run time option with @@ -27,7 +27,7 @@ available in the on-line Apple Developer Documentation and man pages. A goal of PSF-provided (python.org) Python binaries for macOS is to support a wide-range of operating system releases with one set of binaries. Currently, the oldest release supported by python.org -binaries is macOS 10.9; it is still possible to build Python and +binaries is macOS 10.9; it should still be possible to build Python and Python installers on older versions of macOS but we not regularly test on those systems nor provide binaries for them. @@ -49,20 +49,17 @@ Starting with 3.9.1, Python fully supports macOS "weaklinking", meaning it is now possible to build a Python on a current macOS version with a deployment target of an earlier macOS system. For 3.9.1 and later systems, we provide a "macOS 64-bit universal2 installer" -variant, currently build on macOS 11 Big Sur with fat binaries +variant, currently built on macOS 11 Big Sur with fat binaries natively supporting both Apple Silicon (arm64) and Intel-64 (x86_64) Macs running macOS 10.9 or later. -The legacy "macOS 64-bit Intel installer" variant is expected to -be retired prior to the end of 3.9.x support. - build-installer.py requires Apple Developer tools, either from the Command Line Tools package or from a full Xcode installation. You should use the most recent version of either for the operating system version in use. (One notable exception: on macOS 10.6, Snow Leopard, use Xcode 3, not Xcode 4 which was released later in the 10.6 support cycle.) build-installer.py also must be run -with recent versions of Python 3.x or 2.7. On older systems, +with recent versions of Python 3.x. On older systems, due to changes in TLS practices, it may be easier to manually download and cache third-party source distributions used by build-installer.py rather than have it attempt to automatically @@ -76,12 +73,11 @@ download them. - builds the following third-party libraries - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 + * OpenSSL 3.0.x + * Tcl/Tk 8.6.x * NCurses * SQLite * XZ - * libffi * mpdecimal - uses system-supplied versions of third-party libraries @@ -98,36 +94,6 @@ download them. * ``MACOSX_DEPLOYMENT_TARGET=10.9`` * Apple ``clang`` -2. legacy Intel 64-bit, x86_64, for OS X 10.9 (and later):: - - /path/to/bootstrap/python3 build-installer.py \ - --universal-archs=intel-64 \ - --dep-target=10.9 - - - builds the following third-party libraries - - * OpenSSL 1.1.1 - * Tcl/Tk 8.6 - * NCurses - * SQLite - * XZ - * libffi - * mpdecimal - - - uses system-supplied versions of third-party libraries - - * readline module links with Apple BSD editline (libedit) - * zlib - * bz2 - - - recommended build environment: - - * Mac OS X 10.9.5 - * Xcode Command Line Tools 6.2 - * ``MacOSX10.9`` SDK - * ``MACOSX_DEPLOYMENT_TARGET=10.9`` - * Apple ``clang`` - General Prerequisites --------------------- diff --git a/Mac/BuildScript/resources/ReadMe.rtf b/Mac/BuildScript/resources/ReadMe.rtf index efd76b9b1ae64b4..ee5ba4707dfea4b 100644 --- a/Mac/BuildScript/resources/ReadMe.rtf +++ b/Mac/BuildScript/resources/ReadMe.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fswiss\fcharset0 Helvetica-Oblique; \f3\fmodern\fcharset0 CourierNewPSMT;\f4\fmodern\fcharset0 Courier;} {\colortbl;\red255\green255\blue255;} @@ -11,7 +11,7 @@ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\partightenfactor0 \f1\b \cf0 NOTE: -\f0\b0 This is an alpha preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 \cf0 \ \pard\tx720\tx1440\tx2160\tx2880\tx3600\tx4320\tx5040\tx5760\tx6480\tx7200\tx7920\tx8640\pardirnatural\partightenfactor0 @@ -69,7 +69,7 @@ Due to new security checks on macOS 10.15 Catalina, when launching IDLE macOS ma \f1\b \ul Apple Silicon Mac support\ \f0\b0 \ulnone \ -On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the +On Apple Silicon Macs, it is possible to run Python either with native ARM64 code or under Intel 64 emulation using Rosetta2. This option might be useful for testing or if binary wheels are not yet available with native ARM64 binaries. To easily force Python to run in emulation mode, invoke it from a command line shell with the \f4 python3-intel64 \f0 command instead of just \f4 python3 diff --git a/Mac/BuildScript/resources/Welcome.rtf b/Mac/BuildScript/resources/Welcome.rtf index 79851e1f4a69cc8..49d6e22286be267 100644 --- a/Mac/BuildScript/resources/Welcome.rtf +++ b/Mac/BuildScript/resources/Welcome.rtf @@ -1,4 +1,4 @@ -{\rtf1\ansi\ansicpg1252\cocoartf2709 +{\rtf1\ansi\ansicpg1252\cocoartf2761 \cocoascreenfonts1\cocoatextscaling0\cocoaplatform0{\fonttbl\f0\fswiss\fcharset0 Helvetica;\f1\fswiss\fcharset0 Helvetica-Bold;\f2\fmodern\fcharset0 CourierNewPSMT; } {\colortbl;\red255\green255\blue255;} @@ -26,5 +26,5 @@ At the end of this install, click on \ \f1\b NOTE: -\f0\b0 This is an alpha test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ +\f0\b0 This is a beta test preview of Python 3.13.0, the next feature release of Python 3. It is not intended for production use.\ } \ No newline at end of file From 49258efada0cb0fc58ccffc018ff310b8f7f4570 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Tue, 7 May 2024 02:23:27 -0500 Subject: [PATCH 188/220] gh-118107: Fix zipimporter ZIP64 handling. (GH-118108) Add missing import to code that handles too large files and offsets. Use list, not tuple, for a mutable sequence. Add tests to prevent similar mistakes. --------- Co-authored-by: Gregory P. Smith [Google LLC] Co-authored-by: Kirill Podoprigora --- Lib/test/test_zipimport.py | 121 ++++++++++++++++++ .../sparse-zip64-c0-0x000000000.part | Bin 0 -> 4096 bytes .../sparse-zip64-c0-0x100000000.part | Bin 0 -> 4096 bytes .../sparse-zip64-c0-0x200000000.part | Bin 0 -> 546 bytes Lib/zipimport.py | 5 +- Makefile.pre.in | 3 +- ...-04-19-09-28-43.gh-issue-118107.Mdsr1J.rst | 2 + 7 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part create mode 100644 Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part create mode 100644 Lib/test/zipimport_data/sparse-zip64-c0-0x200000000.part create mode 100644 Misc/NEWS.d/next/Library/2024-04-19-09-28-43.gh-issue-118107.Mdsr1J.rst diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index ae49700294330ce..e9c3218d2bb39ee 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -1,8 +1,10 @@ import sys import os import marshal +import glob import importlib import importlib.util +import re import struct import time import unittest @@ -54,6 +56,7 @@ def module_path_to_dotted_name(path): TESTPACK2 = "ziptestpackage2" TEMP_DIR = os.path.abspath("junk95142") TEMP_ZIP = os.path.abspath("junk95142.zip") +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "zipimport_data") pyc_file = importlib.util.cache_from_source(TESTMOD + '.py') pyc_ext = '.pyc' @@ -134,7 +137,9 @@ def getZip64Files(self): def doTest(self, expected_ext, files, *modules, **kw): self.makeZip(files, **kw) + self.doTestWithPreBuiltZip(expected_ext, *modules, **kw) + def doTestWithPreBuiltZip(self, expected_ext, *modules, **kw): sys.path.insert(0, TEMP_ZIP) mod = importlib.import_module(".".join(modules)) @@ -810,6 +815,122 @@ def testZip64CruftAndComment(self): files = self.getZip64Files() self.doTest(".py", files, "f65536", comment=b"c" * ((1 << 16) - 1)) + def testZip64LargeFile(self): + support.requires( + "largefile", + f"test generates files >{0xFFFFFFFF} bytes and takes a long time " + "to run" + ) + + # N.B.: We do alot of gymnastics below in the ZIP_STORED case to save + # and reconstruct a sparse zip on systems that support sparse files. + # Instead of creating a ~8GB zip file mainly consisting of null bytes + # for every run of the test, we create the zip once and save off the + # non-null portions of the resulting file as data blobs with offsets + # that allow re-creating the zip file sparsely. This drops disk space + # usage to ~9KB for the ZIP_STORED case and drops that test time by ~2 + # orders of magnitude. For the ZIP_DEFLATED case, however, we bite the + # bullet. The resulting zip file is ~8MB of non-null data; so the sparse + # trick doesn't work and would result in that full ~8MB zip data file + # being checked in to source control. + parts_glob = f"sparse-zip64-c{self.compression:d}-0x*.part" + full_parts_glob = os.path.join(TEST_DATA_DIR, parts_glob) + pre_built_zip_parts = glob.glob(full_parts_glob) + + self.addCleanup(os_helper.unlink, TEMP_ZIP) + if not pre_built_zip_parts: + if self.compression != ZIP_STORED: + support.requires( + "cpu", + "test requires a lot of CPU for compression." + ) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + f.write(b"data") + f.write(os.linesep.encode()) + f.seek(0xffff_ffff, os.SEEK_CUR) + f.write(os.linesep.encode()) + os.utime(os_helper.TESTFN, (0.0, 0.0)) + with ZipFile( + TEMP_ZIP, + "w", + compression=self.compression, + strict_timestamps=False + ) as z: + z.write(os_helper.TESTFN, "data1") + z.writestr( + ZipInfo("module.py", (1980, 1, 1, 0, 0, 0)), test_src + ) + z.write(os_helper.TESTFN, "data2") + + # This "works" but relies on the zip format having a non-empty + # final page due to the trailing central directory to wind up with + # the correct length file. + def make_sparse_zip_parts(name): + empty_page = b"\0" * 4096 + with open(name, "rb") as f: + part = None + try: + while True: + offset = f.tell() + data = f.read(len(empty_page)) + if not data: + break + if data != empty_page: + if not part: + part_fullname = os.path.join( + TEST_DATA_DIR, + f"sparse-zip64-c{self.compression:d}-" + f"{offset:#011x}.part", + ) + os.makedirs( + os.path.dirname(part_fullname), + exist_ok=True + ) + part = open(part_fullname, "wb") + print("Created", part_fullname) + part.write(data) + else: + if part: + part.close() + part = None + finally: + if part: + part.close() + + if self.compression == ZIP_STORED: + print(f"Creating sparse parts to check in into {TEST_DATA_DIR}:") + make_sparse_zip_parts(TEMP_ZIP) + + else: + def extract_offset(name): + if m := re.search(r"-(0x[0-9a-f]{9})\.part$", name): + return int(m.group(1), base=16) + raise ValueError(f"{name=} does not fit expected pattern.") + offset_parts = [(extract_offset(n), n) for n in pre_built_zip_parts] + with open(TEMP_ZIP, "wb") as f: + for offset, part_fn in sorted(offset_parts): + with open(part_fn, "rb") as part: + f.seek(offset, os.SEEK_SET) + f.write(part.read()) + # Confirm that the reconstructed zip file works and looks right. + with ZipFile(TEMP_ZIP, "r") as z: + self.assertEqual( + z.getinfo("module.py").date_time, (1980, 1, 1, 0, 0, 0) + ) + self.assertEqual( + z.read("module.py"), test_src.encode(), + msg=f"Recreate {full_parts_glob}, unexpected contents." + ) + def assertDataEntry(name): + zinfo = z.getinfo(name) + self.assertEqual(zinfo.date_time, (1980, 1, 1, 0, 0, 0)) + self.assertGreater(zinfo.file_size, 0xffff_ffff) + assertDataEntry("data1") + assertDataEntry("data2") + + self.doTestWithPreBuiltZip(".py", "module") + @support.requires_zlib() class CompressedZipImportTestCase(UncompressedZipImportTestCase): diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x000000000.part new file mode 100644 index 0000000000000000000000000000000000000000..c6beae8e2552d6a7be5c70df6feeb42230270912 GIT binary patch literal 4096 zcmWIWW@gc4fB;2?DAjhu|4_inAi|K6SdwVS$RNPL3RJ-e#4s8p!3EVeN{xoVXb6mk kz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD4Dt{F0RIvV#Q*>R literal 0 HcmV?d00001 diff --git a/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part b/Lib/test/zipimport_data/sparse-zip64-c0-0x100000000.part new file mode 100644 index 0000000000000000000000000000000000000000..74ab03b4648948b5a040716f2d0d1dcd6d5f2d8e GIT binary patch literal 4096 zcmZQzARBN6cr&wzK(#9}80R11^8oTdm=lO|^HWN5QuPWdQ&Q6u(o;*~^AdAYH8icb z6o8;8wWPEtPaz&G7$46CQ)po=GP{7I{0uO^FjUZ-9Vo9PQBZB|~E0Dto#4sA& z7~}jSd>%k~5awjy0g@2Vzz8xsH$SB`CsnVYl97Rf!3HP;G78-YRF}v=4Panc(g-pF z2dYVz0p>vl24<)jjJ^rwKL^rGKpf!B#tsrX0CW?G4{;hR Q8%T^92tn#W4q#va017uX9RL6T literal 0 HcmV?d00001 diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 21d2dca46f569b1..4e41d109865e854 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -517,8 +517,9 @@ def _read_directory(archive): num_extra_values = (len(extra_data) - 4) // 8 if num_extra_values > 3: raise ZipImportError(f"can't read header extra: {archive!r}", path=archive) - values = struct.unpack_from(f"<{min(num_extra_values, 3)}Q", - extra_data, offset=4) + import struct + values = list(struct.unpack_from(f"<{min(num_extra_values, 3)}Q", + extra_data, offset=4)) # N.b. Here be dragons: the ordering of these is different than # the header fields, and it's really easy to get it wrong since diff --git a/Makefile.pre.in b/Makefile.pre.in index 88ec2009fbfffe4..74a438b015a97d9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2483,7 +2483,8 @@ TESTSUBDIRS= idlelib/idle_test \ test/typinganndata \ test/wheeldata \ test/xmltestdata \ - test/xmltestdata/c14n-20 + test/xmltestdata/c14n-20 \ + test/zipimport_data COMPILEALL_OPTS=-j0 diff --git a/Misc/NEWS.d/next/Library/2024-04-19-09-28-43.gh-issue-118107.Mdsr1J.rst b/Misc/NEWS.d/next/Library/2024-04-19-09-28-43.gh-issue-118107.Mdsr1J.rst new file mode 100644 index 000000000000000..0e358d6605e66e8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-19-09-28-43.gh-issue-118107.Mdsr1J.rst @@ -0,0 +1,2 @@ +Fix :mod:`zipimport` reading of ZIP64 files with file entries that are too big or +offset too far. From de1428f8c234a8731ced99cbfe5cd6c5c719e31d Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Tue, 7 May 2024 10:28:51 +0300 Subject: [PATCH 189/220] gh-62090: Simplify argparse usage formatting (GH-105039) Rationale ========= argparse performs a complex formatting of the usage for argument grouping and for line wrapping to fit the terminal width. This formatting has been a constant source of bugs for at least 10 years (see linked issues below) where defensive assertion errors are triggered or brackets and paranthesis are not properly handeled. Problem ======= The current implementation of argparse usage formatting relies on regular expressions to group arguments usage only to separate them again later with another set of regular expressions. This is a complex and error prone approach that caused all the issues linked below. Special casing certain argument formats has not solved the problem. The following are some of the most common issues: - empty `metavar` - mutually exclusive groups with `SUPPRESS`ed arguments - metavars with whitespace - metavars with brackets or paranthesis Solution ======== The following two comments summarize the solution: - https://github.com/python/cpython/issues/82091#issuecomment-1093832187 - https://github.com/python/cpython/issues/77048#issuecomment-1093776995 Mainly, the solution is to rewrite the usage formatting to avoid the group-then-separate approach. Instead, the usage parts are kept separate and only joined together at the end. This allows for a much simpler implementation that is easier to understand and maintain. It avoids the regular expressions approach and fixes the corresponding issues. This closes the following GitHub issues: - #62090 - #62549 - #77048 - #82091 - #89743 - #96310 - #98666 These PRs become obsolete: - #15372 - #96311 --- Lib/argparse.py | 100 ++++--------- Lib/test/test_argparse.py | 134 ++++++++++++++++++ ...3-05-28-11-25-18.gh-issue-62090.opAhDn.rst | 2 + 3 files changed, 164 insertions(+), 72 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-05-28-11-25-18.gh-issue-62090.opAhDn.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 0dbdd67a82f3919..55bf8cdd875a8d8 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -328,17 +328,8 @@ def _format_usage(self, usage, actions, groups, prefix): if len(prefix) + len(usage) > text_width: # break usage into wrappable parts - part_regexp = ( - r'\(.*?\)+(?=\s|$)|' - r'\[.*?\]+(?=\s|$)|' - r'\S+' - ) - opt_usage = format(optionals, groups) - pos_usage = format(positionals, groups) - opt_parts = _re.findall(part_regexp, opt_usage) - pos_parts = _re.findall(part_regexp, pos_usage) - assert ' '.join(opt_parts) == opt_usage - assert ' '.join(pos_parts) == pos_usage + opt_parts = self._get_actions_usage_parts(optionals, groups) + pos_parts = self._get_actions_usage_parts(positionals, groups) # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -391,6 +382,9 @@ def get_lines(parts, indent, prefix=None): return '%s%s\n\n' % (prefix, usage) def _format_actions_usage(self, actions, groups): + return ' '.join(self._get_actions_usage_parts(actions, groups)) + + def _get_actions_usage_parts(self, actions, groups): # find group indices and identify actions in groups group_actions = set() inserts = {} @@ -398,58 +392,26 @@ def _format_actions_usage(self, actions, groups): if not group._group_actions: raise ValueError(f'empty group {group}') + if all(action.help is SUPPRESS for action in group._group_actions): + continue + try: start = actions.index(group._group_actions[0]) except ValueError: continue else: - group_action_count = len(group._group_actions) - end = start + group_action_count + end = start + len(group._group_actions) if actions[start:end] == group._group_actions: - - suppressed_actions_count = 0 - for action in group._group_actions: - group_actions.add(action) - if action.help is SUPPRESS: - suppressed_actions_count += 1 - - exposed_actions_count = group_action_count - suppressed_actions_count - if not exposed_actions_count: - continue - - if not group.required: - if start in inserts: - inserts[start] += ' [' - else: - inserts[start] = '[' - if end in inserts: - inserts[end] += ']' - else: - inserts[end] = ']' - elif exposed_actions_count > 1: - if start in inserts: - inserts[start] += ' (' - else: - inserts[start] = '(' - if end in inserts: - inserts[end] += ')' - else: - inserts[end] = ')' - for i in range(start + 1, end): - inserts[i] = '|' + group_actions.update(group._group_actions) + inserts[start, end] = group # collect all actions format strings parts = [] - for i, action in enumerate(actions): + for action in actions: # suppressed arguments are marked with None - # remove | separators for suppressed arguments if action.help is SUPPRESS: - parts.append(None) - if inserts.get(i) == '|': - inserts.pop(i) - elif inserts.get(i + 1) == '|': - inserts.pop(i + 1) + part = None # produce all arg strings elif not action.option_strings: @@ -461,9 +423,6 @@ def _format_actions_usage(self, actions, groups): if part[0] == '[' and part[-1] == ']': part = part[1:-1] - # add the action string to the list - parts.append(part) - # produce the first way to invoke the option in brackets else: option_string = action.option_strings[0] @@ -484,26 +443,23 @@ def _format_actions_usage(self, actions, groups): if not action.required and action not in group_actions: part = '[%s]' % part - # add the action string to the list - parts.append(part) + # add the action string to the list + parts.append(part) - # insert things at the necessary indices - for i in sorted(inserts, reverse=True): - parts[i:i] = [inserts[i]] - - # join all the action items with spaces - text = ' '.join([item for item in parts if item is not None]) - - # clean up separators for mutually exclusive groups - open = r'[\[(]' - close = r'[\])]' - text = _re.sub(r'(%s) ' % open, r'\1', text) - text = _re.sub(r' (%s)' % close, r'\1', text) - text = _re.sub(r'%s *%s' % (open, close), r'', text) - text = text.strip() + # group mutually exclusive actions + for start, end in sorted(inserts, reverse=True): + group = inserts[start, end] + group_parts = [item for item in parts[start:end] if item is not None] + if group.required: + open, close = "()" if len(group_parts) > 1 else ("", "") + else: + open, close = "[]" + parts[start] = open + " | ".join(group_parts) + close + for i in range(start + 1, end): + parts[i] = None - # return the text - return text + # return the usage parts + return [item for item in parts if item is not None] def _format_text(self, text): if '%(prog)' in text: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 617b1721f3dbb1c..02b499145f6c431 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4255,6 +4255,140 @@ class TestHelpUsagePositionalsOnlyWrap(HelpTestCase): version = '' +class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): + # https://github.com/python/cpython/issues/62549 + # https://github.com/python/cpython/issues/89743 + parser_signature = Sig(prog='PROG') + argument_signatures = [ + Sig('-n1', metavar='()', help='n1'), + Sig('-o1', metavar='(1, 2)', help='o1'), + Sig('-u1', metavar=' (uu) ', help='u1'), + Sig('-v1', metavar='( vv )', help='v1'), + Sig('-w1', metavar='(w)w', help='w1'), + Sig('-x1', metavar='x(x)', help='x1'), + Sig('-y1', metavar='yy)', help='y1'), + Sig('-z1', metavar='(zz', help='z1'), + Sig('-n2', metavar='[]', help='n2'), + Sig('-o2', metavar='[1, 2]', help='o2'), + Sig('-u2', metavar=' [uu] ', help='u2'), + Sig('-v2', metavar='[ vv ]', help='v2'), + Sig('-w2', metavar='[w]w', help='w2'), + Sig('-x2', metavar='x[x]', help='x2'), + Sig('-y2', metavar='yy]', help='y2'), + Sig('-z2', metavar='[zz', help='z2'), + ] + + usage = '''\ + usage: PROG [-h] [-n1 ()] [-o1 (1, 2)] [-u1 (uu) ] [-v1 ( vv )] [-w1 (w)w] + [-x1 x(x)] [-y1 yy)] [-z1 (zz] [-n2 []] [-o2 [1, 2]] [-u2 [uu] ] + [-v2 [ vv ]] [-w2 [w]w] [-x2 x[x]] [-y2 yy]] [-z2 [zz] + ''' + help = usage + '''\ + + options: + -h, --help show this help message and exit + -n1 () n1 + -o1 (1, 2) o1 + -u1 (uu) u1 + -v1 ( vv ) v1 + -w1 (w)w w1 + -x1 x(x) x1 + -y1 yy) y1 + -z1 (zz z1 + -n2 [] n2 + -o2 [1, 2] o2 + -u2 [uu] u2 + -v2 [ vv ] v2 + -w2 [w]w w2 + -x2 x[x] x2 + -y2 yy] y2 + -z2 [zz z2 + ''' + version = '' + + +class TestHelpUsageNoWhitespaceCrash(TestCase): + + def test_all_suppressed_mutex_followed_by_long_arg(self): + # https://github.com/python/cpython/issues/62090 + # https://github.com/python/cpython/issues/96310 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument('--spam', help=argparse.SUPPRESS) + parser.add_argument('--eggs-eggs-eggs-eggs-eggs-eggs') + usage = textwrap.dedent('''\ + usage: PROG [-h] + [--eggs-eggs-eggs-eggs-eggs-eggs EGGS_EGGS_EGGS_EGGS_EGGS_EGGS] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_newline_in_metavar(self): + # https://github.com/python/cpython/issues/77048 + mapping = ['123456', '12345', '12345', '123'] + parser = argparse.ArgumentParser('11111111111111') + parser.add_argument('-v', '--verbose', + help='verbose mode', action='store_true') + parser.add_argument('targets', + help='installation targets', + nargs='+', + metavar='\n'.join(mapping)) + usage = textwrap.dedent('''\ + usage: 11111111111111 [-h] [-v] + 123456 + 12345 + 12345 + 123 [123456 + 12345 + 12345 + 123 ...] + ''') + self.assertEqual(parser.format_usage(), usage) + + def test_empty_metavar_required_arg(self): + # https://github.com/python/cpython/issues/82091 + parser = argparse.ArgumentParser(prog='PROG') + parser.add_argument('--nil', metavar='', required=True) + parser.add_argument('--a', metavar='A' * 70) + usage = ( + 'usage: PROG [-h] --nil \n' + ' [--a AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' + 'AAAAAAAAAAAAAAAAAAAAAAA]\n' + ) + self.assertEqual(parser.format_usage(), usage) + + def test_all_suppressed_mutex_with_optional_nargs(self): + # https://github.com/python/cpython/issues/98666 + parser = argparse.ArgumentParser(prog='PROG') + mutex = parser.add_mutually_exclusive_group() + mutex.add_argument( + '--param1', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + mutex.add_argument( + '--param2', + nargs='?', const='default', metavar='NAME', help=argparse.SUPPRESS) + usage = 'usage: PROG [-h]\n' + self.assertEqual(parser.format_usage(), usage) + + def test_nested_mutex_groups(self): + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument("--spam") + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + gg = g.add_mutually_exclusive_group() + gg.add_argument("--hax") + gg.add_argument("--hox", help=argparse.SUPPRESS) + gg.add_argument("--hex") + g.add_argument("--eggs") + parser.add_argument("--num") + + usage = textwrap.dedent('''\ + usage: PROG [-h] [--spam SPAM | [--hax HAX | --hex HEX] | --eggs EGGS] + [--num NUM] + ''') + self.assertEqual(parser.format_usage(), usage) + + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" diff --git a/Misc/NEWS.d/next/Library/2023-05-28-11-25-18.gh-issue-62090.opAhDn.rst b/Misc/NEWS.d/next/Library/2023-05-28-11-25-18.gh-issue-62090.opAhDn.rst new file mode 100644 index 000000000000000..c5abf7178563e8f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-05-28-11-25-18.gh-issue-62090.opAhDn.rst @@ -0,0 +1,2 @@ +Fix assertion errors caused by whitespace in metavars or ``SUPPRESS``-ed groups +in :mod:`argparse` by simplifying usage formatting. Patch by Ali Hamdan. From b7778d59020ed0e97ca459934585d45c3a4a6469 Mon Sep 17 00:00:00 2001 From: wim glenn Date: Tue, 7 May 2024 03:52:48 -0500 Subject: [PATCH 190/220] Fix typo inaccuracy in _bootstrap_external.py (GH-118619) --- Lib/importlib/_bootstrap_external.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 30a8cd4475a104e..db446776901fc38 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -1464,7 +1464,7 @@ class PathFinder: @staticmethod def invalidate_caches(): """Call the invalidate_caches() method on all path entry finders - stored in sys.path_importer_caches (where implemented).""" + stored in sys.path_importer_cache (where implemented).""" for name, finder in list(sys.path_importer_cache.items()): # Drop entry if finder name is a relative path. The current # working directory may have changed. From 3a6ff1b1d959549546f533b18ec71aa21516328d Mon Sep 17 00:00:00 2001 From: mpage Date: Tue, 7 May 2024 02:04:01 -0700 Subject: [PATCH 191/220] gh-117657: Use clang 18 in TSAN builds (GH-118533) Use clang 18 in TSAN builds We were using clang 14 which is a couple of years old. --- .github/workflows/reusable-tsan.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 8ddb3b3ada32c23..48bd5b547e8cba8 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -29,7 +29,14 @@ jobs: - name: Install Dependencies run: | sudo ./.github/workflows/posix-deps-apt.sh - sudo apt install -y clang + # Install clang-18 + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 18 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 100 + sudo update-alternatives --set clang /usr/bin/clang-18 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-18 # Reduce ASLR to avoid TSAN crashing sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup From fe47d9bee319528ffeb5fd60a615d7f02c7b5585 Mon Sep 17 00:00:00 2001 From: Youfu Zhang <1315097+zhangyoufu@users.noreply.github.com> Date: Tue, 7 May 2024 17:18:45 +0800 Subject: [PATCH 192/220] gh-118314: Fix padding edge case in binascii.a2b_base64 strict mode (GH-118320) Fix an edge case in `binascii.a2b_base64` strict mode, where excessive padding was not detected when no padding is necessary. Co-authored-by: Terry Jan Reedy Co-authored-by: Pieter Eendebak --- Lib/test/test_binascii.py | 15 +++++++++++++++ ...2024-04-26-12-42-29.gh-issue-118314.Z7reGc.rst | 1 + Modules/binascii.c | 7 +++++++ 3 files changed, 23 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-04-26-12-42-29.gh-issue-118314.Z7reGc.rst diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 82dea8a6d731ea3..1f3b6746ce4a626 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -139,13 +139,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -157,8 +165,15 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') def test_base64errors(self): diff --git a/Misc/NEWS.d/next/Library/2024-04-26-12-42-29.gh-issue-118314.Z7reGc.rst b/Misc/NEWS.d/next/Library/2024-04-26-12-42-29.gh-issue-118314.Z7reGc.rst new file mode 100644 index 000000000000000..ff3ee688ca1bfae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-04-26-12-42-29.gh-issue-118314.Z7reGc.rst @@ -0,0 +1 @@ +Fix an edge case in :func:`binascii.a2b_base64` strict mode, where excessive padding is not detected when no padding is necessary. diff --git a/Modules/binascii.c b/Modules/binascii.c index 250f03a9531eedc..6bb01d148b6faac 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -414,6 +414,13 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) if (this_ch == BASE64_PAD) { padding_started = 1; + if (strict_mode && quad_pos == 0) { + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, "Excess padding not allowed"); + } + goto error_end; + } if (quad_pos >= 2 && quad_pos + ++pads >= 4) { /* A pad sequence means we should not parse more input. ** We've already interpreted the data from the quad at this point. From 976212223541b89329d81ab01e3edd14d8f6aaf5 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Tue, 7 May 2024 17:23:06 +0800 Subject: [PATCH 193/220] Fix several typos in documentation (GH-118625) --- Doc/library/datetime.rst | 4 ++-- Doc/library/dis.rst | 2 +- Doc/library/msvcrt.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index f40c6cea83820c4..b20b2027cfcc93c 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -635,7 +635,7 @@ Notes: .. versionchanged:: 3.13 Comparison between :class:`.datetime` object and an instance of the :class:`date` subclass that is not a :class:`!datetime` subclass - no longer coverts the latter to :class:`!date`, ignoring the time part + no longer converts the latter to :class:`!date`, ignoring the time part and the time zone. The default behavior can be changed by overriding the special comparison methods in subclasses. @@ -1257,7 +1257,7 @@ Supported operations: .. versionchanged:: 3.13 Comparison between :class:`.datetime` object and an instance of the :class:`date` subclass that is not a :class:`!datetime` subclass - no longer coverts the latter to :class:`!date`, ignoring the time part + no longer converts the latter to :class:`!date`, ignoring the time part and the time zone. The default behavior can be changed by overriding the special comparison methods in subclasses. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 21ac2c87a1859e0..e255fad55e4a250 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -104,7 +104,7 @@ The following options are accepted: Show offsets of instructions. If :file:`infile` is specified, its disassembled code will be written to stdout. -Otherwise, disassembly is performed on compiled source code recieved from stdin. +Otherwise, disassembly is performed on compiled source code received from stdin. Bytecode analysis ----------------- diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index ac3458c86fd4c4f..72da777cd1407e3 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -211,7 +211,7 @@ Other Functions After you use :func:`CrtSetReportMode` to specify :const:`CRTDBG_MODE_FILE`, you can specify the file handle to receive the message text. *type* must be - one of the :const:`!CRT_\*` constants listed below. *file* shuld be the file + one of the :const:`!CRT_\*` constants listed below. *file* should be the file handle your want specified. Only available in :ref:`debug build of Python `. From 6cf70678e2d4126ed921e6288477ebd6889b9af7 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 7 May 2024 12:35:32 +0200 Subject: [PATCH 194/220] gh-109975: Update 'What's New in Python 3.13' for beta (#118694) --- Doc/whatsnew/3.13.rst | 9 ++------- .../C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst | 5 +++-- .../Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst | 2 +- .../2024-04-17-21-28-24.gh-issue-116931._AS09h.rst | 2 +- .../2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst | 2 +- .../2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst | 5 +++-- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index df38a6154701c34..5b5155b00c46142 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -347,7 +347,7 @@ Other Language Changes New Modules =========== -* None yet. +* None. Improved Modules @@ -1647,7 +1647,7 @@ PEP 594: dead batteries * :mod:`!pipes`: use the :mod:`subprocess` module instead. (Contributed by Victor Stinner in :gh:`104773`.) - * :mod:`!sndhdr`: use the projects :pypi:`filetype_, + * :mod:`!sndhdr`: use the projects :pypi:`filetype`, :pypi:`puremagic`, or :pypi:`python-magic` instead. (Contributed by Victor Stinner in :gh:`104773`.) @@ -1791,11 +1791,6 @@ webbrowser attribute instead. (Contributed by Nikita Sobolev in :gh:`105546`.) -Others ------- - -* None yet - CPython bytecode changes ======================== diff --git a/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst b/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst index 4b7dda610fc2b25..f48d4235d286948 100644 --- a/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst +++ b/Misc/NEWS.d/next/C API/2024-04-08-09-44-29.gh-issue-117534.54ZE_n.rst @@ -1,2 +1,3 @@ -Improve validation logic in the C implementation of :meth:`datetime.fromisoformat` -to better handle invalid years. Patch by Vlad Efanov. +Improve validation logic in the C implementation of +:meth:`datetime.datetime.fromisoformat` to better handle invalid years. +Patch by Vlad Efanov. diff --git a/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst b/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst index 0a2e3e3c540c486..3e9cca5e10fd5db 100644 --- a/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst +++ b/Misc/NEWS.d/next/Library/2018-02-13-10-02-54.bpo-32839.McbVz3.rst @@ -1 +1 @@ -Add the :meth:`after_info` method for Tkinter widgets. +Add the :meth:`!after_info` method for Tkinter widgets. diff --git a/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst b/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst index 98df8f59f71b762..a40276c7b731ed0 100644 --- a/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst +++ b/Misc/NEWS.d/next/Library/2024-04-17-21-28-24.gh-issue-116931._AS09h.rst @@ -1 +1 @@ -Add parameter *fileobj* check for :func:`tarfile.addfile()` +Add parameter *fileobj* check for :func:`tarfile.TarFile.addfile` diff --git a/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst b/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst index daa5fe7c0f29171..aa2da4f837bb6c7 100644 --- a/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst +++ b/Misc/NEWS.d/next/Library/2024-04-24-12-20-48.gh-issue-118013.TKn_kZ.rst @@ -5,5 +5,5 @@ to that instance's class to persist in an internal cache in the class was dynamically created, the class held strong references to other objects which took up a significant amount of memory, and the cache contained the sole strong reference to the class. The fix for the regression -leads to a slowdown in :func:`getattr_static`, but the function should still +leads to a slowdown in :func:`!getattr_static`, but the function should still be significantly faster than it was in Python 3.11. Patch by Alex Waygood. diff --git a/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst b/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst index 9b0ea9978a195eb..d822e36b6b01115 100644 --- a/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst +++ b/Misc/NEWS.d/next/Library/2024-04-24-12-29-33.gh-issue-118221.2k_bac.rst @@ -1,2 +1,3 @@ -Fix a bug where :func:`sqlite3.iterdump` could fail if a custom :attr:`row -factory ` was used. Patch by Erlend Aasland. +Fix a bug where :meth:`sqlite3.Connection.iterdump` could fail if a custom +:attr:`row factory ` was used. Patch by Erlend +Aasland. From 1e428426c836b9a434810a6b99f70454d3a9611e Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 7 May 2024 12:35:51 +0200 Subject: [PATCH 195/220] gh-118650: Exclude `_repr_*` methods from Enum's _sunder_ reservation (GH-118651) --- Lib/enum.py | 5 ++++- .../Library/2024-05-06-16-52-40.gh-issue-118650.qKz5lp.rst | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-06-16-52-40.gh-issue-118650.qKz5lp.rst diff --git a/Lib/enum.py b/Lib/enum.py index 5485306c18843b2..c36fc75a24a2392 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -365,7 +365,10 @@ def __setitem__(self, key, value): '_generate_next_value_', '_numeric_repr_', '_missing_', '_ignore_', '_iter_member_', '_iter_member_by_value_', '_iter_member_by_def_', '_add_alias_', '_add_value_alias_', - ): + # While not in use internally, those are common for pretty + # printing and thus excluded from Enum's reservation of + # _sunder_ names + ) and not key.startswith('_repr_'): raise ValueError( '_sunder_ names, such as %r, are reserved for future Enum use' % (key, ) diff --git a/Misc/NEWS.d/next/Library/2024-05-06-16-52-40.gh-issue-118650.qKz5lp.rst b/Misc/NEWS.d/next/Library/2024-05-06-16-52-40.gh-issue-118650.qKz5lp.rst new file mode 100644 index 000000000000000..85575ea9ce8725b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-06-16-52-40.gh-issue-118650.qKz5lp.rst @@ -0,0 +1,2 @@ +The ``enum`` module allows method named ``_repr_*`` to be defined on +``Enum`` types. From 6f768b71bab837c6c4aac4d3ddd251e55025fe0b Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 7 May 2024 12:50:40 +0200 Subject: [PATCH 196/220] gh-118119: Re-use `sep` in `posixpath.expanduser()` (GH-118120) --- Lib/posixpath.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index f189c3359fbea64..b4547d7893b1cd8 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -298,11 +298,8 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. From 0f8a07d158ba0a0dfd3584d8195f30dd16738555 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 7 May 2024 12:12:28 +0100 Subject: [PATCH 197/220] gh-118418: Deprecate failing to pass a value to the *type_params* parameter of some private `typing` APIs (#118695) --- Lib/test/test_typing.py | 25 +++++++++++ Lib/typing.py | 41 ++++++++++++++----- ...-05-07-11-23-11.gh-issue-118418.QPMdJm.rst | 6 +++ 3 files changed, 61 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-07-11-23-11.gh-issue-118418.QPMdJm.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8f0be1fd3f55e54..012613302d1b532 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6308,6 +6308,31 @@ def test_or(self): self.assertEqual(X | "x", Union[X, "x"]) self.assertEqual("x" | X, Union["x", X]) + def test_deprecation_for_no_type_params_passed_to__evaluate(self): + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing._eval_type' is deprecated" + ) + ) as cm: + self.assertEqual(typing._eval_type(list["int"], globals(), {}), list[int]) + + self.assertEqual(cm.filename, __file__) + + f = ForwardRef("int") + + with self.assertWarnsRegex( + DeprecationWarning, + ( + "Failing to pass a value to the 'type_params' parameter " + "of 'typing.ForwardRef._evaluate' is deprecated" + ) + ) as cm: + self.assertIs(f._evaluate(globals(), {}, recursive_guard=frozenset()), int) + + self.assertEqual(cm.filename, __file__) + @lru_cache() def cached_func(x, y): diff --git a/Lib/typing.py b/Lib/typing.py index c159fcfda68ee8f..e48583673a7be22 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -437,13 +437,38 @@ def inner(*args, **kwds): return decorator -def _eval_type(t, globalns, localns, type_params=None, *, recursive_guard=frozenset()): +def _deprecation_warning_for_no_type_params_passed(funcname: str) -> None: + import warnings + + depr_message = ( + f"Failing to pass a value to the 'type_params' parameter " + f"of {funcname!r} is deprecated, as it leads to incorrect behaviour " + f"when calling {funcname} on a stringified annotation " + f"that references a PEP 695 type parameter. " + f"It will be disallowed in Python 3.15." + ) + warnings.warn(depr_message, category=DeprecationWarning, stacklevel=3) + + +class _Sentinel: + __slots__ = () + def __repr__(self): + return '' + + +_sentinel = _Sentinel() + + +def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset()): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). recursive_guard is used to prevent infinite recursion with a recursive ForwardRef. """ + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing._eval_type") + type_params = () if isinstance(t, ForwardRef): return t._evaluate(globalns, localns, type_params, recursive_guard=recursive_guard) if isinstance(t, (_GenericAlias, GenericAlias, types.UnionType)): @@ -1018,7 +1043,10 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): self.__forward_is_class__ = is_class self.__forward_module__ = module - def _evaluate(self, globalns, localns, type_params=None, *, recursive_guard): + def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): + if type_params is _sentinel: + _deprecation_warning_for_no_type_params_passed("typing.ForwardRef._evaluate") + type_params = () if self.__forward_arg__ in recursive_guard: return self if not self.__forward_evaluated__ or localns is not globalns: @@ -2998,15 +3026,6 @@ def __new__(cls, typename, bases, ns): return nm_tpl -class _Sentinel: - __slots__ = () - def __repr__(self): - return '' - - -_sentinel = _Sentinel() - - def NamedTuple(typename, fields=_sentinel, /, **kwargs): """Typed version of namedtuple. diff --git a/Misc/NEWS.d/next/Library/2024-05-07-11-23-11.gh-issue-118418.QPMdJm.rst b/Misc/NEWS.d/next/Library/2024-05-07-11-23-11.gh-issue-118418.QPMdJm.rst new file mode 100644 index 000000000000000..be371c507cbf798 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-07-11-23-11.gh-issue-118418.QPMdJm.rst @@ -0,0 +1,6 @@ +A :exc:`DeprecationWarning` is now emitted if you fail to pass a value to +the new *type_params* parameter of ``typing._eval_type()`` or +``typing.ForwardRef._evaluate()``. (Using either of these private and +undocumented functions is discouraged to begin with, but failing to pass a +value to the ``type_params`` parameter may lead to incorrect behaviour on +Python 3.12 or newer.) From 44a9f3db2b40ba41999002799a74e6b6f2a3a50a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 7 May 2024 12:34:19 +0100 Subject: [PATCH 198/220] gh-109653: What's new: Note improved import times for several stdlib modules in Python 3.13 (#118697) --- Doc/whatsnew/3.13.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 5b5155b00c46142..d875527220f79f9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -999,6 +999,15 @@ Optimizations section above for details. (Contributed by Jakub Kulik in :gh:`113117`.) +* Several standard library modules have had their import times significantly + improved. For example, the import time of the :mod:`typing` module has been + reduced by around a third by removing dependencies on :mod:`re` and + :mod:`contextlib`. Other modules to enjoy import-time speedups include + :mod:`importlib.metadata`, :mod:`threading`, :mod:`enum`, :mod:`functools` + and :mod:`email.utils`. + (Contributed by Alex Waygood, Shantanu Jain, Adam Turner, Daniel Hollas and + others in :gh:`109653`.) + .. _whatsnew313-jit-compiler: Experimental JIT Compiler From 48e52fe2c9a7b33671f6b5d1420a71a6f31ad64b Mon Sep 17 00:00:00 2001 From: Momo Eissenhauer Date: Tue, 7 May 2024 13:42:18 +0200 Subject: [PATCH 199/220] gh-118310: Fix documentation for `enum.Enum.__new__` (GH-118311) The provided example was incorrect: - The example enum was missing the `int` mixin as implied by the context - The value of `int('1a', 16)` was incorrectly given as 17 (should be 26) --- Doc/library/enum.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 00f617e5ffc5e74..6837b45894b3a92 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -402,13 +402,15 @@ Data Types in the member assignment will be passed; e.g. >>> from enum import Enum - >>> class MyIntEnum(Enum): - ... SEVENTEEN = '1a', 16 + >>> class MyIntEnum(int, Enum): + ... TWENTYSIX = '1a', 16 - results in the call ``int('1a', 16)`` and a value of ``17`` for the member. + results in the call ``int('1a', 16)`` and a value of ``26`` for the member. - .. note:: When writing a custom ``__new__``, do not use ``super().__new__`` -- - call the appropriate ``__new__`` instead. + .. note:: + + When writing a custom ``__new__``, do not use ``super().__new__`` -- + call the appropriate ``__new__`` instead. .. method:: Enum.__repr__(self) From 04859228aa11756558807bcf99ccff78e4e8c56d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 7 May 2024 13:47:21 +0200 Subject: [PATCH 200/220] gh-118518: Rename `PYTHONPERFJITSUPPORT` and `-X perfjit` with underscores (#118693) --- Doc/c-api/init_config.rst | 8 ++++---- Doc/howto/perf_profiling.rst | 8 ++++---- Doc/using/cmdline.rst | 10 +++++----- Doc/whatsnew/3.13.rst | 2 +- Lib/test/test_perf_profiler.py | 2 +- .../2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst | 4 ++-- Python/initconfig.c | 4 ++-- Python/sysmodule.c | 2 +- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 63ec25b8e60badb..5195f6cccfe9dfd 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1250,10 +1250,10 @@ PyConfig If non-zero, initialize the perf trampoline. See :ref:`perf_profiling` for more information. - Set by :option:`-X perf <-X>` command line option and by the - :envvar:`PYTHONPERFSUPPORT` environment variable for perf support - with stack pointers and :option:`-X perfjit <-X>` command line option - and by the :envvar:`PYTHONPERFJITSUPPORT` environment variable for perf + Set by :option:`-X perf <-X>` command-line option and by the + :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support + with stack pointers and :option:`-X perf_jit <-X>` command-line option + and by the :envvar:`PYTHON_PERF_JIT_SUPPORT` environment variable for perf support with DWARF JIT information. Default: ``-1``. diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index 0ce66f31274a5f8..ed2b76ff4f410c2 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -216,19 +216,19 @@ needs to generate unwinding information for every Python function call on the fl ``perf`` will take more time to process the data because it will need to use the DWARF debugging information to unwind the stack and this is a slow process. -To enable this mode, you can use the environment variable :envvar:`PYTHONPERFJITSUPPORT` or the -:option:`-X perfjit <-X>` option, which will enable the JIT mode for the ``perf`` profiler. +To enable this mode, you can use the environment variable :envvar:`PYTHON_PERF_JIT_SUPPORT` or the +:option:`-X perf_jit <-X>` option, which will enable the JIT mode for the ``perf`` profiler. When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file. - $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperfjit my_script.py + $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py $ perf inject -i perf.data --jit $ perf report -g -i perf.data or using the environment variable:: - $ PYTHONPERFJITSUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py + $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py $ perf inject -i perf.data --jit $ perf report -g -i perf.data diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 522e6e5ccd290d9..2d95fa9474033f5 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -587,11 +587,11 @@ Miscellaneous options .. versionadded:: 3.12 - * ``-X perfjit`` enables support for the Linux ``perf`` profiler with DWARF + * ``-X perf_jit`` enables support for the Linux ``perf`` profiler with DWARF support. When this option is provided, the ``perf`` profiler will be able - to report Python calls using DWARF ifnormation. This option is only available on + to report Python calls using DWARF information. This option is only available on some platforms and will do nothing if is not supported on the current - system. The default value is "off". See also :envvar:`PYTHONPERFJITSUPPORT` + system. The default value is "off". See also :envvar:`PYTHON_PERF_JIT_SUPPORT` and :ref:`perf_profiling`. .. versionadded:: 3.13 @@ -1137,7 +1137,7 @@ conflict. .. versionadded:: 3.12 -.. envvar:: PYTHONPERFJITSUPPORT +.. envvar:: PYTHON_PERF_JIT_SUPPORT If this variable is set to a nonzero value, it enables support for the Linux ``perf`` profiler so Python calls can be detected by it @@ -1145,7 +1145,7 @@ conflict. If set to ``0``, disable Linux ``perf`` profiler support. - See also the :option:`-X perfjit <-X>` command-line option + See also the :option:`-X perf_jit <-X>` command-line option and :ref:`perf_profiling`. .. versionadded:: 3.13 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index d875527220f79f9..b084e78889c81f9 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -272,7 +272,7 @@ Other Language Changes * Add :ref:`support for the perf profiler ` working without frame pointers through the new environment variable - :envvar:`PYTHONPERFJITSUPPORT` and command-line option :option:`-X perfjit + :envvar:`PYTHON_PERF_JIT_SUPPORT` and command-line option :option:`-X perf_jit <-X>` (Contributed by Pablo Galindo in :gh:`118518`.) * The new :envvar:`PYTHON_HISTORY` environment variable can be used to change diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 9f72a46dc91c35a..496983f7b49f52e 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -494,7 +494,7 @@ class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): def run_perf(self, script_dir, script, activate_trampoline=True): if activate_trampoline: return run_perf( - script_dir, sys.executable, "-Xperfjit", script, use_jit=True + script_dir, sys.executable, "-Xperf_jit", script, use_jit=True ) return run_perf(script_dir, sys.executable, script, use_jit=True) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst index 7d4c003019bdef0..4c7c18abb16bf34 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-02-20-32-42.gh-issue-118518.m-JbTi.rst @@ -1,4 +1,4 @@ Allow the Linux perf support to work without frame pointers using perf's advanced JIT support. The feature is activated when using the -``PYTHONPERFJITSUPPORT`` environment variable or when running Python with -``-Xperfjit``. Patch by Pablo Galindo +``PYTHON_PERF_JIT_SUPPORT`` environment variable or when running Python with +``-Xperf_jit``. Patch by Pablo Galindo. diff --git a/Python/initconfig.c b/Python/initconfig.c index 0e53d60e3537af3..a28c08c5318ddce 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1703,7 +1703,7 @@ config_init_perf_profiling(PyConfig *config) if (xoption) { config->perf_profiling = 1; } - env = config_get_env(config, "PYTHONPERFJITSUPPORT"); + env = config_get_env(config, "PYTHON_PERF_JIT_SUPPORT"); if (env) { if (_Py_str_to_int(env, &active) != 0) { active = 0; @@ -1712,7 +1712,7 @@ config_init_perf_profiling(PyConfig *config) config->perf_profiling = 2; } } - xoption = config_get_xoption(config, L"perfjit"); + xoption = config_get_xoption(config, L"perf_jit"); if (xoption) { config->perf_profiling = 2; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 601d8ae32bb34ba..4da13e4552e786d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2294,7 +2294,7 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } - else if (strcmp(backend, "perfjit") == 0) { + else if (strcmp(backend, "perf_jit") == 0) { _PyPerf_Callbacks cur_cb; _PyPerfTrampoline_GetCallbacks(&cur_cb); if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { From b60d4c0d53b6aafbf4a6e560b4cb6f1d5c7240c8 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 7 May 2024 15:01:06 +0300 Subject: [PATCH 201/220] gh-118090: Improve error message for empty type param brackets (GH-118091) --- Grammar/python.gram | 15 +- Lib/test/test_syntax.py | 41 + ...-04-19-11-57-46.gh-issue-118090.eGAQ0B.rst | 1 + Parser/parser.c | 1759 +++++++++-------- 4 files changed, 1016 insertions(+), 800 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-57-46.gh-issue-118090.eGAQ0B.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 1c1c53c4b73ace1..c04bc641779c04e 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -269,11 +269,11 @@ function_def[stmt_ty]: function_def_raw[stmt_ty]: | invalid_def_raw - | 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { _PyAST_FunctionDef(n->v.Name.id, (params) ? params : CHECK(arguments_ty, _PyPegen_empty_arguments(p)), b, NULL, a, NEW_TYPE_COMMENT(p, tc), t, EXTRA) } - | 'async' 'def' n=NAME t=[type_params] &&'(' params=[params] ')' a=['->' z=expression { z }] &&':' tc=[func_type_comment] b=block { + | 'async' 'def' n=NAME t=[type_params] '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { CHECK_VERSION( stmt_ty, 5, @@ -641,7 +641,9 @@ type_alias[stmt_ty]: # Type parameter declaration # -------------------------- -type_params[asdl_type_param_seq*]: '[' t=type_param_seq ']' { +type_params[asdl_type_param_seq*]: + | invalid_type_params + | '[' t=type_param_seq ']' { CHECK_VERSION(asdl_type_param_seq *, 12, "Type parameter lists are", t) } type_param_seq[asdl_type_param_seq*]: a[asdl_type_param_seq*]=','.type_param+ [','] { a } @@ -1392,6 +1394,7 @@ invalid_for_stmt: invalid_def_raw: | ['async'] a='def' NAME [type_params] '(' [params] ')' ['->' expression] ':' NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block after function definition on line %d", a->lineno) } + | ['async'] 'def' NAME [type_params] &&'(' [params] ')' ['->' expression] &&':' [func_type_comment] block invalid_class_def_raw: | 'class' NAME [type_params] ['(' [arguments] ')'] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='class' NAME [type_params] ['(' [arguments] ')'] ':' NEWLINE !INDENT { @@ -1435,3 +1438,9 @@ invalid_arithmetic: | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } invalid_factor: | ('+' | '-' | '~') a='not' b=factor { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } + +invalid_type_params: + | '[' token=']' { + RAISE_SYNTAX_ERROR_STARTING_FROM( + token, + "Type parameter list cannot be empty")} diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index de783f714509a31..b978838ea7003f8 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1213,6 +1213,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -2027,6 +2043,31 @@ def f(x: *b) Invalid expressions in type scopes: + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + >>> type A[T: (x:=3)] = int Traceback (most recent call last): ... diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-57-46.gh-issue-118090.eGAQ0B.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-57-46.gh-issue-118090.eGAQ0B.rst new file mode 100644 index 000000000000000..2abbbfec06417c6 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-19-11-57-46.gh-issue-118090.eGAQ0B.rst @@ -0,0 +1 @@ +Improve :exc:`SyntaxError` message for empty type param brackets. diff --git a/Parser/parser.c b/Parser/parser.c index e34fcada15ebf50..7bfc17a92e29de4 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -30,11 +30,11 @@ static KeywordToken *reserved_keywords[] = { }, (KeywordToken[]) { {"del", 616}, - {"def", 675}, + {"def", 677}, {"for", 672}, {"try", 644}, {"and", 582}, - {"not", 679}, + {"not", 681}, {NULL, -1}, }, (KeywordToken[]) { @@ -51,8 +51,8 @@ static KeywordToken *reserved_keywords[] = { {"raise", 525}, {"yield", 580}, {"break", 508}, - {"async", 674}, - {"class", 677}, + {"async", 676}, + {"class", 679}, {"while", 667}, {"False", 615}, {"await", 590}, @@ -333,286 +333,288 @@ static char *soft_keywords[] = { #define invalid_conversion_character_type 1246 #define invalid_arithmetic_type 1247 #define invalid_factor_type 1248 -#define _loop0_1_type 1249 -#define _loop0_2_type 1250 -#define _loop1_3_type 1251 -#define _loop0_5_type 1252 -#define _gather_4_type 1253 -#define _tmp_6_type 1254 -#define _tmp_7_type 1255 -#define _tmp_8_type 1256 -#define _tmp_9_type 1257 -#define _tmp_10_type 1258 -#define _tmp_11_type 1259 -#define _tmp_12_type 1260 -#define _tmp_13_type 1261 -#define _loop1_14_type 1262 -#define _tmp_15_type 1263 -#define _tmp_16_type 1264 -#define _tmp_17_type 1265 -#define _loop0_19_type 1266 -#define _gather_18_type 1267 -#define _loop0_21_type 1268 -#define _gather_20_type 1269 -#define _tmp_22_type 1270 -#define _tmp_23_type 1271 -#define _loop0_24_type 1272 -#define _loop1_25_type 1273 -#define _loop0_27_type 1274 -#define _gather_26_type 1275 -#define _tmp_28_type 1276 -#define _loop0_30_type 1277 -#define _gather_29_type 1278 -#define _tmp_31_type 1279 -#define _loop1_32_type 1280 -#define _tmp_33_type 1281 -#define _tmp_34_type 1282 -#define _tmp_35_type 1283 -#define _loop0_36_type 1284 -#define _loop0_37_type 1285 -#define _loop0_38_type 1286 -#define _loop1_39_type 1287 -#define _loop0_40_type 1288 -#define _loop1_41_type 1289 -#define _loop1_42_type 1290 -#define _loop1_43_type 1291 -#define _loop0_44_type 1292 -#define _loop1_45_type 1293 -#define _loop0_46_type 1294 -#define _loop1_47_type 1295 -#define _loop0_48_type 1296 -#define _loop0_49_type 1297 -#define _loop1_50_type 1298 -#define _loop0_52_type 1299 -#define _gather_51_type 1300 -#define _loop0_54_type 1301 -#define _gather_53_type 1302 -#define _loop0_56_type 1303 -#define _gather_55_type 1304 -#define _loop0_58_type 1305 -#define _gather_57_type 1306 -#define _tmp_59_type 1307 -#define _loop1_60_type 1308 -#define _loop1_61_type 1309 -#define _tmp_62_type 1310 -#define _tmp_63_type 1311 -#define _loop1_64_type 1312 -#define _loop0_66_type 1313 -#define _gather_65_type 1314 -#define _tmp_67_type 1315 -#define _tmp_68_type 1316 -#define _tmp_69_type 1317 -#define _tmp_70_type 1318 -#define _loop0_72_type 1319 -#define _gather_71_type 1320 -#define _loop0_74_type 1321 -#define _gather_73_type 1322 -#define _tmp_75_type 1323 -#define _loop0_77_type 1324 -#define _gather_76_type 1325 -#define _loop0_79_type 1326 -#define _gather_78_type 1327 -#define _loop0_81_type 1328 -#define _gather_80_type 1329 -#define _loop1_82_type 1330 -#define _loop1_83_type 1331 -#define _loop0_85_type 1332 -#define _gather_84_type 1333 -#define _loop1_86_type 1334 -#define _loop1_87_type 1335 -#define _loop1_88_type 1336 -#define _tmp_89_type 1337 -#define _loop0_91_type 1338 -#define _gather_90_type 1339 -#define _tmp_92_type 1340 -#define _tmp_93_type 1341 -#define _tmp_94_type 1342 -#define _tmp_95_type 1343 -#define _tmp_96_type 1344 -#define _tmp_97_type 1345 -#define _loop0_98_type 1346 -#define _loop0_99_type 1347 -#define _loop0_100_type 1348 -#define _loop1_101_type 1349 -#define _loop0_102_type 1350 -#define _loop1_103_type 1351 -#define _loop1_104_type 1352 -#define _loop1_105_type 1353 -#define _loop0_106_type 1354 -#define _loop1_107_type 1355 -#define _loop0_108_type 1356 -#define _loop1_109_type 1357 -#define _loop0_110_type 1358 -#define _loop1_111_type 1359 -#define _loop0_112_type 1360 -#define _loop0_113_type 1361 -#define _loop1_114_type 1362 -#define _tmp_115_type 1363 -#define _loop0_117_type 1364 -#define _gather_116_type 1365 -#define _loop1_118_type 1366 -#define _loop0_119_type 1367 -#define _loop0_120_type 1368 -#define _tmp_121_type 1369 -#define _tmp_122_type 1370 -#define _loop0_124_type 1371 -#define _gather_123_type 1372 -#define _tmp_125_type 1373 -#define _loop0_127_type 1374 -#define _gather_126_type 1375 -#define _loop0_129_type 1376 -#define _gather_128_type 1377 -#define _loop0_131_type 1378 -#define _gather_130_type 1379 -#define _loop0_133_type 1380 -#define _gather_132_type 1381 -#define _loop0_134_type 1382 -#define _loop0_136_type 1383 -#define _gather_135_type 1384 -#define _loop1_137_type 1385 -#define _tmp_138_type 1386 -#define _loop0_140_type 1387 -#define _gather_139_type 1388 -#define _loop0_142_type 1389 -#define _gather_141_type 1390 -#define _loop0_144_type 1391 -#define _gather_143_type 1392 -#define _loop0_146_type 1393 -#define _gather_145_type 1394 -#define _loop0_148_type 1395 -#define _gather_147_type 1396 -#define _tmp_149_type 1397 -#define _tmp_150_type 1398 -#define _loop0_152_type 1399 -#define _gather_151_type 1400 -#define _tmp_153_type 1401 -#define _tmp_154_type 1402 -#define _tmp_155_type 1403 -#define _tmp_156_type 1404 -#define _tmp_157_type 1405 -#define _tmp_158_type 1406 -#define _tmp_159_type 1407 -#define _tmp_160_type 1408 -#define _tmp_161_type 1409 -#define _tmp_162_type 1410 -#define _loop0_163_type 1411 -#define _loop0_164_type 1412 -#define _loop0_165_type 1413 -#define _tmp_166_type 1414 -#define _tmp_167_type 1415 -#define _tmp_168_type 1416 -#define _tmp_169_type 1417 -#define _loop0_170_type 1418 -#define _loop0_171_type 1419 -#define _loop0_172_type 1420 -#define _loop1_173_type 1421 -#define _tmp_174_type 1422 -#define _loop0_175_type 1423 -#define _tmp_176_type 1424 -#define _loop0_177_type 1425 -#define _loop1_178_type 1426 -#define _tmp_179_type 1427 -#define _tmp_180_type 1428 -#define _tmp_181_type 1429 -#define _loop0_182_type 1430 -#define _tmp_183_type 1431 -#define _tmp_184_type 1432 -#define _loop1_185_type 1433 -#define _tmp_186_type 1434 -#define _loop0_187_type 1435 -#define _loop0_188_type 1436 -#define _loop0_189_type 1437 -#define _loop0_191_type 1438 -#define _gather_190_type 1439 -#define _tmp_192_type 1440 -#define _loop0_193_type 1441 -#define _tmp_194_type 1442 -#define _loop0_195_type 1443 -#define _loop1_196_type 1444 -#define _loop1_197_type 1445 -#define _tmp_198_type 1446 -#define _tmp_199_type 1447 -#define _loop0_200_type 1448 -#define _tmp_201_type 1449 -#define _tmp_202_type 1450 -#define _tmp_203_type 1451 -#define _loop0_205_type 1452 -#define _gather_204_type 1453 -#define _loop0_207_type 1454 -#define _gather_206_type 1455 -#define _loop0_209_type 1456 -#define _gather_208_type 1457 -#define _loop0_211_type 1458 -#define _gather_210_type 1459 -#define _loop0_213_type 1460 -#define _gather_212_type 1461 -#define _tmp_214_type 1462 -#define _loop0_215_type 1463 -#define _loop1_216_type 1464 -#define _tmp_217_type 1465 -#define _loop0_218_type 1466 -#define _loop1_219_type 1467 -#define _tmp_220_type 1468 -#define _tmp_221_type 1469 -#define _tmp_222_type 1470 -#define _tmp_223_type 1471 -#define _tmp_224_type 1472 -#define _tmp_225_type 1473 -#define _tmp_226_type 1474 -#define _tmp_227_type 1475 -#define _tmp_228_type 1476 -#define _tmp_229_type 1477 -#define _loop0_231_type 1478 -#define _gather_230_type 1479 -#define _tmp_232_type 1480 -#define _tmp_233_type 1481 -#define _tmp_234_type 1482 -#define _tmp_235_type 1483 -#define _tmp_236_type 1484 -#define _tmp_237_type 1485 -#define _tmp_238_type 1486 -#define _loop0_239_type 1487 -#define _tmp_240_type 1488 -#define _tmp_241_type 1489 -#define _tmp_242_type 1490 -#define _tmp_243_type 1491 -#define _tmp_244_type 1492 -#define _tmp_245_type 1493 -#define _tmp_246_type 1494 -#define _tmp_247_type 1495 -#define _tmp_248_type 1496 -#define _tmp_249_type 1497 -#define _tmp_250_type 1498 -#define _tmp_251_type 1499 -#define _tmp_252_type 1500 -#define _tmp_253_type 1501 -#define _tmp_254_type 1502 -#define _tmp_255_type 1503 -#define _loop0_256_type 1504 -#define _tmp_257_type 1505 -#define _tmp_258_type 1506 -#define _tmp_259_type 1507 -#define _tmp_260_type 1508 -#define _tmp_261_type 1509 -#define _tmp_262_type 1510 -#define _tmp_263_type 1511 -#define _tmp_264_type 1512 -#define _tmp_265_type 1513 -#define _tmp_266_type 1514 -#define _tmp_267_type 1515 -#define _tmp_268_type 1516 -#define _tmp_269_type 1517 -#define _tmp_270_type 1518 -#define _tmp_271_type 1519 -#define _tmp_272_type 1520 -#define _loop0_274_type 1521 -#define _gather_273_type 1522 -#define _tmp_275_type 1523 -#define _tmp_276_type 1524 -#define _tmp_277_type 1525 -#define _tmp_278_type 1526 -#define _tmp_279_type 1527 -#define _tmp_280_type 1528 +#define invalid_type_params_type 1249 +#define _loop0_1_type 1250 +#define _loop0_2_type 1251 +#define _loop1_3_type 1252 +#define _loop0_5_type 1253 +#define _gather_4_type 1254 +#define _tmp_6_type 1255 +#define _tmp_7_type 1256 +#define _tmp_8_type 1257 +#define _tmp_9_type 1258 +#define _tmp_10_type 1259 +#define _tmp_11_type 1260 +#define _tmp_12_type 1261 +#define _tmp_13_type 1262 +#define _loop1_14_type 1263 +#define _tmp_15_type 1264 +#define _tmp_16_type 1265 +#define _tmp_17_type 1266 +#define _loop0_19_type 1267 +#define _gather_18_type 1268 +#define _loop0_21_type 1269 +#define _gather_20_type 1270 +#define _tmp_22_type 1271 +#define _tmp_23_type 1272 +#define _loop0_24_type 1273 +#define _loop1_25_type 1274 +#define _loop0_27_type 1275 +#define _gather_26_type 1276 +#define _tmp_28_type 1277 +#define _loop0_30_type 1278 +#define _gather_29_type 1279 +#define _tmp_31_type 1280 +#define _loop1_32_type 1281 +#define _tmp_33_type 1282 +#define _tmp_34_type 1283 +#define _tmp_35_type 1284 +#define _loop0_36_type 1285 +#define _loop0_37_type 1286 +#define _loop0_38_type 1287 +#define _loop1_39_type 1288 +#define _loop0_40_type 1289 +#define _loop1_41_type 1290 +#define _loop1_42_type 1291 +#define _loop1_43_type 1292 +#define _loop0_44_type 1293 +#define _loop1_45_type 1294 +#define _loop0_46_type 1295 +#define _loop1_47_type 1296 +#define _loop0_48_type 1297 +#define _loop0_49_type 1298 +#define _loop1_50_type 1299 +#define _loop0_52_type 1300 +#define _gather_51_type 1301 +#define _loop0_54_type 1302 +#define _gather_53_type 1303 +#define _loop0_56_type 1304 +#define _gather_55_type 1305 +#define _loop0_58_type 1306 +#define _gather_57_type 1307 +#define _tmp_59_type 1308 +#define _loop1_60_type 1309 +#define _loop1_61_type 1310 +#define _tmp_62_type 1311 +#define _tmp_63_type 1312 +#define _loop1_64_type 1313 +#define _loop0_66_type 1314 +#define _gather_65_type 1315 +#define _tmp_67_type 1316 +#define _tmp_68_type 1317 +#define _tmp_69_type 1318 +#define _tmp_70_type 1319 +#define _loop0_72_type 1320 +#define _gather_71_type 1321 +#define _loop0_74_type 1322 +#define _gather_73_type 1323 +#define _tmp_75_type 1324 +#define _loop0_77_type 1325 +#define _gather_76_type 1326 +#define _loop0_79_type 1327 +#define _gather_78_type 1328 +#define _loop0_81_type 1329 +#define _gather_80_type 1330 +#define _loop1_82_type 1331 +#define _loop1_83_type 1332 +#define _loop0_85_type 1333 +#define _gather_84_type 1334 +#define _loop1_86_type 1335 +#define _loop1_87_type 1336 +#define _loop1_88_type 1337 +#define _tmp_89_type 1338 +#define _loop0_91_type 1339 +#define _gather_90_type 1340 +#define _tmp_92_type 1341 +#define _tmp_93_type 1342 +#define _tmp_94_type 1343 +#define _tmp_95_type 1344 +#define _tmp_96_type 1345 +#define _tmp_97_type 1346 +#define _loop0_98_type 1347 +#define _loop0_99_type 1348 +#define _loop0_100_type 1349 +#define _loop1_101_type 1350 +#define _loop0_102_type 1351 +#define _loop1_103_type 1352 +#define _loop1_104_type 1353 +#define _loop1_105_type 1354 +#define _loop0_106_type 1355 +#define _loop1_107_type 1356 +#define _loop0_108_type 1357 +#define _loop1_109_type 1358 +#define _loop0_110_type 1359 +#define _loop1_111_type 1360 +#define _loop0_112_type 1361 +#define _loop0_113_type 1362 +#define _loop1_114_type 1363 +#define _tmp_115_type 1364 +#define _loop0_117_type 1365 +#define _gather_116_type 1366 +#define _loop1_118_type 1367 +#define _loop0_119_type 1368 +#define _loop0_120_type 1369 +#define _tmp_121_type 1370 +#define _tmp_122_type 1371 +#define _loop0_124_type 1372 +#define _gather_123_type 1373 +#define _tmp_125_type 1374 +#define _loop0_127_type 1375 +#define _gather_126_type 1376 +#define _loop0_129_type 1377 +#define _gather_128_type 1378 +#define _loop0_131_type 1379 +#define _gather_130_type 1380 +#define _loop0_133_type 1381 +#define _gather_132_type 1382 +#define _loop0_134_type 1383 +#define _loop0_136_type 1384 +#define _gather_135_type 1385 +#define _loop1_137_type 1386 +#define _tmp_138_type 1387 +#define _loop0_140_type 1388 +#define _gather_139_type 1389 +#define _loop0_142_type 1390 +#define _gather_141_type 1391 +#define _loop0_144_type 1392 +#define _gather_143_type 1393 +#define _loop0_146_type 1394 +#define _gather_145_type 1395 +#define _loop0_148_type 1396 +#define _gather_147_type 1397 +#define _tmp_149_type 1398 +#define _tmp_150_type 1399 +#define _loop0_152_type 1400 +#define _gather_151_type 1401 +#define _tmp_153_type 1402 +#define _tmp_154_type 1403 +#define _tmp_155_type 1404 +#define _tmp_156_type 1405 +#define _tmp_157_type 1406 +#define _tmp_158_type 1407 +#define _tmp_159_type 1408 +#define _tmp_160_type 1409 +#define _tmp_161_type 1410 +#define _tmp_162_type 1411 +#define _loop0_163_type 1412 +#define _loop0_164_type 1413 +#define _loop0_165_type 1414 +#define _tmp_166_type 1415 +#define _tmp_167_type 1416 +#define _tmp_168_type 1417 +#define _tmp_169_type 1418 +#define _loop0_170_type 1419 +#define _loop0_171_type 1420 +#define _loop0_172_type 1421 +#define _loop1_173_type 1422 +#define _tmp_174_type 1423 +#define _loop0_175_type 1424 +#define _tmp_176_type 1425 +#define _loop0_177_type 1426 +#define _loop1_178_type 1427 +#define _tmp_179_type 1428 +#define _tmp_180_type 1429 +#define _tmp_181_type 1430 +#define _loop0_182_type 1431 +#define _tmp_183_type 1432 +#define _tmp_184_type 1433 +#define _loop1_185_type 1434 +#define _tmp_186_type 1435 +#define _loop0_187_type 1436 +#define _loop0_188_type 1437 +#define _loop0_189_type 1438 +#define _loop0_191_type 1439 +#define _gather_190_type 1440 +#define _tmp_192_type 1441 +#define _loop0_193_type 1442 +#define _tmp_194_type 1443 +#define _loop0_195_type 1444 +#define _loop1_196_type 1445 +#define _loop1_197_type 1446 +#define _tmp_198_type 1447 +#define _tmp_199_type 1448 +#define _loop0_200_type 1449 +#define _tmp_201_type 1450 +#define _tmp_202_type 1451 +#define _tmp_203_type 1452 +#define _loop0_205_type 1453 +#define _gather_204_type 1454 +#define _loop0_207_type 1455 +#define _gather_206_type 1456 +#define _loop0_209_type 1457 +#define _gather_208_type 1458 +#define _loop0_211_type 1459 +#define _gather_210_type 1460 +#define _loop0_213_type 1461 +#define _gather_212_type 1462 +#define _tmp_214_type 1463 +#define _loop0_215_type 1464 +#define _loop1_216_type 1465 +#define _tmp_217_type 1466 +#define _loop0_218_type 1467 +#define _loop1_219_type 1468 +#define _tmp_220_type 1469 +#define _tmp_221_type 1470 +#define _tmp_222_type 1471 +#define _tmp_223_type 1472 +#define _tmp_224_type 1473 +#define _tmp_225_type 1474 +#define _tmp_226_type 1475 +#define _tmp_227_type 1476 +#define _tmp_228_type 1477 +#define _tmp_229_type 1478 +#define _tmp_230_type 1479 +#define _loop0_232_type 1480 +#define _gather_231_type 1481 +#define _tmp_233_type 1482 +#define _tmp_234_type 1483 +#define _tmp_235_type 1484 +#define _tmp_236_type 1485 +#define _tmp_237_type 1486 +#define _tmp_238_type 1487 +#define _tmp_239_type 1488 +#define _loop0_240_type 1489 +#define _tmp_241_type 1490 +#define _tmp_242_type 1491 +#define _tmp_243_type 1492 +#define _tmp_244_type 1493 +#define _tmp_245_type 1494 +#define _tmp_246_type 1495 +#define _tmp_247_type 1496 +#define _tmp_248_type 1497 +#define _tmp_249_type 1498 +#define _tmp_250_type 1499 +#define _tmp_251_type 1500 +#define _tmp_252_type 1501 +#define _tmp_253_type 1502 +#define _tmp_254_type 1503 +#define _tmp_255_type 1504 +#define _tmp_256_type 1505 +#define _loop0_257_type 1506 +#define _tmp_258_type 1507 +#define _tmp_259_type 1508 +#define _tmp_260_type 1509 +#define _tmp_261_type 1510 +#define _tmp_262_type 1511 +#define _tmp_263_type 1512 +#define _tmp_264_type 1513 +#define _tmp_265_type 1514 +#define _tmp_266_type 1515 +#define _tmp_267_type 1516 +#define _tmp_268_type 1517 +#define _tmp_269_type 1518 +#define _tmp_270_type 1519 +#define _tmp_271_type 1520 +#define _tmp_272_type 1521 +#define _tmp_273_type 1522 +#define _loop0_275_type 1523 +#define _gather_274_type 1524 +#define _tmp_276_type 1525 +#define _tmp_277_type 1526 +#define _tmp_278_type 1527 +#define _tmp_279_type 1528 +#define _tmp_280_type 1529 +#define _tmp_281_type 1530 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -863,6 +865,7 @@ static void *invalid_replacement_field_rule(Parser *p); static void *invalid_conversion_character_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); +static void *invalid_type_params_rule(Parser *p); static asdl_seq *_loop0_1_rule(Parser *p); static asdl_seq *_loop0_2_rule(Parser *p); static asdl_seq *_loop1_3_rule(Parser *p); @@ -1092,17 +1095,17 @@ static void *_tmp_226_rule(Parser *p); static void *_tmp_227_rule(Parser *p); static void *_tmp_228_rule(Parser *p); static void *_tmp_229_rule(Parser *p); -static asdl_seq *_loop0_231_rule(Parser *p); -static asdl_seq *_gather_230_rule(Parser *p); -static void *_tmp_232_rule(Parser *p); +static void *_tmp_230_rule(Parser *p); +static asdl_seq *_loop0_232_rule(Parser *p); +static asdl_seq *_gather_231_rule(Parser *p); static void *_tmp_233_rule(Parser *p); static void *_tmp_234_rule(Parser *p); static void *_tmp_235_rule(Parser *p); static void *_tmp_236_rule(Parser *p); static void *_tmp_237_rule(Parser *p); static void *_tmp_238_rule(Parser *p); -static asdl_seq *_loop0_239_rule(Parser *p); -static void *_tmp_240_rule(Parser *p); +static void *_tmp_239_rule(Parser *p); +static asdl_seq *_loop0_240_rule(Parser *p); static void *_tmp_241_rule(Parser *p); static void *_tmp_242_rule(Parser *p); static void *_tmp_243_rule(Parser *p); @@ -1118,8 +1121,8 @@ static void *_tmp_252_rule(Parser *p); static void *_tmp_253_rule(Parser *p); static void *_tmp_254_rule(Parser *p); static void *_tmp_255_rule(Parser *p); -static asdl_seq *_loop0_256_rule(Parser *p); -static void *_tmp_257_rule(Parser *p); +static void *_tmp_256_rule(Parser *p); +static asdl_seq *_loop0_257_rule(Parser *p); static void *_tmp_258_rule(Parser *p); static void *_tmp_259_rule(Parser *p); static void *_tmp_260_rule(Parser *p); @@ -1135,14 +1138,15 @@ static void *_tmp_269_rule(Parser *p); static void *_tmp_270_rule(Parser *p); static void *_tmp_271_rule(Parser *p); static void *_tmp_272_rule(Parser *p); -static asdl_seq *_loop0_274_rule(Parser *p); -static asdl_seq *_gather_273_rule(Parser *p); -static void *_tmp_275_rule(Parser *p); +static void *_tmp_273_rule(Parser *p); +static asdl_seq *_loop0_275_rule(Parser *p); +static asdl_seq *_gather_274_rule(Parser *p); static void *_tmp_276_rule(Parser *p); static void *_tmp_277_rule(Parser *p); static void *_tmp_278_rule(Parser *p); static void *_tmp_279_rule(Parser *p); static void *_tmp_280_rule(Parser *p); +static void *_tmp_281_rule(Parser *p); // file: statements? $ @@ -4366,7 +4370,7 @@ class_def_raw_rule(Parser *p) asdl_stmt_seq* c; void *t; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='class' + (_keyword = _PyPegen_expect_token(p, 679)) // token='class' && (a = _PyPegen_name_token(p)) // NAME && @@ -4474,8 +4478,8 @@ function_def_rule(Parser *p) // function_def_raw: // | invalid_def_raw -// | 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block -// | 'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block +// | 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block +// | 'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block static stmt_ty function_def_raw_rule(Parser *p) { @@ -4516,12 +4520,12 @@ function_def_raw_rule(Parser *p) D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_def_raw")); } - { // 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + { // 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token * _keyword; Token * _literal; Token * _literal_1; @@ -4533,13 +4537,13 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 675)) // token='def' + (_keyword = _PyPegen_expect_token(p, 677)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + (_literal = _PyPegen_expect_token(p, 7)) // token='(' && (params = params_rule(p), !p->error_indicator) // params? && @@ -4547,14 +4551,14 @@ function_def_raw_rule(Parser *p) && (a = _tmp_34_rule(p), !p->error_indicator) // ['->' expression] && - (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -4574,14 +4578,14 @@ function_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); } - { // 'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + { // 'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c> function_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token * _keyword; Token * _keyword_1; Token * _literal; @@ -4594,15 +4598,15 @@ function_def_raw_rule(Parser *p) void *t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && - (_keyword_1 = _PyPegen_expect_token(p, 675)) // token='def' + (_keyword_1 = _PyPegen_expect_token(p, 677)) // token='def' && (n = _PyPegen_name_token(p)) // NAME && (t = type_params_rule(p), !p->error_indicator) // type_params? && - (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + (_literal = _PyPegen_expect_token(p, 7)) // token='(' && (params = params_rule(p), !p->error_indicator) // params? && @@ -4610,14 +4614,14 @@ function_def_raw_rule(Parser *p) && (a = _tmp_35_rule(p), !p->error_indicator) // ['->' expression] && - (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + (_literal_2 = _PyPegen_expect_token(p, 11)) // token=':' && (tc = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? && (b = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + D(fprintf(stderr, "%*c+ function_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -4637,7 +4641,7 @@ function_def_raw_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s function_def_raw[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async' 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async' 'def' NAME type_params? '(' params? ')' ['->' expression] ':' func_type_comment? block")); } _res = NULL; done: @@ -6441,7 +6445,7 @@ for_stmt_rule(Parser *p) expr_ty t; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='for' && @@ -6676,7 +6680,7 @@ with_stmt_rule(Parser *p) asdl_withitem_seq* a; asdl_stmt_seq* b; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 635)) // token='with' && @@ -6728,7 +6732,7 @@ with_stmt_rule(Parser *p) asdl_stmt_seq* b; void *tc; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 635)) // token='with' && @@ -10553,7 +10557,7 @@ type_alias_rule(Parser *p) return _res; } -// type_params: '[' type_param_seq ']' +// type_params: invalid_type_params | '[' type_param_seq ']' static asdl_type_param_seq* type_params_rule(Parser *p) { @@ -10566,6 +10570,25 @@ type_params_rule(Parser *p) } asdl_type_param_seq* _res = NULL; int _mark = p->mark; + if (p->call_invalid_rules) { // invalid_type_params + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_type_params")); + void *invalid_type_params_var; + if ( + (invalid_type_params_var = invalid_type_params_rule(p)) // invalid_type_params + ) + { + D(fprintf(stderr, "%*c+ type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_type_params")); + _res = invalid_type_params_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s type_params[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_type_params")); + } { // '[' type_param_seq ']' if (p->error_indicator) { p->level--; @@ -12100,7 +12123,7 @@ inversion_rule(Parser *p) Token * _keyword; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='not' + (_keyword = _PyPegen_expect_token(p, 681)) // token='not' && (a = inversion_rule(p)) // inversion ) @@ -12754,7 +12777,7 @@ notin_bitwise_or_rule(Parser *p) Token * _keyword_1; expr_ty a; if ( - (_keyword = _PyPegen_expect_token(p, 679)) // token='not' + (_keyword = _PyPegen_expect_token(p, 681)) // token='not' && (_keyword_1 = _PyPegen_expect_token(p, 673)) // token='in' && @@ -12851,7 +12874,7 @@ isnot_bitwise_or_rule(Parser *p) if ( (_keyword = _PyPegen_expect_token(p, 589)) // token='is' && - (_keyword_1 = _PyPegen_expect_token(p, 679)) // token='not' + (_keyword_1 = _PyPegen_expect_token(p, 681)) // token='not' && (a = bitwise_or_rule(p)) // bitwise_or ) @@ -17022,7 +17045,7 @@ for_if_clause_rule(Parser *p) expr_ty b; asdl_expr_seq* c; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' && (_keyword_1 = _PyPegen_expect_token(p, 672)) // token='for' && @@ -17108,7 +17131,7 @@ for_if_clause_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings void *_tmp_121_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 672)) // token='for' && @@ -22561,7 +22584,7 @@ invalid_for_target_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings expr_ty a; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 672)) // token='for' && @@ -22924,7 +22947,7 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 635)) // token='with' && @@ -22962,7 +22985,7 @@ invalid_with_stmt_rule(Parser *p) UNUSED(_opt_var_1); // Silence compiler warnings Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 635)) // token='with' && @@ -23024,7 +23047,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (a = _PyPegen_expect_token(p, 635)) // token='with' && @@ -23067,7 +23090,7 @@ invalid_with_stmt_indent_rule(Parser *p) Token * a; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (a = _PyPegen_expect_token(p, 635)) // token='with' && @@ -24353,7 +24376,7 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 672)) // token='for' && @@ -24394,7 +24417,7 @@ invalid_for_stmt_rule(Parser *p) expr_ty star_expressions_var; expr_ty star_targets_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && (a = _PyPegen_expect_token(p, 672)) // token='for' && @@ -24432,6 +24455,7 @@ invalid_for_stmt_rule(Parser *p) // invalid_def_raw: // | 'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT +// | 'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block static void * invalid_def_raw_rule(Parser *p) { @@ -24465,9 +24489,9 @@ invalid_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_opt_var = _PyPegen_expect_token(p, 674), !p->error_indicator) // 'async'? + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? && - (a = _PyPegen_expect_token(p, 675)) // token='def' + (a = _PyPegen_expect_token(p, 677)) // token='def' && (name_var = _PyPegen_name_token(p)) // NAME && @@ -24501,6 +24525,60 @@ invalid_def_raw_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_def_raw[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); } + { // 'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_def_raw[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + Token * _keyword; + Token * _literal; + Token * _literal_1; + Token * _literal_2; + void *_opt_var; + UNUSED(_opt_var); // Silence compiler warnings + void *_opt_var_1; + UNUSED(_opt_var_1); // Silence compiler warnings + void *_opt_var_2; + UNUSED(_opt_var_2); // Silence compiler warnings + void *_opt_var_3; + UNUSED(_opt_var_3); // Silence compiler warnings + void *_opt_var_4; + UNUSED(_opt_var_4); // Silence compiler warnings + asdl_stmt_seq* block_var; + expr_ty name_var; + if ( + (_opt_var = _PyPegen_expect_token(p, 676), !p->error_indicator) // 'async'? + && + (_keyword = _PyPegen_expect_token(p, 677)) // token='def' + && + (name_var = _PyPegen_name_token(p)) // NAME + && + (_opt_var_1 = type_params_rule(p), !p->error_indicator) // type_params? + && + (_literal = _PyPegen_expect_forced_token(p, 7, "(")) // forced_token='(' + && + (_opt_var_2 = params_rule(p), !p->error_indicator) // params? + && + (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' + && + (_opt_var_3 = _tmp_228_rule(p), !p->error_indicator) // ['->' expression] + && + (_literal_2 = _PyPegen_expect_forced_token(p, 11, ":")) // forced_token=':' + && + (_opt_var_4 = func_type_comment_rule(p), !p->error_indicator) // func_type_comment? + && + (block_var = block_rule(p)) // block + ) + { + D(fprintf(stderr, "%*c+ invalid_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + _res = _PyPegen_dummy_name(p, _opt_var, _keyword, name_var, _opt_var_1, _literal, _opt_var_2, _literal_1, _opt_var_3, _literal_2, _opt_var_4, block_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_def_raw[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'async'? 'def' NAME type_params? &&'(' params? ')' ['->' expression] &&':' func_type_comment? block")); + } _res = NULL; done: p->level--; @@ -24536,13 +24614,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='class' + (_keyword = _PyPegen_expect_token(p, 679)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_228_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -24575,13 +24653,13 @@ invalid_class_def_raw_rule(Parser *p) expr_ty name_var; Token * newline_var; if ( - (a = _PyPegen_expect_token(p, 677)) // token='class' + (a = _PyPegen_expect_token(p, 679)) // token='class' && (name_var = _PyPegen_name_token(p)) // NAME && (_opt_var = type_params_rule(p), !p->error_indicator) // type_params? && - (_opt_var_1 = _tmp_229_rule(p), !p->error_indicator) // ['(' arguments? ')'] + (_opt_var_1 = _tmp_230_rule(p), !p->error_indicator) // ['(' arguments? ')'] && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -24631,11 +24709,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_230_var; + asdl_seq * _gather_231_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_230_var = _gather_230_rule(p)) // ','.double_starred_kvpair+ + (_gather_231_var = _gather_231_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -24643,7 +24721,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_230_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_231_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -24696,7 +24774,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_232_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_233_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -24806,7 +24884,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_233_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_234_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -25051,7 +25129,7 @@ invalid_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_234_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_235_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -25083,7 +25161,7 @@ invalid_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_235_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_236_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -25147,9 +25225,9 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_236_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_237_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_237_rule, p) + _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_238_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -25173,7 +25251,7 @@ invalid_replacement_field_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_replacement_field[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_239_var; + asdl_seq * _loop0_240_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings void *_opt_var_1; @@ -25186,11 +25264,11 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_238_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_239_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_239_var = _loop0_239_rule(p)) // fstring_format_spec* + (_loop0_240_var = _loop0_240_rule(p)) // fstring_format_spec* && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25227,7 +25305,7 @@ invalid_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_240_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_241_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -25274,7 +25352,7 @@ invalid_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_241_rule, p) + _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_242_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -25341,16 +25419,16 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_242_var; + void *_tmp_243_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_242_var = _tmp_242_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_243_var = _tmp_243_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && - (a = _PyPegen_expect_token(p, 679)) // token='not' + (a = _PyPegen_expect_token(p, 681)) // token='not' && (b = inversion_rule(p)) // inversion ) @@ -25393,13 +25471,13 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_243_var; + void *_tmp_244_var; Token * a; expr_ty b; if ( - (_tmp_243_var = _tmp_243_rule(p)) // '+' | '-' | '~' + (_tmp_244_var = _tmp_244_rule(p)) // '+' | '-' | '~' && - (a = _PyPegen_expect_token(p, 679)) // token='not' + (a = _PyPegen_expect_token(p, 681)) // token='not' && (b = factor_rule(p)) // factor ) @@ -25423,6 +25501,52 @@ invalid_factor_rule(Parser *p) return _res; } +// invalid_type_params: '[' ']' +static void * +invalid_type_params_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '[' ']' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_type_params[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'[' ']'")); + Token * _literal; + Token * token; + if ( + (_literal = _PyPegen_expect_token(p, 9)) // token='[' + && + (token = _PyPegen_expect_token(p, 10)) // token=']' + ) + { + D(fprintf(stderr, "%*c+ invalid_type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' ']'")); + _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Type parameter list cannot be empty" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_type_params[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'[' ']'")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // _loop0_1: NEWLINE static asdl_seq * _loop0_1_rule(Parser *p) @@ -25824,7 +25948,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'def'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 675)) // token='def' + (_keyword = _PyPegen_expect_token(p, 677)) // token='def' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'def'")); @@ -25862,7 +25986,7 @@ _tmp_7_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_7[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_7[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -25900,7 +26024,7 @@ _tmp_8_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_8[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'class'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 677)) // token='class' + (_keyword = _PyPegen_expect_token(p, 679)) // token='class' ) { D(fprintf(stderr, "%*c+ _tmp_8[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class'")); @@ -25976,7 +26100,7 @@ _tmp_9_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_9[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_9[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -26033,7 +26157,7 @@ _tmp_10_rule(Parser *p) D(fprintf(stderr, "%*c> _tmp_10[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'")); Token * _keyword; if ( - (_keyword = _PyPegen_expect_token(p, 674)) // token='async' + (_keyword = _PyPegen_expect_token(p, 676)) // token='async' ) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'")); @@ -26238,12 +26362,12 @@ _loop1_14_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_14[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_244_var; + void *_tmp_245_var; while ( - (_tmp_244_var = _tmp_244_rule(p)) // star_targets '=' + (_tmp_245_var = _tmp_245_rule(p)) // star_targets '=' ) { - _res = _tmp_244_var; + _res = _tmp_245_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26807,12 +26931,12 @@ _loop0_24_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_24[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_245_var; + void *_tmp_246_var; while ( - (_tmp_245_var = _tmp_245_rule(p)) // '.' | '...' + (_tmp_246_var = _tmp_246_rule(p)) // '.' | '...' ) { - _res = _tmp_245_var; + _res = _tmp_246_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -26874,12 +26998,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_246_var; + void *_tmp_247_var; while ( - (_tmp_246_var = _tmp_246_rule(p)) // '.' | '...' + (_tmp_247_var = _tmp_247_rule(p)) // '.' | '...' ) { - _res = _tmp_246_var; + _res = _tmp_247_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27272,12 +27396,12 @@ _loop1_32_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_32[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_247_var; + void *_tmp_248_var; while ( - (_tmp_247_var = _tmp_247_rule(p)) // '@' named_expression NEWLINE + (_tmp_248_var = _tmp_248_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_247_var; + _res = _tmp_248_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30402,12 +30526,12 @@ _loop1_82_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' expression)")); - void *_tmp_248_var; + void *_tmp_249_var; while ( - (_tmp_248_var = _tmp_248_rule(p)) // ',' expression + (_tmp_249_var = _tmp_249_rule(p)) // ',' expression ) { - _res = _tmp_248_var; + _res = _tmp_249_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30474,12 +30598,12 @@ _loop1_83_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_249_var; + void *_tmp_250_var; while ( - (_tmp_249_var = _tmp_249_rule(p)) // ',' star_expression + (_tmp_250_var = _tmp_250_rule(p)) // ',' star_expression ) { - _res = _tmp_249_var; + _res = _tmp_250_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30663,12 +30787,12 @@ _loop1_86_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_250_var; + void *_tmp_251_var; while ( - (_tmp_250_var = _tmp_250_rule(p)) // 'or' conjunction + (_tmp_251_var = _tmp_251_rule(p)) // 'or' conjunction ) { - _res = _tmp_250_var; + _res = _tmp_251_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30735,12 +30859,12 @@ _loop1_87_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_251_var; + void *_tmp_252_var; while ( - (_tmp_251_var = _tmp_251_rule(p)) // 'and' inversion + (_tmp_252_var = _tmp_252_rule(p)) // 'and' inversion ) { - _res = _tmp_251_var; + _res = _tmp_252_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30927,7 +31051,7 @@ _loop0_91_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_252_rule(p)) // slice | starred_expression + (elem = _tmp_253_rule(p)) // slice | starred_expression ) { _res = elem; @@ -30992,7 +31116,7 @@ _gather_90_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_252_rule(p)) // slice | starred_expression + (elem = _tmp_253_rule(p)) // slice | starred_expression && (seq = _loop0_91_rule(p)) // _loop0_91 ) @@ -32534,12 +32658,12 @@ _loop1_114_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); - void *_tmp_253_var; + void *_tmp_254_var; while ( - (_tmp_253_var = _tmp_253_rule(p)) // fstring | string + (_tmp_254_var = _tmp_254_rule(p)) // fstring | string ) { - _res = _tmp_253_var; + _res = _tmp_254_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32844,12 +32968,12 @@ _loop0_119_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_254_var; + void *_tmp_255_var; while ( - (_tmp_254_var = _tmp_254_rule(p)) // 'if' disjunction + (_tmp_255_var = _tmp_255_rule(p)) // 'if' disjunction ) { - _res = _tmp_254_var; + _res = _tmp_255_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32911,12 +33035,12 @@ _loop0_120_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_255_var; + void *_tmp_256_var; while ( - (_tmp_255_var = _tmp_255_rule(p)) // 'if' disjunction + (_tmp_256_var = _tmp_256_rule(p)) // 'if' disjunction ) { - _res = _tmp_255_var; + _res = _tmp_256_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32969,20 +33093,20 @@ _tmp_121_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_256_var; + asdl_seq * _loop0_257_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty bitwise_or_var; if ( (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - (_loop0_256_var = _loop0_256_rule(p)) // ((',' bitwise_or))* + (_loop0_257_var = _loop0_257_rule(p)) // ((',' bitwise_or))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_256_var, _opt_var); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_257_var, _opt_var); goto done; } p->mark = _mark; @@ -33087,7 +33211,7 @@ _loop0_124_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -33153,7 +33277,7 @@ _gather_123_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_257_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_258_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && (seq = _loop0_124_rule(p)) // _loop0_124 ) @@ -33714,12 +33838,12 @@ _loop0_134_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_258_var; + void *_tmp_259_var; while ( - (_tmp_258_var = _tmp_258_rule(p)) // ',' star_target + (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target ) { - _res = _tmp_258_var; + _res = _tmp_259_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33898,12 +34022,12 @@ _loop1_137_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_259_var; + void *_tmp_260_var; while ( - (_tmp_259_var = _tmp_259_rule(p)) // ',' star_target + (_tmp_260_var = _tmp_260_rule(p)) // ',' star_target ) { - _res = _tmp_259_var; + _res = _tmp_260_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34629,13 +34753,13 @@ _tmp_150_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_260_var; + void *_tmp_261_var; if ( - (_tmp_260_var = _tmp_260_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_261_var = _tmp_261_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_260_var; + _res = _tmp_261_var; goto done; } p->mark = _mark; @@ -34700,7 +34824,7 @@ _loop0_152_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_261_rule(p)) // starred_expression !'=' + (elem = _tmp_262_rule(p)) // starred_expression !'=' ) { _res = elem; @@ -34765,7 +34889,7 @@ _gather_151_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_261_rule(p)) // starred_expression !'=' + (elem = _tmp_262_rule(p)) // starred_expression !'=' && (seq = _loop0_152_rule(p)) // _loop0_152 ) @@ -35518,12 +35642,12 @@ _loop0_164_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_262_var; + void *_tmp_263_var; while ( - (_tmp_262_var = _tmp_262_rule(p)) // star_targets '=' + (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' ) { - _res = _tmp_262_var; + _res = _tmp_263_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -35585,12 +35709,12 @@ _loop0_165_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_263_var; + void *_tmp_264_var; while ( - (_tmp_263_var = _tmp_263_rule(p)) // star_targets '=' + (_tmp_264_var = _tmp_264_rule(p)) // star_targets '=' ) { - _res = _tmp_263_var; + _res = _tmp_264_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -36560,15 +36684,15 @@ _tmp_180_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_180[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_264_var; + void *_tmp_265_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_264_var = _tmp_264_rule(p)) // ')' | '**' + (_tmp_265_var = _tmp_265_rule(p)) // ')' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_180[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_264_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); goto done; } p->mark = _mark; @@ -37716,15 +37840,15 @@ _tmp_198_rule(Parser *p) } D(fprintf(stderr, "%*c> _tmp_198[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_265_var; + void *_tmp_266_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_265_var = _tmp_265_rule(p)) // ':' | '**' + (_tmp_266_var = _tmp_266_rule(p)) // ':' | '**' ) { D(fprintf(stderr, "%*c+ _tmp_198[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_265_var); + _res = _PyPegen_dummy_name(p, _literal, _tmp_266_var); goto done; } p->mark = _mark; @@ -38220,7 +38344,7 @@ _loop0_207_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_266_rule(p)) // expression ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38285,7 +38409,7 @@ _gather_206_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_266_rule(p)) // expression ['as' star_target] + (elem = _tmp_267_rule(p)) // expression ['as' star_target] && (seq = _loop0_207_rule(p)) // _loop0_207 ) @@ -38337,7 +38461,7 @@ _loop0_209_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_267_rule(p)) // expressions ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38402,7 +38526,7 @@ _gather_208_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_267_rule(p)) // expressions ['as' star_target] + (elem = _tmp_268_rule(p)) // expressions ['as' star_target] && (seq = _loop0_209_rule(p)) // _loop0_209 ) @@ -38454,7 +38578,7 @@ _loop0_211_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -38519,7 +38643,7 @@ _gather_210_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_268_rule(p)) // expression ['as' star_target] + (elem = _tmp_269_rule(p)) // expression ['as' star_target] && (seq = _loop0_211_rule(p)) // _loop0_211 ) @@ -38571,7 +38695,7 @@ _loop0_213_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -38636,7 +38760,7 @@ _gather_212_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_269_rule(p)) // expressions ['as' star_target] + (elem = _tmp_270_rule(p)) // expressions ['as' star_target] && (seq = _loop0_213_rule(p)) // _loop0_213 ) @@ -39056,7 +39180,7 @@ _tmp_220_rule(Parser *p) if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_270_rule(p), !p->error_indicator) // ['as' NAME] + (_opt_var = _tmp_271_rule(p), !p->error_indicator) // ['as' NAME] ) { D(fprintf(stderr, "%*c+ _tmp_220[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); @@ -39376,9 +39500,50 @@ _tmp_227_rule(Parser *p) return _res; } -// _tmp_228: '(' arguments? ')' +// _tmp_228: '->' expression static void * _tmp_228_rule(Parser *p) +{ + if (p->level++ == MAXSTACK) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // '->' expression + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'->' expression")); + Token * _literal; + expr_ty expression_var; + if ( + (_literal = _PyPegen_expect_token(p, 51)) // token='->' + && + (expression_var = expression_rule(p)) // expression + ) + { + D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); + _res = _PyPegen_dummy_name(p, _literal, expression_var); + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'->' expression")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_229: '(' arguments? ')' +static void * +_tmp_229_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39394,7 +39559,7 @@ _tmp_228_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_228[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39407,12 +39572,12 @@ _tmp_228_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_228[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_228[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39421,9 +39586,9 @@ _tmp_228_rule(Parser *p) return _res; } -// _tmp_229: '(' arguments? ')' +// _tmp_230: '(' arguments? ')' static void * -_tmp_229_rule(Parser *p) +_tmp_230_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39439,7 +39604,7 @@ _tmp_229_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_229[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c> _tmp_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); Token * _literal; Token * _literal_1; void *_opt_var; @@ -39452,12 +39617,12 @@ _tmp_229_rule(Parser *p) (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_229[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); + D(fprintf(stderr, "%*c+ _tmp_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = _PyPegen_dummy_name(p, _literal, _opt_var, _literal_1); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_229[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_230[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'(' arguments? ')'")); } _res = NULL; @@ -39466,9 +39631,9 @@ _tmp_229_rule(Parser *p) return _res; } -// _loop0_231: ',' double_starred_kvpair +// _loop0_232: ',' double_starred_kvpair static asdl_seq * -_loop0_231_rule(Parser *p) +_loop0_232_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39493,7 +39658,7 @@ _loop0_231_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -39525,7 +39690,7 @@ _loop0_231_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_231[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_232[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -39542,9 +39707,9 @@ _loop0_231_rule(Parser *p) return _seq; } -// _gather_230: double_starred_kvpair _loop0_231 +// _gather_231: double_starred_kvpair _loop0_232 static asdl_seq * -_gather_230_rule(Parser *p) +_gather_231_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39555,27 +39720,27 @@ _gather_230_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_231 + { // double_starred_kvpair _loop0_232 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_230[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c> _gather_231[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_232")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_231_rule(p)) // _loop0_231 + (seq = _loop0_232_rule(p)) // _loop0_232 ) { - D(fprintf(stderr, "%*c+ _gather_230[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c+ _gather_231[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_232")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_230[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_231")); + D(fprintf(stderr, "%*c%s _gather_231[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_232")); } _res = NULL; done: @@ -39583,9 +39748,9 @@ _gather_230_rule(Parser *p) return _res; } -// _tmp_232: '}' | ',' +// _tmp_233: '}' | ',' static void * -_tmp_232_rule(Parser *p) +_tmp_233_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39601,18 +39766,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39620,18 +39785,18 @@ _tmp_232_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_232[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_232[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_232[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39640,9 +39805,9 @@ _tmp_232_rule(Parser *p) return _res; } -// _tmp_233: '}' | ',' +// _tmp_234: '}' | ',' static void * -_tmp_233_rule(Parser *p) +_tmp_234_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39658,18 +39823,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -39677,18 +39842,18 @@ _tmp_233_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_233[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_233[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_233[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -39697,9 +39862,9 @@ _tmp_233_rule(Parser *p) return _res; } -// _tmp_234: '=' | '!' | ':' | '}' +// _tmp_235: '=' | '!' | ':' | '}' static void * -_tmp_234_rule(Parser *p) +_tmp_235_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39715,18 +39880,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -39734,18 +39899,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39753,18 +39918,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39772,18 +39937,18 @@ _tmp_234_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_234[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_234[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_234[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39792,9 +39957,9 @@ _tmp_234_rule(Parser *p) return _res; } -// _tmp_235: '!' | ':' | '}' +// _tmp_236: '!' | ':' | '}' static void * -_tmp_235_rule(Parser *p) +_tmp_236_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39810,18 +39975,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -39829,18 +39994,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39848,18 +40013,18 @@ _tmp_235_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_235[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_235[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_235[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39868,9 +40033,9 @@ _tmp_235_rule(Parser *p) return _res; } -// _tmp_236: '!' NAME +// _tmp_237: '!' NAME static void * -_tmp_236_rule(Parser *p) +_tmp_237_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39886,7 +40051,7 @@ _tmp_236_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_236[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39895,12 +40060,12 @@ _tmp_236_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_236[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_236[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -39909,9 +40074,9 @@ _tmp_236_rule(Parser *p) return _res; } -// _tmp_237: ':' | '}' +// _tmp_238: ':' | '}' static void * -_tmp_237_rule(Parser *p) +_tmp_238_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39927,18 +40092,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -39946,18 +40111,18 @@ _tmp_237_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_237[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_237[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_237[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -39966,9 +40131,9 @@ _tmp_237_rule(Parser *p) return _res; } -// _tmp_238: '!' NAME +// _tmp_239: '!' NAME static void * -_tmp_238_rule(Parser *p) +_tmp_239_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -39984,7 +40149,7 @@ _tmp_238_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_238[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -39993,12 +40158,12 @@ _tmp_238_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_238[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_239[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_238[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_239[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40007,9 +40172,9 @@ _tmp_238_rule(Parser *p) return _res; } -// _loop0_239: fstring_format_spec +// _loop0_240: fstring_format_spec static asdl_seq * -_loop0_239_rule(Parser *p) +_loop0_240_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40034,7 +40199,7 @@ _loop0_239_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_239[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); + D(fprintf(stderr, "%*c> _loop0_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring_format_spec")); expr_ty fstring_format_spec_var; while ( (fstring_format_spec_var = fstring_format_spec_rule(p)) // fstring_format_spec @@ -40057,7 +40222,7 @@ _loop0_239_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_239[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_240[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring_format_spec")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -40074,9 +40239,9 @@ _loop0_239_rule(Parser *p) return _seq; } -// _tmp_240: '!' NAME +// _tmp_241: '!' NAME static void * -_tmp_240_rule(Parser *p) +_tmp_241_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40092,7 +40257,7 @@ _tmp_240_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_240[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -40101,12 +40266,12 @@ _tmp_240_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_240[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_240[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -40115,9 +40280,9 @@ _tmp_240_rule(Parser *p) return _res; } -// _tmp_241: ':' | '}' +// _tmp_242: ':' | '}' static void * -_tmp_241_rule(Parser *p) +_tmp_242_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40133,18 +40298,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -40152,18 +40317,18 @@ _tmp_241_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_241[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_241[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_241[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -40172,9 +40337,9 @@ _tmp_241_rule(Parser *p) return _res; } -// _tmp_242: '+' | '-' | '*' | '/' | '%' | '//' | '@' +// _tmp_243: '+' | '-' | '*' | '/' | '%' | '//' | '@' static void * -_tmp_242_rule(Parser *p) +_tmp_243_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40190,18 +40355,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -40209,18 +40374,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '*' @@ -40228,18 +40393,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '/' @@ -40247,18 +40412,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } { // '%' @@ -40266,18 +40431,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 24)) // token='%' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } { // '//' @@ -40285,18 +40450,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } { // '@' @@ -40304,18 +40469,18 @@ _tmp_242_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_242[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_242[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_242[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -40324,9 +40489,9 @@ _tmp_242_rule(Parser *p) return _res; } -// _tmp_243: '+' | '-' | '~' +// _tmp_244: '+' | '-' | '~' static void * -_tmp_243_rule(Parser *p) +_tmp_244_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40342,18 +40507,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -40361,18 +40526,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '~' @@ -40380,18 +40545,18 @@ _tmp_243_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_243[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_243[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_243[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; @@ -40400,9 +40565,9 @@ _tmp_243_rule(Parser *p) return _res; } -// _tmp_244: star_targets '=' +// _tmp_245: star_targets '=' static void * -_tmp_244_rule(Parser *p) +_tmp_245_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40418,7 +40583,7 @@ _tmp_244_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_244[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -40427,7 +40592,7 @@ _tmp_244_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_244[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40437,7 +40602,7 @@ _tmp_244_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_244[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -40446,9 +40611,9 @@ _tmp_244_rule(Parser *p) return _res; } -// _tmp_245: '.' | '...' +// _tmp_246: '.' | '...' static void * -_tmp_245_rule(Parser *p) +_tmp_246_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40464,18 +40629,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40483,18 +40648,18 @@ _tmp_245_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_245[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_245[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_245[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40503,9 +40668,9 @@ _tmp_245_rule(Parser *p) return _res; } -// _tmp_246: '.' | '...' +// _tmp_247: '.' | '...' static void * -_tmp_246_rule(Parser *p) +_tmp_247_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40521,18 +40686,18 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -40540,18 +40705,18 @@ _tmp_246_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_246[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_246[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_246[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -40560,9 +40725,9 @@ _tmp_246_rule(Parser *p) return _res; } -// _tmp_247: '@' named_expression NEWLINE +// _tmp_248: '@' named_expression NEWLINE static void * -_tmp_247_rule(Parser *p) +_tmp_248_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40578,7 +40743,7 @@ _tmp_247_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_247[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -40590,7 +40755,7 @@ _tmp_247_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_247[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40600,7 +40765,7 @@ _tmp_247_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_247[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -40609,9 +40774,9 @@ _tmp_247_rule(Parser *p) return _res; } -// _tmp_248: ',' expression +// _tmp_249: ',' expression static void * -_tmp_248_rule(Parser *p) +_tmp_249_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40627,7 +40792,7 @@ _tmp_248_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_248[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty c; if ( @@ -40636,7 +40801,7 @@ _tmp_248_rule(Parser *p) (c = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ _tmp_248[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40646,7 +40811,7 @@ _tmp_248_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_248[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } _res = NULL; @@ -40655,9 +40820,9 @@ _tmp_248_rule(Parser *p) return _res; } -// _tmp_249: ',' star_expression +// _tmp_250: ',' star_expression static void * -_tmp_249_rule(Parser *p) +_tmp_250_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40673,7 +40838,7 @@ _tmp_249_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_249[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -40682,7 +40847,7 @@ _tmp_249_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_249[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40692,7 +40857,7 @@ _tmp_249_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_249[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -40701,9 +40866,9 @@ _tmp_249_rule(Parser *p) return _res; } -// _tmp_250: 'or' conjunction +// _tmp_251: 'or' conjunction static void * -_tmp_250_rule(Parser *p) +_tmp_251_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40719,7 +40884,7 @@ _tmp_250_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_250[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -40728,7 +40893,7 @@ _tmp_250_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_250[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40738,7 +40903,7 @@ _tmp_250_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_250[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -40747,9 +40912,9 @@ _tmp_250_rule(Parser *p) return _res; } -// _tmp_251: 'and' inversion +// _tmp_252: 'and' inversion static void * -_tmp_251_rule(Parser *p) +_tmp_252_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40765,7 +40930,7 @@ _tmp_251_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_251[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -40774,7 +40939,7 @@ _tmp_251_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_251[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40784,7 +40949,7 @@ _tmp_251_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_251[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -40793,9 +40958,9 @@ _tmp_251_rule(Parser *p) return _res; } -// _tmp_252: slice | starred_expression +// _tmp_253: slice | starred_expression static void * -_tmp_252_rule(Parser *p) +_tmp_253_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40811,18 +40976,18 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -40830,18 +40995,18 @@ _tmp_252_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_252[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_252[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_252[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -40850,9 +41015,9 @@ _tmp_252_rule(Parser *p) return _res; } -// _tmp_253: fstring | string +// _tmp_254: fstring | string static void * -_tmp_253_rule(Parser *p) +_tmp_254_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40868,18 +41033,18 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); expr_ty fstring_var; if ( (fstring_var = fstring_rule(p)) // fstring ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); _res = fstring_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); } { // string @@ -40887,18 +41052,18 @@ _tmp_253_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_253[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); expr_ty string_var; if ( (string_var = string_rule(p)) // string ) { - D(fprintf(stderr, "%*c+ _tmp_253[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); _res = string_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_253[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); } _res = NULL; @@ -40907,9 +41072,9 @@ _tmp_253_rule(Parser *p) return _res; } -// _tmp_254: 'if' disjunction +// _tmp_255: 'if' disjunction static void * -_tmp_254_rule(Parser *p) +_tmp_255_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40925,7 +41090,7 @@ _tmp_254_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_254[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40934,7 +41099,7 @@ _tmp_254_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_254[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40944,7 +41109,7 @@ _tmp_254_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_254[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40953,9 +41118,9 @@ _tmp_254_rule(Parser *p) return _res; } -// _tmp_255: 'if' disjunction +// _tmp_256: 'if' disjunction static void * -_tmp_255_rule(Parser *p) +_tmp_256_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -40971,7 +41136,7 @@ _tmp_255_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_255[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -40980,7 +41145,7 @@ _tmp_255_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_255[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_256[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -40990,7 +41155,7 @@ _tmp_255_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_255[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_256[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -40999,9 +41164,9 @@ _tmp_255_rule(Parser *p) return _res; } -// _loop0_256: (',' bitwise_or) +// _loop0_257: (',' bitwise_or) static asdl_seq * -_loop0_256_rule(Parser *p) +_loop0_257_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41026,13 +41191,13 @@ _loop0_256_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_256[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_271_var; + D(fprintf(stderr, "%*c> _loop0_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_272_var; while ( - (_tmp_271_var = _tmp_271_rule(p)) // ',' bitwise_or + (_tmp_272_var = _tmp_272_rule(p)) // ',' bitwise_or ) { - _res = _tmp_271_var; + _res = _tmp_272_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -41049,7 +41214,7 @@ _loop0_256_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_256[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_257[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -41066,9 +41231,9 @@ _loop0_256_rule(Parser *p) return _seq; } -// _tmp_257: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_258: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_257_rule(Parser *p) +_tmp_258_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41084,18 +41249,18 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -41103,20 +41268,20 @@ _tmp_257_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_257[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_272_var; + D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_273_var; if ( - (_tmp_272_var = _tmp_272_rule(p)) // assignment_expression | expression !':=' + (_tmp_273_var = _tmp_273_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_257[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_272_var; + D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_273_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_257[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -41125,9 +41290,9 @@ _tmp_257_rule(Parser *p) return _res; } -// _tmp_258: ',' star_target +// _tmp_259: ',' star_target static void * -_tmp_258_rule(Parser *p) +_tmp_259_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41143,7 +41308,7 @@ _tmp_258_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_258[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41152,7 +41317,7 @@ _tmp_258_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_258[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41162,7 +41327,7 @@ _tmp_258_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_258[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41171,9 +41336,9 @@ _tmp_258_rule(Parser *p) return _res; } -// _tmp_259: ',' star_target +// _tmp_260: ',' star_target static void * -_tmp_259_rule(Parser *p) +_tmp_260_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41189,7 +41354,7 @@ _tmp_259_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_259[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -41198,7 +41363,7 @@ _tmp_259_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_259[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -41208,7 +41373,7 @@ _tmp_259_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_259[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -41217,10 +41382,10 @@ _tmp_259_rule(Parser *p) return _res; } -// _tmp_260: +// _tmp_261: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_260_rule(Parser *p) +_tmp_261_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41236,24 +41401,24 @@ _tmp_260_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_260[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_273_var; + D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_274_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_273_var = _gather_273_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_274_var = _gather_274_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_260[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_273_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_274_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_260[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -41262,9 +41427,9 @@ _tmp_260_rule(Parser *p) return _res; } -// _tmp_261: starred_expression !'=' +// _tmp_262: starred_expression !'=' static void * -_tmp_261_rule(Parser *p) +_tmp_262_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41280,7 +41445,7 @@ _tmp_261_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_261[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression @@ -41288,12 +41453,12 @@ _tmp_261_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_261[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_261[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; @@ -41302,9 +41467,9 @@ _tmp_261_rule(Parser *p) return _res; } -// _tmp_262: star_targets '=' +// _tmp_263: star_targets '=' static void * -_tmp_262_rule(Parser *p) +_tmp_263_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41320,7 +41485,7 @@ _tmp_262_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_262[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41329,12 +41494,12 @@ _tmp_262_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_262[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_262[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41343,9 +41508,9 @@ _tmp_262_rule(Parser *p) return _res; } -// _tmp_263: star_targets '=' +// _tmp_264: star_targets '=' static void * -_tmp_263_rule(Parser *p) +_tmp_264_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41361,7 +41526,7 @@ _tmp_263_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_263[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty star_targets_var; if ( @@ -41370,12 +41535,12 @@ _tmp_263_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_263[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = _PyPegen_dummy_name(p, star_targets_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_263[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -41384,9 +41549,9 @@ _tmp_263_rule(Parser *p) return _res; } -// _tmp_264: ')' | '**' +// _tmp_265: ')' | '**' static void * -_tmp_264_rule(Parser *p) +_tmp_265_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41402,18 +41567,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -41421,18 +41586,18 @@ _tmp_264_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_264[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_264[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_264[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41441,9 +41606,9 @@ _tmp_264_rule(Parser *p) return _res; } -// _tmp_265: ':' | '**' +// _tmp_266: ':' | '**' static void * -_tmp_265_rule(Parser *p) +_tmp_266_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41459,18 +41624,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -41478,18 +41643,18 @@ _tmp_265_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_265[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_265[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_265[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -41498,9 +41663,9 @@ _tmp_265_rule(Parser *p) return _res; } -// _tmp_266: expression ['as' star_target] +// _tmp_267: expression ['as' star_target] static void * -_tmp_266_rule(Parser *p) +_tmp_267_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41516,22 +41681,22 @@ _tmp_266_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_266[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_275_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_266[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_266[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41540,9 +41705,9 @@ _tmp_266_rule(Parser *p) return _res; } -// _tmp_267: expressions ['as' star_target] +// _tmp_268: expressions ['as' star_target] static void * -_tmp_267_rule(Parser *p) +_tmp_268_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41558,22 +41723,22 @@ _tmp_267_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_267[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_276_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_277_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_267[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_267[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41582,9 +41747,9 @@ _tmp_267_rule(Parser *p) return _res; } -// _tmp_268: expression ['as' star_target] +// _tmp_269: expression ['as' star_target] static void * -_tmp_268_rule(Parser *p) +_tmp_269_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41600,22 +41765,22 @@ _tmp_268_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_268[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_277_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_268[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_268[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -41624,9 +41789,9 @@ _tmp_268_rule(Parser *p) return _res; } -// _tmp_269: expressions ['as' star_target] +// _tmp_270: expressions ['as' star_target] static void * -_tmp_269_rule(Parser *p) +_tmp_270_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41642,22 +41807,22 @@ _tmp_269_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_269[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_278_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_279_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_269[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_269[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -41666,9 +41831,9 @@ _tmp_269_rule(Parser *p) return _res; } -// _tmp_270: 'as' NAME +// _tmp_271: 'as' NAME static void * -_tmp_270_rule(Parser *p) +_tmp_271_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41684,7 +41849,7 @@ _tmp_270_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_270[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' NAME")); Token * _keyword; expr_ty name_var; if ( @@ -41693,12 +41858,12 @@ _tmp_270_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_270[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); + D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = _PyPegen_dummy_name(p, _keyword, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_270[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' NAME")); } _res = NULL; @@ -41707,9 +41872,9 @@ _tmp_270_rule(Parser *p) return _res; } -// _tmp_271: ',' bitwise_or +// _tmp_272: ',' bitwise_or static void * -_tmp_271_rule(Parser *p) +_tmp_272_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41725,7 +41890,7 @@ _tmp_271_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_271[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); Token * _literal; expr_ty bitwise_or_var; if ( @@ -41734,12 +41899,12 @@ _tmp_271_rule(Parser *p) (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_271[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_271[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; @@ -41748,9 +41913,9 @@ _tmp_271_rule(Parser *p) return _res; } -// _tmp_272: assignment_expression | expression !':=' +// _tmp_273: assignment_expression | expression !':=' static void * -_tmp_272_rule(Parser *p) +_tmp_273_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41766,18 +41931,18 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -41785,7 +41950,7 @@ _tmp_272_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_272[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -41793,12 +41958,12 @@ _tmp_272_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_272[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_272[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_273[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -41807,9 +41972,9 @@ _tmp_272_rule(Parser *p) return _res; } -// _loop0_274: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_275: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_274_rule(Parser *p) +_loop0_275_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41834,13 +41999,13 @@ _loop0_274_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_279_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -41866,7 +42031,7 @@ _loop0_274_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_274[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_275[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -41883,10 +42048,10 @@ _loop0_274_rule(Parser *p) return _seq; } -// _gather_273: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 +// _gather_274: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 static asdl_seq * -_gather_273_rule(Parser *p) +_gather_274_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41897,27 +42062,27 @@ _gather_273_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_273[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + D(fprintf(stderr, "%*c> _gather_274[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_279_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_280_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_274_rule(p)) // _loop0_274 + (seq = _loop0_275_rule(p)) // _loop0_275 ) { - D(fprintf(stderr, "%*c+ _gather_273[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + D(fprintf(stderr, "%*c+ _gather_274[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_273[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_274")); + D(fprintf(stderr, "%*c%s _gather_274[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_275")); } _res = NULL; done: @@ -41925,9 +42090,9 @@ _gather_273_rule(Parser *p) return _res; } -// _tmp_275: 'as' star_target +// _tmp_276: 'as' star_target static void * -_tmp_275_rule(Parser *p) +_tmp_276_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41943,7 +42108,7 @@ _tmp_275_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_275[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41952,12 +42117,12 @@ _tmp_275_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_275[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_275[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -41966,9 +42131,9 @@ _tmp_275_rule(Parser *p) return _res; } -// _tmp_276: 'as' star_target +// _tmp_277: 'as' star_target static void * -_tmp_276_rule(Parser *p) +_tmp_277_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -41984,7 +42149,7 @@ _tmp_276_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_276[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -41993,12 +42158,12 @@ _tmp_276_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_276[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_276[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42007,9 +42172,9 @@ _tmp_276_rule(Parser *p) return _res; } -// _tmp_277: 'as' star_target +// _tmp_278: 'as' star_target static void * -_tmp_277_rule(Parser *p) +_tmp_278_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42025,7 +42190,7 @@ _tmp_277_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_277[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42034,12 +42199,12 @@ _tmp_277_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_277[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_277[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42048,9 +42213,9 @@ _tmp_277_rule(Parser *p) return _res; } -// _tmp_278: 'as' star_target +// _tmp_279: 'as' star_target static void * -_tmp_278_rule(Parser *p) +_tmp_279_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42066,7 +42231,7 @@ _tmp_278_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_278[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -42075,12 +42240,12 @@ _tmp_278_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_278[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_278[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; @@ -42089,9 +42254,9 @@ _tmp_278_rule(Parser *p) return _res; } -// _tmp_279: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_280: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_279_rule(Parser *p) +_tmp_280_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42107,18 +42272,18 @@ _tmp_279_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -42126,20 +42291,20 @@ _tmp_279_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_279[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_280_var; + D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_281_var; if ( - (_tmp_280_var = _tmp_280_rule(p)) // assignment_expression | expression !':=' + (_tmp_281_var = _tmp_281_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_279[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_280_var; + D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_281_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_279[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -42148,9 +42313,9 @@ _tmp_279_rule(Parser *p) return _res; } -// _tmp_280: assignment_expression | expression !':=' +// _tmp_281: assignment_expression | expression !':=' static void * -_tmp_280_rule(Parser *p) +_tmp_281_rule(Parser *p) { if (p->level++ == MAXSTACK) { _Pypegen_stack_overflow(p); @@ -42166,18 +42331,18 @@ _tmp_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -42185,7 +42350,7 @@ _tmp_280_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_280[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_281[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -42193,12 +42358,12 @@ _tmp_280_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_280[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_281[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_280[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_281[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; From 65f5e586a1239ed1a66d8284773d7b02ce40e480 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 7 May 2024 15:07:32 +0300 Subject: [PATCH 202/220] gh-66410: Do not stringify arguments of Tkinter callback (GH-98592) Callbacks registered in the tkinter module now take arguments as various Python objects (int, float, bytes, tuple), not just str. To restore the previous behavior set tkinter module global wantobject to 1 before creating the Tk object or call the wantobject() method of the Tk object with argument 1. Calling it with argument 2 restores the current default behavior. --- Doc/whatsnew/3.13.rst | 10 ++++++ Lib/idlelib/redirector.py | 1 + Lib/test/test_tcl.py | 35 ++++++++++++------- Lib/tkinter/__init__.py | 7 ++-- ...2-10-24-12-05-19.gh-issue-66410.du4UKW.rst | 7 ++++ Modules/_tkinter.c | 17 +++++---- Modules/clinic/_tkinter.c.h | 8 ++--- 7 files changed, 60 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2022-10-24-12-05-19.gh-issue-66410.du4UKW.rst diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index b084e78889c81f9..daa8cf194204fa0 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1859,6 +1859,16 @@ Changes in the Python API to :c:func:`PyUnstable_Code_GetFirstFree`. (Contributed by Bogdan Romanyuk in :gh:`115781`.) +* Callbacks registered in the :mod:`tkinter` module now take arguments as + various Python objects (``int``, ``float``, ``bytes``, ``tuple``), + not just ``str``. + To restore the previous behavior set :mod:`!tkinter` module global + :data:`!wantobject` to ``1`` before creating the + :class:`!Tk` object or call the :meth:`!wantobject` + method of the :class:`!Tk` object with argument ``1``. + Calling it with argument ``2`` restores the current default behavior. + (Contributed by Serhiy Storchaka in :gh:`66410`.) + Build Changes ============= diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 08728956abd9008..8e2ba68d3815bf0 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -106,6 +106,7 @@ def dispatch(self, operation, *args): to *args to accomplish that. For an example, see colorizer.py. ''' + operation = str(operation) # can be a Tcl_Obj m = self._operations.get(operation) try: if m: diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index ebdb58f91d3d8af..553d54329d7939c 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -482,29 +482,36 @@ def testfunc(arg): return arg self.interp.createcommand('testfunc', testfunc) self.addCleanup(self.interp.tk.deletecommand, 'testfunc') - def check(value, expected=None, *, eq=self.assertEqual): - if expected is None: - expected = value + def check(value, expected1=None, expected2=None, *, eq=self.assertEqual): + expected = value + if self.wantobjects >= 2: + if expected2 is not None: + expected = expected2 + expected_type = type(expected) + else: + if expected1 is not None: + expected = expected1 + expected_type = str nonlocal result result = None r = self.interp.call('testfunc', value) - self.assertIsInstance(result, str) + self.assertIsInstance(result, expected_type) eq(result, expected) - self.assertIsInstance(r, str) + self.assertIsInstance(r, expected_type) eq(r, expected) def float_eq(actual, expected): self.assertAlmostEqual(float(actual), expected, delta=abs(expected) * 1e-10) - check(True, '1') - check(False, '0') + check(True, '1', 1) + check(False, '0', 0) check('string') check('string\xbd') check('string\u20ac') check('string\U0001f4bb') if sys.platform != 'win32': - check('<\udce2\udc82\udcac>', '<\u20ac>') - check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>') + check('<\udce2\udc82\udcac>', '<\u20ac>', '<\u20ac>') + check('<\udced\udca0\udcbd\udced\udcb2\udcbb>', '<\U0001f4bb>', '<\U0001f4bb>') check('') check(b'string', 'string') check(b'string\xe2\x82\xac', 'string\xe2\x82\xac') @@ -526,9 +533,13 @@ def float_eq(actual, expected): check(float('inf'), eq=float_eq) check(-float('inf'), eq=float_eq) # XXX NaN representation can be not parsable by float() - check((), '') - check((1, (2,), (3, 4), '5 6', ()), '1 2 {3 4} {5 6} {}') - check([1, [2,], [3, 4], '5 6', []], '1 2 {3 4} {5 6} {}') + check((), '', '') + check((1, (2,), (3, 4), '5 6', ()), + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) + check([1, [2,], [3, 4], '5 6', []], + '1 2 {3 4} {5 6} {}', + (1, (2,), (3, 4), '5 6', '')) def test_splitlist(self): splitlist = self.interp.tk.splitlist diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index dc6ee9a1b472257..daecf4eb2ea5220 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -40,7 +40,7 @@ from tkinter.constants import * import re -wantobjects = 1 +wantobjects = 2 _debug = False # set to True to print executed Tcl/Tk commands TkVersion = float(_tkinter.TK_VERSION) @@ -1762,7 +1762,10 @@ def getint_event(s): try: e.type = EventType(T) except ValueError: - e.type = T + try: + e.type = EventType(str(T)) # can be int + except ValueError: + e.type = T try: e.widget = self._nametowidget(W) except KeyError: diff --git a/Misc/NEWS.d/next/Library/2022-10-24-12-05-19.gh-issue-66410.du4UKW.rst b/Misc/NEWS.d/next/Library/2022-10-24-12-05-19.gh-issue-66410.du4UKW.rst new file mode 100644 index 000000000000000..044fd1876acd3e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2022-10-24-12-05-19.gh-issue-66410.du4UKW.rst @@ -0,0 +1,7 @@ +Callbacks registered in the :mod:`tkinter` module now take arguments as +various Python objects (``int``, ``float``, ``bytes``, ``tuple``), not just +``str``. To restore the previous behavior set :mod:`!tkinter` module global +:data:`~tkinter.wantobject` to ``1`` before creating the +:class:`~tkinter.Tk` object or call the :meth:`~tkinter.Tk.wantobject` +method of the :class:`!Tk` object with argument ``1``. Calling it with +argument ``2`` restores the current default behavior. diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index fc8af244adf4ed4..c7e271faa4cf34e 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2248,7 +2248,7 @@ _tkinter_tkapp_splitlist(TkappObject *self, PyObject *arg) /* Client data struct */ typedef struct { - PyObject *self; + TkappObject *self; PyObject *func; } PythonCmd_ClientData; @@ -2272,6 +2272,7 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, PyObject *args, *res; int i; Tcl_Obj *obj_res; + int objargs = data->self->wantobjects >= 2; ENTER_PYTHON @@ -2280,7 +2281,8 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, return PythonCmd_Error(interp); for (i = 0; i < (objc - 1); i++) { - PyObject *s = unicodeFromTclObj(objv[i + 1]); + PyObject *s = objargs ? FromObj(data->self, objv[i + 1]) + : unicodeFromTclObj(objv[i + 1]); if (!s) { Py_DECREF(args); return PythonCmd_Error(interp); @@ -2383,7 +2385,8 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); - data->self = Py_NewRef(self); + Py_INCREF(self); + data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { Tcl_Condition cond = NULL; @@ -2897,10 +2900,10 @@ Tkapp_WantObjects(PyObject *self, PyObject *args) { int wantobjects = -1; - if (!PyArg_ParseTuple(args, "|p:wantobjects", &wantobjects)) + if (!PyArg_ParseTuple(args, "|i:wantobjects", &wantobjects)) return NULL; if (wantobjects == -1) - return PyBool_FromLong(((TkappObject*)self)->wantobjects); + return PyLong_FromLong(((TkappObject*)self)->wantobjects); ((TkappObject*)self)->wantobjects = wantobjects; Py_RETURN_NONE; @@ -3086,7 +3089,7 @@ _tkinter.create baseName: str = "" className: str = "Tk" interactive: bool = False - wantobjects: bool = False + wantobjects: int = 0 wantTk: bool = True if false, then Tk_Init() doesn't get called sync: bool = False @@ -3102,7 +3105,7 @@ _tkinter_create_impl(PyObject *module, const char *screenName, const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=e3315607648e6bb4 input=09afef9adea70a19]*/ +/*[clinic end generated code: output=e3315607648e6bb4 input=7e382ba431bed537]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 192c49dba216e25..2b1ac954b4d5701 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -676,7 +676,7 @@ PyDoc_STRVAR(_tkinter__flatten__doc__, PyDoc_STRVAR(_tkinter_create__doc__, "create($module, screenName=None, baseName=\'\', className=\'Tk\',\n" -" interactive=False, wantobjects=False, wantTk=True, sync=False,\n" +" interactive=False, wantobjects=0, wantTk=True, sync=False,\n" " use=None, /)\n" "--\n" "\n" @@ -777,8 +777,8 @@ _tkinter_create(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (nargs < 5) { goto skip_optional; } - wantobjects = PyObject_IsTrue(args[4]); - if (wantobjects < 0) { + wantobjects = PyLong_AsInt(args[4]); + if (wantobjects == -1 && PyErr_Occurred()) { goto exit; } if (nargs < 6) { @@ -888,4 +888,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=86a515890d48a2ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d90c1a9850c63249 input=a9049054013a1b77]*/ From ad3d877a126bc892d1c598cf1357a2c39fd466c7 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 7 May 2024 15:43:18 +0300 Subject: [PATCH 203/220] Remove several unused imports in `_pyrepl` (#118668) --- Lib/_pyrepl/reader.py | 1 - Lib/_pyrepl/simple_interact.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index a7ef988da12a6a2..071dfe54aba8fb3 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -23,7 +23,6 @@ from contextlib import contextmanager from dataclasses import dataclass, field, fields -import re import unicodedata from _colorize import can_colorize, ANSIColors # type: ignore[import-not-found] diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index ce79d0d547ebcea..4bc8368169336a3 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -32,7 +32,6 @@ import code from types import ModuleType -from .console import Event from .readline import _get_reader, multiline_input from .unix_console import _error @@ -82,7 +81,6 @@ def showtraceback(self): def run_multiline_interactive_console( mainmodule: ModuleType | None= None, future_flags: int = 0 ) -> None: - import code import __main__ from .readline import _setup _setup() From 7d90b8aadbd6993ee50a73b7536f769334718423 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 7 May 2024 13:54:56 +0100 Subject: [PATCH 204/220] gh-111201: Allow bracketed paste to work (GH-118700) --- Lib/_pyrepl/commands.py | 10 ++++++++++ Lib/_pyrepl/reader.py | 2 ++ Lib/_pyrepl/unix_console.py | 9 +++++++++ Lib/test/test_pyrepl.py | 40 +++++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 60ceb30d2cd77dc..bb6bebace30ec87 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -462,3 +462,13 @@ class paste_mode(Command): def do(self) -> None: self.reader.paste_mode = not self.reader.paste_mode self.reader.dirty = True + + +class enable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = True + +class disable_bracketed_paste(Command): + def do(self) -> None: + self.reader.paste_mode = False + self.reader.insert("\n") diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 071dfe54aba8fb3..e36f65c176e81f2 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -127,6 +127,8 @@ def make_default_commands() -> dict[CommandName, type[Command]]: (r"\M-9", "digit-arg"), # (r'\M-\n', 'insert-nl'), ("\\\\", "self-insert"), + (r"\x1b[200~", "enable_bracketed_paste"), + (r"\x1b[201~", "disable_bracketed_paste"), ] + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"] + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()] diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index c22b1d5b5bc2904..605318c82ae2eae 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -336,10 +336,13 @@ def prepare(self): except ValueError: pass + self.__enable_bracketed_paste() + def restore(self): """ Restore the console to the default state """ + self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) @@ -525,6 +528,12 @@ def clear(self): self.__posxy = 0, 0 self.screen = [] + def __enable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004h") + + def __disable_bracketed_paste(self) -> None: + os.write(self.output_fd, b"\x1b[?2004l") + def __setup_movement(self): """ Set up the movement functions based on the terminal capabilities. diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index 3df76e02b231dc2..b7ae91b919527a5 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -817,6 +817,46 @@ def test_paste_not_in_paste_mode(self): output = multiline_input(reader) self.assertEqual(output, output_code) + def test_bracketed_paste(self): + """Test that bracketed paste using \x1b[200~ and \x1b[201~ works.""" + # fmt: off + input_code = ( + 'def a():\n' + ' for x in range(10):\n' + '\n' + ' if x%2:\n' + ' print(x)\n' + '\n' + ' else:\n' + ' pass\n' + ) + # fmt: on + + output_code = ( + 'def a():\n' + ' for x in range(10):\n' + '\n' + ' if x%2:\n' + ' print(x)\n' + '\n' + ' else:\n' + ' pass\n' + '\n' + ) + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + class TestReader(TestCase): def assert_screen_equals(self, reader, expected): From a855f824a2f5a310ffa58a973a8fe9feaa2500b3 Mon Sep 17 00:00:00 2001 From: Kerim Kabirov Date: Tue, 7 May 2024 15:28:16 +0200 Subject: [PATCH 205/220] Remove Python 3.5 hardcoded version in the tutorial appendix (GH-117612) --- Doc/tutorial/appendix.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 10eb1432175a84c..b8faf756698097f 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -62,7 +62,7 @@ Executable Python Scripts On BSD'ish Unix systems, Python scripts can be made directly executable, like shell scripts, by putting the line :: - #!/usr/bin/env python3.5 + #!/usr/bin/env python3 (assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning of the script and giving the file an executable mode. The ``#!`` must be the @@ -129,7 +129,7 @@ of your user site-packages directory. Start Python and run this code:: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' + '/home/user/.local/lib/python3.x/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless From 71080b8a0fe5da46fb97659060db76fd95a3fb61 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 7 May 2024 07:16:05 -0700 Subject: [PATCH 206/220] gh-118660: Add second type parameter to (Async)ContextManager (#118681) Co-authored-by: Alex Waygood --- Doc/library/typing.rst | 20 ++++++++++++++-- Lib/test/test_typing.py | 23 +++++++++++++++---- Lib/typing.py | 2 +- ...-05-06-18-13-02.gh-issue-118660.n01Vb7.rst | 4 ++++ 4 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-06-18-13-02.gh-issue-118660.n01Vb7.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 652a3f1f70519c4..f53080e0610cb1f 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3819,10 +3819,15 @@ Aliases to other ABCs in :mod:`collections.abc` Aliases to :mod:`contextlib` ABCs """"""""""""""""""""""""""""""""" -.. class:: ContextManager(Generic[T_co]) +.. class:: ContextManager(Generic[T_co, ExitT_co]) Deprecated alias to :class:`contextlib.AbstractContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__enter__` method. The optional second type parameter, ``ExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__exit__` method. + .. versionadded:: 3.5.4 .. deprecated:: 3.9 @@ -3830,10 +3835,18 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. -.. class:: AsyncContextManager(Generic[T_co]) + .. versionchanged:: 3.13 + Added the optional second type parameter, ``ExitT_co``. + +.. class:: AsyncContextManager(Generic[T_co, AExitT_co]) Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. + The first type parameter, ``T_co``, represents the type returned by + the :meth:`~object.__aenter__` method. The optional second type parameter, ``AExitT_co``, + which defaults to ``bool | None``, represents the type returned by the + :meth:`~object.__aexit__` method. + .. versionadded:: 3.6.2 .. deprecated:: 3.9 @@ -3841,6 +3854,9 @@ Aliases to :mod:`contextlib` ABCs now supports subscripting (``[]``). See :pep:`585` and :ref:`types-genericalias`. + .. versionchanged:: 3.13 + Added the optional second type parameter, ``AExitT_co``. + Deprecation Timeline of Major Features ====================================== diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 012613302d1b532..bd116bb1ab72134 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7511,6 +7511,15 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) + def test_contextmanager_type_params(self): + cm1 = typing.ContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.ContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) + + type gen_cm[T1, T2] = typing.ContextManager[T1, T2] + self.assertEqual(get_args(gen_cm.__value__[int, None]), (int, types.NoneType)) + def test_async_contextmanager(self): class NotACM: pass @@ -7522,11 +7531,17 @@ def manager(): cm = manager() self.assertNotIsInstance(cm, typing.AsyncContextManager) - self.assertEqual(typing.AsyncContextManager[int].__args__, (int,)) + self.assertEqual(typing.AsyncContextManager[int].__args__, (int, bool | None)) with self.assertRaises(TypeError): isinstance(42, typing.AsyncContextManager[int]) with self.assertRaises(TypeError): - typing.AsyncContextManager[int, str] + typing.AsyncContextManager[int, str, float] + + def test_asynccontextmanager_type_params(self): + cm1 = typing.AsyncContextManager[int] + self.assertEqual(get_args(cm1), (int, bool | None)) + cm2 = typing.AsyncContextManager[int, None] + self.assertEqual(get_args(cm2), (int, types.NoneType)) class TypeTests(BaseTestCase): @@ -9953,7 +9968,7 @@ def test_special_attrs(self): typing.ValuesView: 'ValuesView', # Subscribed ABC classes typing.AbstractSet[Any]: 'AbstractSet', - typing.AsyncContextManager[Any]: 'AsyncContextManager', + typing.AsyncContextManager[Any, Any]: 'AsyncContextManager', typing.AsyncGenerator[Any, Any]: 'AsyncGenerator', typing.AsyncIterable[Any]: 'AsyncIterable', typing.AsyncIterator[Any]: 'AsyncIterator', @@ -9963,7 +9978,7 @@ def test_special_attrs(self): typing.ChainMap[Any, Any]: 'ChainMap', typing.Collection[Any]: 'Collection', typing.Container[Any]: 'Container', - typing.ContextManager[Any]: 'ContextManager', + typing.ContextManager[Any, Any]: 'ContextManager', typing.Coroutine[Any, Any, Any]: 'Coroutine', typing.Counter[Any]: 'Counter', typing.DefaultDict[Any, Any]: 'DefaultDict', diff --git a/Lib/typing.py b/Lib/typing.py index e48583673a7be22..8e61f50477bcc21 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -3783,7 +3783,7 @@ def __getattr__(attr): obj = _alias(getattr(re, attr), 1) elif attr in {"ContextManager", "AsyncContextManager"}: import contextlib - obj = _alias(getattr(contextlib, f"Abstract{attr}"), 1, name=attr) + obj = _alias(getattr(contextlib, f"Abstract{attr}"), 2, name=attr, defaults=(bool | None,)) else: raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") globals()[attr] = obj diff --git a/Misc/NEWS.d/next/Library/2024-05-06-18-13-02.gh-issue-118660.n01Vb7.rst b/Misc/NEWS.d/next/Library/2024-05-06-18-13-02.gh-issue-118660.n01Vb7.rst new file mode 100644 index 000000000000000..846a7acb6999b8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-06-18-13-02.gh-issue-118660.n01Vb7.rst @@ -0,0 +1,4 @@ +Add an optional second type parameter to :class:`typing.ContextManager` and +:class:`typing.AsyncContextManager`, representing the return types of +:meth:`~object.__exit__` and :meth:`~object.__aexit__` respectively. +This parameter defaults to ``bool | None``. From e5413ec7831608f6cb4f39d805785cb1d7b67702 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Tue, 7 May 2024 16:31:56 +0200 Subject: [PATCH 207/220] gh-118682: Revert forcing str commands, allow class commands in pyrepl (#118709) --- Lib/_pyrepl/reader.py | 10 +++++++--- Lib/test/test_pyrepl.py | 9 +++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index e36f65c176e81f2..d84c164a05308de 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -568,12 +568,16 @@ def do_cmd(self, cmd: tuple[str, list[str]]) -> None: """`cmd` is a tuple of "event_name" and "event", which in the current implementation is always just the "buffer" which happens to be a list of single-character strings.""" - assert isinstance(cmd[0], str) trace("received command {cmd}", cmd=cmd) - command_type = self.commands.get(cmd[0], commands.invalid_command) - command = command_type(self, *cmd) # type: ignore[arg-type] + if isinstance(cmd[0], str): + command_type = self.commands.get(cmd[0], commands.invalid_command) + elif isinstance(cmd[0], type): + command_type = cmd[0] + else: + return # nothing to do + command = command_type(self, *cmd) # type: ignore[arg-type] command.do() self.after_command(command) diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index b7ae91b919527a5..bb9e36bc69ba4da 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -976,6 +976,15 @@ def test_setpos_fromxy_in_wrapped_line(self): reader.setpos_from_xy(0, 1) self.assertEqual(reader.pos, 9) + def test_up_arrow_after_ctrl_r(self): + events = iter([ + Event(evt='key', data='\x12', raw=bytearray(b'\x12')), + Event(evt='key', data='up', raw=bytearray(b'\x1bOA')), + ]) + + reader, _ = handle_all_events(events) + self.assert_screen_equals(reader, "") + if __name__ == "__main__": unittest.main() From 26bab423fb6e08f9df23c5c8f55e3d019150c454 Mon Sep 17 00:00:00 2001 From: Abhinav Upadhyay Date: Tue, 7 May 2024 20:20:41 +0530 Subject: [PATCH 208/220] bpo-115773: Use the right variable name based on the field we are trying read (#118591) Co-authored-by: Pablo Galindo Salgado --- Modules/_testexternalinspection.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_testexternalinspection.c b/Modules/_testexternalinspection.c index 696c00c237dccf5..2a665affb5e7f81 100644 --- a/Modules/_testexternalinspection.c +++ b/Modules/_testexternalinspection.c @@ -553,12 +553,12 @@ get_stack_trace(PyObject* self, PyObject* args) if (bytes_read == -1) { return NULL; } - off_t thread_state_list_head = local_debug_offsets.runtime_state.interpreters_head; + off_t interpreter_state_list_head = local_debug_offsets.runtime_state.interpreters_head; void* address_of_interpreter_state; bytes_read = read_memory( pid, - (void*)(runtime_start_address + thread_state_list_head), + (void*)(runtime_start_address + interpreter_state_list_head), sizeof(void*), &address_of_interpreter_state); if (bytes_read == -1) { From a94ac566285662b214ca97d74481e07e51ccd7d9 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 7 May 2024 17:01:49 +0100 Subject: [PATCH 209/220] gh-111201: Allow pasted code to contain multiple statements in the REPL (#118712) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/commands.py | 5 ++++- Lib/_pyrepl/reader.py | 1 + Lib/_pyrepl/readline.py | 3 ++- Lib/_pyrepl/simple_interact.py | 7 +++++-- Lib/code.py | 4 ++-- Lib/test/test_pyrepl.py | 22 +++++++++++++++++++--- 6 files changed, 33 insertions(+), 9 deletions(-) diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index bb6bebace30ec87..456cba0769c952e 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -460,6 +460,8 @@ def do(self) -> None: class paste_mode(Command): def do(self) -> None: + if not self.reader.paste_mode: + self.reader.was_paste_mode_activated = True self.reader.paste_mode = not self.reader.paste_mode self.reader.dirty = True @@ -467,8 +469,9 @@ def do(self) -> None: class enable_bracketed_paste(Command): def do(self) -> None: self.reader.paste_mode = True + self.reader.was_paste_mode_activated = True class disable_bracketed_paste(Command): def do(self) -> None: self.reader.paste_mode = False - self.reader.insert("\n") + self.reader.dirty = True diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index d84c164a05308de..d15a150180811d6 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -221,6 +221,7 @@ class Reader: dirty: bool = False finished: bool = False paste_mode: bool = False + was_paste_mode_activated: bool = False commands: dict[str, type[Command]] = field(default_factory=make_default_commands) last_command: type[Command] | None = None syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 37ba98d4c8c87a5..d28a7f3779f3021 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -298,10 +298,11 @@ def multiline_input(self, more_lines, ps1, ps2): reader.more_lines = more_lines reader.ps1 = reader.ps2 = ps1 reader.ps3 = reader.ps4 = ps2 - return reader.readline() + return reader.readline(), reader.was_paste_mode_activated finally: reader.more_lines = saved reader.paste_mode = False + reader.was_paste_mode_activated = False def parse_and_bind(self, string: str) -> None: pass # XXX we don't support parsing GNU-readline-style init files diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 4bc8368169336a3..31b2097a78a2268 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -135,7 +135,7 @@ def more_lines(unicodetext: str) -> bool: ps1 = getattr(sys, "ps1", ">>> ") ps2 = getattr(sys, "ps2", "... ") try: - statement = multiline_input(more_lines, ps1, ps2) + statement, contains_pasted_code = multiline_input(more_lines, ps1, ps2) except EOFError: break @@ -144,7 +144,10 @@ def more_lines(unicodetext: str) -> bool: input_name = f"" linecache._register_code(input_name, statement, "") # type: ignore[attr-defined] - more = console.push(_strip_final_indent(statement), filename=input_name) # type: ignore[call-arg] + symbol = "single" if not contains_pasted_code else "exec" + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol=symbol) # type: ignore[call-arg] + if contains_pasted_code and more: + more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg] assert not more input_n += 1 except KeyboardInterrupt: diff --git a/Lib/code.py b/Lib/code.py index 1ee1ad62ff4506e..9d124563f728c23 100644 --- a/Lib/code.py +++ b/Lib/code.py @@ -281,7 +281,7 @@ def interact(self, banner=None, exitmsg=None): elif exitmsg != '': self.write('%s\n' % exitmsg) - def push(self, line, filename=None): + def push(self, line, filename=None, _symbol="single"): """Push a line to the interpreter. The line should not have a trailing newline; it may have @@ -299,7 +299,7 @@ def push(self, line, filename=None): source = "\n".join(self.buffer) if filename is None: filename = self.filename - more = self.runsource(source, filename) + more = self.runsource(source, filename, symbol=_symbol) if not more: self.resetbuffer() return more diff --git a/Lib/test/test_pyrepl.py b/Lib/test/test_pyrepl.py index bb9e36bc69ba4da..c8990b699b214cb 100644 --- a/Lib/test/test_pyrepl.py +++ b/Lib/test/test_pyrepl.py @@ -23,6 +23,7 @@ from _pyrepl.readline import ReadlineAlikeReader, ReadlineConfig from _pyrepl.simple_interact import _strip_final_indent from _pyrepl.unix_eventqueue import EventQueue +from _pyrepl.simple_interact import InteractiveColoredConsole def more_lines(unicodetext, namespace=None): @@ -830,7 +831,6 @@ def test_bracketed_paste(self): ' else:\n' ' pass\n' ) - # fmt: on output_code = ( 'def a():\n' @@ -841,8 +841,8 @@ def test_bracketed_paste(self): '\n' ' else:\n' ' pass\n' - '\n' ) + # fmt: on paste_start = "\x1b[200~" paste_end = "\x1b[201~" @@ -857,6 +857,22 @@ def test_bracketed_paste(self): output = multiline_input(reader) self.assertEqual(output, output_code) + def test_bracketed_paste_single_line(self): + input_code = "oneline" + + paste_start = "\x1b[200~" + paste_end = "\x1b[201~" + + events = itertools.chain( + code_to_events(paste_start), + code_to_events(input_code), + code_to_events(paste_end), + code_to_events("\n"), + ) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, input_code) + class TestReader(TestCase): def assert_screen_equals(self, reader, expected): @@ -986,5 +1002,5 @@ def test_up_arrow_after_ctrl_r(self): self.assert_screen_equals(reader, "") -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() From b9caa0977c512a5e7966ebfdc64fabdc4f3e4971 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 7 May 2024 17:25:15 +0100 Subject: [PATCH 210/220] gh-118518: Improve perf docs (#118708) --- Doc/howto/perf_profiling.rst | 70 ++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index ed2b76ff4f410c2..06459d1b2229648 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -162,12 +162,12 @@ the :option:`!-X` option takes precedence over the environment variable. Example, using the environment variable:: - $ PYTHONPERFSUPPORT=1 python script.py + $ PYTHONPERFSUPPORT=1 perf record -F 9999 -g -o perf.data python script.py $ perf report -g -i perf.data Example, using the :option:`!-X` option:: - $ python -X perf script.py + $ perf record -F 9999 -g -o perf.data python -X perf script.py $ perf report -g -i perf.data Example, using the :mod:`sys` APIs in file :file:`example.py`: @@ -184,7 +184,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: ...then:: - $ python ./example.py + $ perf record -F 9999 -g -o perf.data python ./example.py $ perf report -g -i perf.data @@ -210,31 +210,57 @@ of ``perf``. How to work without frame pointers ---------------------------------- -If you are working with a Python interpreter that has been compiled without frame pointers -you can still use the ``perf`` profiler but the overhead will be a bit higher because Python -needs to generate unwinding information for every Python function call on the fly. Additionally, -``perf`` will take more time to process the data because it will need to use the DWARF debugging -information to unwind the stack and this is a slow process. +If you are working with a Python interpreter that has been compiled without +frame pointers, you can still use the ``perf`` profiler, but the overhead will be +a bit higher because Python needs to generate unwinding information for every +Python function call on the fly. Additionally, ``perf`` will take more time to +process the data because it will need to use the DWARF debugging information to +unwind the stack and this is a slow process. -To enable this mode, you can use the environment variable :envvar:`PYTHON_PERF_JIT_SUPPORT` or the -:option:`-X perf_jit <-X>` option, which will enable the JIT mode for the ``perf`` profiler. +To enable this mode, you can use the environment variable +:envvar:`PYTHON_PERF_JIT_SUPPORT` or the :option:`-X perf_jit <-X>` option, +which will enable the JIT mode for the ``perf`` profiler. -When using the perf JIT mode, you need an extra step before you can run ``perf report``. You need to -call the ``perf inject`` command to inject the JIT information into the ``perf.data`` file. +.. note:: + + Due to a bug in the ``perf`` tool, only ``perf`` versions higher than v6.8 + will work with the JIT mode. The fix was also backported to the v6.7.2 + version of the tool. + + Note that when checking the version of the ``perf`` tool (which can be done + by running ``perf version``) you must take into account that some distros + add some custom version numbers including a ``-`` character. This means + that ``perf 6.7-3`` is not necessarily ``perf 6.7.3``. + +When using the perf JIT mode, you need an extra step before you can run ``perf +report``. You need to call the ``perf inject`` command to inject the JIT +information into the ``perf.data`` file.:: $ perf record -F 9999 -g --call-graph dwarf -o perf.data python -Xperf_jit my_script.py - $ perf inject -i perf.data --jit - $ perf report -g -i perf.data + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data or using the environment variable:: $ PYTHON_PERF_JIT_SUPPORT=1 perf record -F 9999 -g --call-graph dwarf -o perf.data python my_script.py - $ perf inject -i perf.data --jit - $ perf report -g -i perf.data - -Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take snapshots of the stack of -the process being profiled and save the information in the ``perf.data`` file. By default the size of -the stack dump is 8192 bytes but the user can change the size by passing the size after comma like -``--call-graph dwarf,4096``. The size of the stack dump is important because if the size is too small -``perf`` will not be able to unwind the stack and the output will be incomplete. + $ perf inject -i perf.data --jit --output perf.jit.data + $ perf report -g -i perf.jit.data + +``perf inject --jit`` command will read ``perf.data``, +automatically pick up the perf dump file that Python creates (in +``/tmp/perf-$PID.dump``), and then create ``perf.jit.data`` which merges all the +JIT information together. It should also create a lot of ``jitted-XXXX-N.so`` +files in the current directory which are ELF images for all the JIT trampolines +that were created by Python. + +.. warning:: + Notice that when using ``--call-graph dwarf`` the ``perf`` tool will take + snapshots of the stack of the process being profiled and save the + information in the ``perf.data`` file. By default the size of the stack dump + is 8192 bytes but the user can change the size by passing the size after + comma like ``--call-graph dwarf,4096``. The size of the stack dump is + important because if the size is too small ``perf`` will not be able to + unwind the stack and the output will be incomplete. On the other hand, if + the size is too big, then ``perf`` won't be able to sample the process as + frequently as it would like as the overhead will be higher. From f58833ebf8161aaa33844d63fb51b1575180cfbb Mon Sep 17 00:00:00 2001 From: chrysn Date: Tue, 7 May 2024 20:11:18 +0200 Subject: [PATCH 211/220] gh-118650: Document `Enum._repr_*` reservation exclusion (GH-118698) --- Doc/library/enum.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 6837b45894b3a92..21f41b73086c980 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -861,9 +861,15 @@ Supported ``_sunder_`` names For :class:`Flag` classes the next value chosen will be the next highest power-of-two. +- While ``_sunder_`` names are generally reserved for the further development + of the :class:`Enum` class and can not be used, some are explicitly allowed: + + - ``_repr_*`` (e.g. ``_repr_html_``), as used in `IPython's rich display`_ + .. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` .. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` +.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_``, ``_repr_*`` +.. _`IPython's rich display`: https://ipython.readthedocs.io/en/stable/config/integrating.html#rich-display --------------- From 2c19987db525f61e85dadb40b368c97254adc7cc Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 7 May 2024 15:27:28 -0400 Subject: [PATCH 212/220] gh-118729: Temporarily skip `test_multiprocessing_pool_circular_import` (#118732) The `pool_in_threads.py` test file may crash in free-threaded builds, which can lead to the Tsan test hanging. Skip it for now until we fix the underlying issue. --- Lib/test/test_importlib/test_threaded_import.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 9af1e4d505c66e0..3477112927a0b61 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -17,7 +17,7 @@ from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) -from test.support import script_helper, threading_helper +from test.support import script_helper, threading_helper, requires_gil_enabled threading_helper.requires_working_threading(module=True) @@ -248,6 +248,9 @@ def test_concurrent_futures_circular_import(self): 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) + # gh-118727 and gh-118729: pool_in_threads.py may crash in free-threaded + # builds, which can hang the Tsan test so temporarily skip it for now. + @requires_gil_enabled("gh-118727: test may crash in free-threaded builds") def test_multiprocessing_pool_circular_import(self): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), From 37959e25cbbe1d207c660b5bc9583b9bd1403f1a Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 7 May 2024 20:41:07 +0100 Subject: [PATCH 213/220] gh-118518: Check for perf version and not kernel version in test_perf_profiler (#118640) --- Lib/test/test_perf_profiler.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index 496983f7b49f52e..7c1bbeed168d2e4 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -479,17 +479,23 @@ def compile_trampolines_for_all_functions(): if f"py::foo_fork:{script}" in line or f"py::bar_fork:{script}" in line: self.assertIn(line, child_perf_file_contents) -def _is_kernel_version_at_least(major, minor): + +def _is_perf_vesion_at_least(major, minor): + # The output of perf --version looks like "perf version 6.7-3" but + # it can also be perf version "perf version 5.15.143" try: - with open("/proc/version") as f: - version = f.readline().split()[2] - except FileNotFoundError: + output = subprocess.check_output(["perf", "--version"], text=True) + except (subprocess.CalledProcessError, FileNotFoundError): return False + version = output.split()[2] + version = version.split("-")[0] version = version.split(".") - return int(version[0]) > major or (int(version[0]) == major and int(version[1]) >= minor) + version = tuple(map(int, version)) + return version >= (major, minor) + @unittest.skipUnless(perf_command_works(), "perf command doesn't work") -@unittest.skipUnless(_is_kernel_version_at_least(6, 6), "perf command may not work due to a perf bug") +@unittest.skipUnless(_is_perf_vesion_at_least(6, 6), "perf command may not work due to a perf bug") class TestPerfProfilerWithDwarf(unittest.TestCase, TestPerfProfilerMixin): def run_perf(self, script_dir, script, activate_trampoline=True): if activate_trampoline: From f5488550614858fdc7b442076fe73bdfa3cdd738 Mon Sep 17 00:00:00 2001 From: "T. Wouters" Date: Tue, 7 May 2024 23:00:25 +0200 Subject: [PATCH 214/220] gh-109975: Make a rough editorial pass over What's New (#118711) Make a rough editorial pass over Python 3.13's What's New document. Add the release highlights, remove or merge some duplicated entries, and reorder some of the sections (removals should really go before future deprecations). --- Doc/whatsnew/3.13.rst | 1189 +++++++++++++++++++++-------------------- 1 file changed, 611 insertions(+), 578 deletions(-) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index daa8cf194204fa0..69264a3672bbbf3 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -3,7 +3,7 @@ What's New In Python 3.13 **************************** -:Editor: TBD +:Editor: Thomas Wouters .. Rules for maintenance: @@ -49,6 +49,10 @@ This article explains the new features in Python 3.13, compared to 3.12. For full details, see the :ref:`changelog `. +.. seealso:: + + :pep:`719` -- Python 3.13 Release Schedule + .. note:: Prerelease users should be aware that this document is currently in draft @@ -56,31 +60,34 @@ For full details, see the :ref:`changelog `. so it's worth checking back even after reading earlier versions. -Summary -- Release highlights +Summary -- Release Highlights ============================= .. This section singles out the most important changes in Python 3.13. Brevity is key. -.. PEP-sized items next. - -Important deprecations, removals or restrictions: +Python 3.13 beta is the pre-release of the next version of the Python +programming language, with a mix of changes to the language, the +implementation and the standard library. The biggest changes to the +implementation include a new interactive interpreter, and experimental +support for dropping the Global Interpreter Lock (:pep:`703`) and a +Just-In-Time compiler (:pep:`744`). The library changes contain removal of +deprecated APIs and modules, as well as the usual improvements in +user-friendliness and correctness. -* :ref:`PEP 594 `: The remaining 19 "dead batteries" - have been removed from the standard library: - :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, - :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, - :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, - :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. +.. PEP-sized items next. -* :pep:`602` ("Annual Release Cycle for Python") has been updated: +Interpreter improvements: - * Python 3.9 - 3.12 have one and a half years of full support, - followed by three and a half years of security fixes. - * Python 3.13 and later have two years of full support, - followed by three years of security fixes. +* A greatly improved :ref:`interactive interpreter + ` and + :ref:`improved error messages `. -Interpreter improvements: +* Color support in the new :ref:`interactive interpreter + `, + as well as in :ref:`tracebacks ` + and :ref:`doctest ` output. This can be disabled through the + :envvar:`PYTHON_COLORS` and |NO_COLOR|_ environment variables. * :pep:`744`: A basic :ref:`JIT compiler ` was added. It is currently disabled by default (though we may turn it on later). @@ -96,37 +103,69 @@ New typing features: * :pep:`696`: Type parameters (:data:`typing.TypeVar`, :data:`typing.ParamSpec`, and :data:`typing.TypeVarTuple`) now support defaults. + * :pep:`742`: :data:`typing.TypeIs` was added, providing more intuitive type narrowing behavior. +* :pep:`705`: :data:`typing.ReadOnly` was added, to mark an item of a + :class:`typing.TypedDict` as read-only for type checkers. + Free-threading: * :pep:`703`: CPython 3.13 has experimental support for running with the :term:`global interpreter lock` disabled when built with ``--disable-gil``. See :ref:`Free-threaded CPython ` for more details. +Platform support: + +* :pep:`730`: Apple's iOS is now an officially supported platform. Official + Android support (:pep:`738`) is in the works as well. + +Removed modules: + +* :ref:`PEP 594 `: The remaining 19 "dead batteries" + have been removed from the standard library: + :mod:`!aifc`, :mod:`!audioop`, :mod:`!cgi`, :mod:`!cgitb`, :mod:`!chunk`, + :mod:`!crypt`, :mod:`!imghdr`, :mod:`!mailcap`, :mod:`!msilib`, :mod:`!nis`, + :mod:`!nntplib`, :mod:`!ossaudiodev`, :mod:`!pipes`, :mod:`!sndhdr`, :mod:`!spwd`, + :mod:`!sunau`, :mod:`!telnetlib`, :mod:`!uu` and :mod:`!xdrlib`. + +* Also removed were the :mod:`!tkinter.tix` and :mod:`!lib2to3` modules, and the + ``2to3`` program. + +Release schedule changes: + +* :pep:`602` ("Annual Release Cycle for Python") has been updated: + + * Python 3.9 - 3.12 have one and a half years of full support, + followed by three and a half years of security fixes. + * Python 3.13 and later have two years of full support, + followed by three years of security fixes. + New Features ============ +.. _whatsnew313-better-interactive-interpreter: + A Better Interactive Interpreter -------------------------------- On Unix-like systems like Linux or macOS, Python now uses a new -:term:`interactive` shell. When the user starts the :term:`REPL` -from a tty, and both :mod:`curses` and :mod:`readline` are available, -the interactive shell now supports the following new features: - -* colorized prompts; -* multiline editing with history preservation; -* interactive help browsing using :kbd:`F1` with a separate command - history; -* history browsing using :kbd:`F2` that skips output as well as the - :term:`>>>` and :term:`...` prompts; -* "paste mode" with :kbd:`F3` that makes pasting larger blocks of code - easier (press :kbd:`F3` again to return to the regular prompt); -* ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, - and :kbd:`quit` without the need to use call parentheses after the - command name. +:term:`interactive` shell. When the user starts the :term:`REPL` from an +interactive terminal, and both :mod:`curses` and :mod:`readline` are +available, the interactive shell now supports the following new features: + +* Colorized prompts. +* Multiline editing with history preservation. +* Interactive help browsing using :kbd:`F1` with a separate command + history. +* History browsing using :kbd:`F2` that skips output as well as the + :term:`>>>` and :term:`...` prompts. +* "Paste mode" with :kbd:`F3` that makes pasting larger blocks of code + easier (press :kbd:`F3` again to return to the regular prompt). +* The ability to issue REPL-specific commands like :kbd:`help`, :kbd:`exit`, + and :kbd:`quit` without the need to use call parentheses after the command + name. If the new interactive shell is not desired, it can be disabled via the :envvar:`PYTHON_BASIC_REPL` environment variable. @@ -136,6 +175,8 @@ For more on interactive mode, see :ref:`tut-interac`. (Contributed by Pablo Galindo Salgado, Łukasz Langa, and Lysandros Nikolaou in :gh:`111201` based on code from the PyPy project.) +.. _whatsnew313-improved-error-messages: + Improved Error Messages ----------------------- @@ -227,6 +268,105 @@ Support For Mobile Platforms (PEP written and implementation contributed by Russell Keith-Magee in :gh:`114099`.) +.. _whatsnew313-jit-compiler: + +Experimental JIT Compiler +========================= + +When CPython is configured using the ``--enable-experimental-jit`` option, +a just-in-time compiler is added which may speed up some Python programs. + +The internal architecture is roughly as follows. + +* We start with specialized *Tier 1 bytecode*. + See :ref:`What's new in 3.11 ` for details. + +* When the Tier 1 bytecode gets hot enough, it gets translated + to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + +* The Tier 2 IR uses the same stack-based VM as Tier 1, but the + instruction format is better suited to translation to machine code. + +* We have several optimization passes for Tier 2 IR, which are applied + before it is interpreted or translated to machine code. + +* There is a Tier 2 interpreter, but it is mostly intended for debugging + the earlier stages of the optimization pipeline. + The Tier 2 interpreter can be enabled by configuring Python + with ``--enable-experimental-jit=interpreter``. + +* When the JIT is enabled, the optimized + Tier 2 IR is translated to machine code, which is then executed. + +* The machine code translation process uses a technique called + *copy-and-patch*. It has no runtime dependencies, but there is a new + build-time dependency on LLVM. + +The ``--enable-experimental-jit`` flag has the following optional values: + +* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. + +* ``yes`` (default if the flag is present without optional value) + -- Enable the JIT. To disable the JIT at runtime, + pass the environment variable ``PYTHON_JIT=0``. + +* ``yes-off`` -- Build the JIT but disable it by default. + To enable the JIT at runtime, pass the environment variable + ``PYTHON_JIT=1``. + +* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. + The interpreter can be disabled by running with + ``PYTHON_JIT=0``. + +(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT +or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) + +See :pep:`744` for more details. + +(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. +Tier 2 IR by Mark Shannon and Guido van Rossum. +Tier 2 optimizer by Ken Jin.) + +.. _free-threaded-cpython: + +Free-threaded CPython +===================== + +CPython will run with the :term:`global interpreter lock` (GIL) disabled when +configured using the ``--disable-gil`` option at build time. This is an +experimental feature and therefore isn't used by default. Users need to +either compile their own interpreter, or install one of the experimental +builds that are marked as *free-threaded*. + +Free-threaded execution allows for full utilization of the available +processing power by running threads in parallel on available CPU cores. +While not all software will benefit from this automatically, programs +designed with threading in mind will run faster on multicore hardware. + +Work is still ongoing: expect some bugs and a substantial single-threaded +performance hit. + +The free-threaded build still supports optionally running with the GIL +enabled at runtime using the environment variable :envvar:`PYTHON_GIL` or +the command line option :option:`-X gil`. + +To check if the current interpreter is configured with ``--disable-gil``, +use ``sysconfig.get_config_var("Py_GIL_DISABLED")``. To check if the :term:`GIL` +is actually disabled in the running process, the :func:`!sys._is_gil_enabled` +function can be used. + +C-API extension modules need to be built specifically for the free-threaded +build. Extensions that support running with the :term:`GIL` disabled should +use the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should +use :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support +running with the GIL disabled. Importing C extensions that don't use these +mechanisms will cause the GIL to be enabled, unless the GIL was explicitly +disabled with the :envvar:`PYTHON_GIL` environment variable or the +:option:`-X gil=0` option. + +pip 24.1b1 or newer is required to install packages with C extensions in the +free-threaded build. + Other Language Changes ====================== @@ -500,20 +640,22 @@ dis the ``show_offsets`` parameter. (Contributed by Irit Katriel in :gh:`112137`.) +.. _whatsnew313-doctest: + doctest ------- -* The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped - tests. Add :attr:`doctest.DocTestRunner.skips` and - :attr:`doctest.TestResults.skipped` attributes. - (Contributed by Victor Stinner in :gh:`108794`.) - * Color is added to the output by default. This can be controlled via the new :envvar:`PYTHON_COLORS` environment variable as well as the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. See also :ref:`using-on-controlling-color`. (Contributed by Hugo van Kemenade in :gh:`117225`.) +* The :meth:`doctest.DocTestRunner.run` method now counts the number of skipped + tests. Add :attr:`doctest.DocTestRunner.skips` and + :attr:`doctest.TestResults.skipped` attributes. + (Contributed by Victor Stinner in :gh:`108794`.) + email ----- @@ -1008,109 +1150,245 @@ Optimizations (Contributed by Alex Waygood, Shantanu Jain, Adam Turner, Daniel Hollas and others in :gh:`109653`.) -.. _whatsnew313-jit-compiler: - -Experimental JIT Compiler -========================= +Removed Modules And APIs +======================== -When CPython is configured using the ``--enable-experimental-jit`` option, -a just-in-time compiler is added which may speed up some Python programs. +.. _whatsnew313-pep594: -The internal architecture is roughly as follows. +PEP 594: dead batteries (and other module removals) +--------------------------------------------------- -* We start with specialized *Tier 1 bytecode*. - See :ref:`What's new in 3.11 ` for details. +* :pep:`594` removed 19 modules from the standard library, + deprecated in Python 3.11: -* When the Tier 1 bytecode gets hot enough, it gets translated - to a new, purely internal *Tier 2 IR*, a.k.a. micro-ops ("uops"). + * :mod:`!aifc`. + (Contributed by Victor Stinner in :gh:`104773`.) -* The Tier 2 IR uses the same stack-based VM as Tier 1, but the - instruction format is better suited to translation to machine code. + * :mod:`!audioop`. + (Contributed by Victor Stinner in :gh:`104773`.) -* We have several optimization passes for Tier 2 IR, which are applied - before it is interpreted or translated to machine code. + * :mod:`!chunk`. + (Contributed by Victor Stinner in :gh:`104773`.) -* There is a Tier 2 interpreter, but it is mostly intended for debugging - the earlier stages of the optimization pipeline. - The Tier 2 interpreter can be enabled by configuring Python - with ``--enable-experimental-jit=interpreter``. + * :mod:`!cgi` and :mod:`!cgitb`. -* When the JIT is enabled, the optimized - Tier 2 IR is translated to machine code, which is then executed. + * ``cgi.FieldStorage`` can typically be replaced with + :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, + and the :mod:`email.message` module or :pypi:`multipart` + PyPI project for ``POST`` and ``PUT``. -* The machine code translation process uses an architecture called - *copy-and-patch*. It has no runtime dependencies, but there is a new - build-time dependency on LLVM. + * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` + directly on the desired query string, except for ``multipart/form-data`` + input, which can be handled as described for ``cgi.parse_multipart()``. -The ``--enable-experimental-jit`` flag has the following optional values: + * ``cgi.parse_header()`` can be replaced with the functionality in the + :mod:`email` package, which implements the same MIME RFCs. For example, + with :class:`email.message.EmailMessage`:: -* ``no`` (default) -- Disable the entire Tier 2 and JIT pipeline. + from email.message import EmailMessage + msg = EmailMessage() + msg['content-type'] = 'application/json; charset="utf8"' + main, params = msg.get_content_type(), msg['content-type'].params -* ``yes`` (default if the flag is present without optional value) - -- Enable the JIT. To disable the JIT at runtime, - pass the environment variable ``PYTHON_JIT=0``. + * ``cgi.parse_multipart()`` can be replaced with the functionality in the + :mod:`email` package (e.g. :class:`email.message.EmailMessage` and + :class:`email.message.Message`) which implements the same MIME RFCs, or + with the :pypi:`multipart` PyPI project. -* ``yes-off`` -- Build the JIT but disable it by default. - To enable the JIT at runtime, pass the environment variable - ``PYTHON_JIT=1``. + (Contributed by Victor Stinner in :gh:`104773`.) -* ``interpreter`` -- Enable the Tier 2 interpreter but disable the JIT. - The interpreter can be disabled by running with - ``PYTHON_JIT=0``. + * :mod:`!crypt` module and its private :mod:`!_crypt` extension. + The :mod:`hashlib` module is a potential replacement for certain use cases. + Otherwise, the following PyPI projects can be used: -(On Windows, use ``PCbuild/build.bat --experimental-jit`` to enable the JIT -or ``--experimental-jit-interpreter`` to enable the Tier 2 interpreter.) + * :pypi:`bcrypt`: + Modern password hashing for your software and your servers. + * :pypi:`passlib`: + Comprehensive password hashing framework supporting over 30 schemes. + * :pypi:`argon2-cffi`: + The secure Argon2 password hashing algorithm. + * :pypi:`legacycrypt`: + :mod:`ctypes` wrapper to the POSIX crypt library call and associated functionality. + * :pypi:`crypt_r`: + Fork of the :mod:`!crypt` module, wrapper to the :manpage:`crypt_r(3)` library + call and associated functionality. -See :pep:`744` for more details. + (Contributed by Victor Stinner in :gh:`104773`.) -(JIT by Brandt Bucher, inspired by a paper by Haoran Xu and Fredrik Kjolstad. -Tier 2 IR by Mark Shannon and Guido van Rossum. -Tier 2 optimizer by Ken Jin.) + * :mod:`!imghdr`: use the projects :pypi:`filetype`, + :pypi:`puremagic`, or :pypi:`python-magic` instead. + (Contributed by Victor Stinner in :gh:`104773`.) -.. _free-threaded-cpython: + * :mod:`!mailcap`. + The :mod:`mimetypes` module provides an alternative. + (Contributed by Victor Stinner in :gh:`104773`.) -Free-threaded CPython -===================== + * :mod:`!msilib`. + (Contributed by Zachary Ware in :gh:`104773`.) -CPython will run with the :term:`global interpreter lock` (GIL) disabled when -configured using the ``--disable-gil`` option at build time. This is an -experimental feature and therefore isn't used by default. Users need to -either compile their own interpreter, or install one of the experimental -builds that are marked as *free-threaded*. + * :mod:`!nis`. + (Contributed by Victor Stinner in :gh:`104773`.) -Free-threaded execution allows for full utilization of the available -processing power by running threads in parallel on available CPU cores. -While not all software will benefit from this automatically, programs -designed with threading in mind will run faster on multicore hardware. + * :mod:`!nntplib`: + the :pypi:`nntplib` PyPI project can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) -Work is still ongoing: expect some bugs and a substantial single-threaded -performance hit. + * :mod:`!ossaudiodev`: use the + `pygame project `_ for audio playback. + (Contributed by Victor Stinner in :gh:`104780`.) -The free-threaded build still supports optionally running with GIL enabled at -runtime using the environment variable :envvar:`PYTHON_GIL` or the command line -option :option:`-X gil`. + * :mod:`!pipes`: use the :mod:`subprocess` module instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* Use :func:`!sys._is_gil_enabled` to determine if the :term:`GIL` is enabled. + * :mod:`!sndhdr`: use the projects :pypi:`filetype`, + :pypi:`puremagic`, or :pypi:`python-magic` instead. + (Contributed by Victor Stinner in :gh:`104773`.) -* Use ``sysconfig.get_config_var("Py_GIL_DISABLED")`` to identify CPython - builds configured with ``--disable-gil``. + * :mod:`!spwd`: + the :pypi:`python-pam` project can be used instead. + (Contributed by Victor Stinner in :gh:`104773`.) -C-API extensions need to be built specifically for the free-threaded build. + * :mod:`!sunau`. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3 ` or + :pypi:`Exscript` instead. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!uu`: the :mod:`base64` module is a modern alternative. + (Contributed by Victor Stinner in :gh:`104773`.) + + * :mod:`!xdrlib`. + (Contributed by Victor Stinner in :gh:`104773`.) + +* Remove the ``2to3`` program and the :mod:`!lib2to3` module, + deprecated in Python 3.11. + (Contributed by Victor Stinner in :gh:`104780`.) + +* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The + third-party Tix library which the module wrapped is unmaintained. + (Contributed by Zachary Ware in :gh:`75552`.) + +configparser +------------ + +* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, + deprecated in the docstring since Python 3.2, + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104886`.) + +importlib +--------- + +* Remove deprecated :meth:`~object.__getitem__` access for + :class:`!importlib.metadata.EntryPoint` objects. + (Contributed by Jason R. Coombs in :gh:`113175`.) + +locale +------ + +* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: + use ``locale.setlocale(locale.LC_ALL, "")`` instead. + (Contributed by Victor Stinner in :gh:`104783`.) + +logging +------- + +* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and + ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated + since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` + method, :meth:`!logging.LoggerAdapter.warning` method and + :func:`logging.warning` function. + (Contributed by Victor Stinner in :gh:`105376`.) + +pathlib +------- + +* Remove support for using :class:`pathlib.Path` objects as context managers. + This functionality was deprecated and made a no-op in Python 3.9. + +re +-- + +* Remove undocumented, never working, and deprecated ``re.template`` function + and ``re.TEMPLATE`` flag (and ``re.T`` alias). + (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) + + +turtle +------ + +* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, + deprecated in docs since Python 3.1 + and with a deprecation warning since Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104876`.) + +typing +------ + +* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, + are now removed. The items in those namespaces can be imported directly + from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) + +* Remove support for the keyword-argument method of creating + :class:`typing.TypedDict` types, deprecated in Python 3.11. + (Contributed by Tomas Roun in :gh:`104786`.) + +unittest +-------- + +* Remove the following :mod:`unittest` functions, deprecated in Python 3.11: + + * :func:`!unittest.findTestCases` + * :func:`!unittest.makeSuite` + * :func:`!unittest.getTestCaseNames` + + Use :class:`~unittest.TestLoader` methods instead: + + * :meth:`unittest.TestLoader.loadTestsFromModule` + * :meth:`unittest.TestLoader.loadTestsFromTestCase` + * :meth:`unittest.TestLoader.getTestCaseNames` + + (Contributed by Hugo van Kemenade in :gh:`104835`.) + +* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` + method, deprecated in Python 3.11. + (Contributed by Hugo van Kemenade in :gh:`104992`.) + +urllib +------ + +* Remove *cafile*, *capath* and *cadefault* parameters of the + :func:`urllib.request.urlopen` function, deprecated in Python 3.6: pass + the *context* parameter instead. Use + :meth:`ssl.SSLContext.load_cert_chain` to load specific certificates, or + let :func:`ssl.create_default_context` select the system's trusted CA + certificates for you. + (Contributed by Victor Stinner in :gh:`105382`.) -* Extensions that support running with the :term:`GIL` disabled should use - the :c:data:`Py_mod_gil` slot. Extensions using single-phase init should use - :c:func:`PyUnstable_Module_SetGIL` to indicate whether they support running - with the GIL disabled. Importing C extensions that don't use these mechanisms - will cause the GIL to be enabled unless the GIL was explicitly disabled with - the :envvar:`PYTHON_GIL` environment variable or the :option:`-X gil=0` - option. +webbrowser +---------- + +* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, + deprecated in Python 3.11. + Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. + (Contributed by Hugo van Kemenade in :gh:`104804`.) -* pip 24.1b1 or newer is required to install packages with C extensions in the - free-threaded build. +* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. + Use :attr:`webbrowser.MacOSXOSAScript.name ` + attribute instead. + (Contributed by Nikita Sobolev in :gh:`105546`.) +New Deprecations +================ -Deprecated -========== +* Removed chained :class:`classmethod` descriptors (introduced in + :gh:`63272`). This can no longer be used to wrap other descriptors + such as :class:`property`. The core design of this feature was flawed + and caused a number of downstream problems. To "pass-through" a + :class:`classmethod`, consider using the :attr:`!__wrapped__` + attribute that was added in Python 3.10. (Contributed by Raymond + Hettinger in :gh:`89519`.) * :mod:`array`: :mod:`array`'s ``'u'`` format code, deprecated in docs since Python 3.3, emits :exc:`DeprecationWarning` since 3.13 @@ -1161,6 +1439,16 @@ Deprecated security and functionality bugs. This includes removal of the ``--cgi`` flag to the ``python -m http.server`` command line in 3.15. +* :mod:`mimetypes`: Passing file path instead of URL in :func:`~mimetypes.guess_type` is + :term:`soft deprecated`. Use :func:`~mimetypes.guess_file_type` instead. + (Contributed by Serhiy Storchaka in :gh:`66543`.) + +* :mod:`re`: Passing optional arguments *maxsplit*, *count* and *flags* in module-level + functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional + arguments is now deprecated. In future Python versions these parameters will be + :ref:`keyword-only `. + (Contributed by Serhiy Storchaka in :gh:`56166`.) + * :mod:`pathlib`: :meth:`pathlib.PurePath.is_reserved` is deprecated and scheduled for removal in Python 3.15. Use :func:`os.path.isreserved` to detect reserved @@ -1250,10 +1538,6 @@ Deprecated .. Add deprecations above alphabetically, not here at the end. -* Passing file path instead of URL in :func:`~mimetypes.guess_type` is :term:`soft deprecated`. - Use :func:`~mimetypes.guess_file_type` instead. - (Contributed by Serhiy Storchaka in :gh:`66543`.) - Pending Removal in Python 3.14 ------------------------------ @@ -1361,6 +1645,7 @@ Pending Removal in Python 3.14 :class:`~xml.etree.ElementTree.Element` is deprecated and will raise an exception in Python 3.14. + Pending Removal in Python 3.15 ------------------------------ @@ -1566,428 +1851,92 @@ although there is currently no date scheduled for their removal. use :meth:`~zipimport.zipimporter.exec_module` instead. -Removed -======= +CPython Bytecode Changes +======================== -.. _whatsnew313-pep594: +* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a + yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was + changed to add a bit indicating whether the except-depth is 1, which + is needed to optimize closing of generators. + (Contributed by Irit Katriel in :gh:`111354`.) -PEP 594: dead batteries ------------------------ -* :pep:`594` removed 19 modules from the standard library, - deprecated in Python 3.11: +C API Changes +============= - * :mod:`!aifc`. - (Contributed by Victor Stinner in :gh:`104773`.) +New Features +------------ - * :mod:`!audioop`. - (Contributed by Victor Stinner in :gh:`104773`.) +* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before including + :file:`Python.h` when using ``#`` formats in + :ref:`format codes `. + APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. + (Contributed by Inada Naoki in :gh:`104922`.) - * :mod:`!chunk`. - (Contributed by Victor Stinner in :gh:`104773`.) +* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and + :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` + in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. + It makes these functions compatible with arguments of type + :c:expr:`const char * const *`, :c:expr:`const char **` or + :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C + without an explicit type cast. + This can be overridden with the :c:macro:`PY_CXX_CONST` macro. + (Contributed by Serhiy Storchaka in :gh:`65210`.) - * :mod:`!cgi` and :mod:`!cgitb`. +* Add :c:func:`PyImport_AddModuleRef`: similar to + :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead + of a :term:`borrowed reference`. + (Contributed by Victor Stinner in :gh:`105922`.) - * ``cgi.FieldStorage`` can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, - and the :mod:`email.message` module or :pypi:`multipart` - PyPI project for ``POST`` and ``PUT``. +* Add :c:func:`PyWeakref_GetRef` function: similar to + :c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or + ``NULL`` if the referent is no longer live. + (Contributed by Victor Stinner in :gh:`105927`.) - * ``cgi.parse()`` can be replaced by calling :func:`urllib.parse.parse_qs` - directly on the desired query string, except for ``multipart/form-data`` - input, which can be handled as described for ``cgi.parse_multipart()``. +* Add :c:func:`PyObject_GetOptionalAttr` and + :c:func:`PyObject_GetOptionalAttrString`, variants of + :c:func:`PyObject_GetAttr` and :c:func:`PyObject_GetAttrString` which + don't raise :exc:`AttributeError` if the attribute is not found. + These variants are more convenient and faster if the missing attribute + should not be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106521`.) - * ``cgi.parse_header()`` can be replaced with the functionality in the - :mod:`email` package, which implements the same MIME RFCs. For example, - with :class:`email.message.EmailMessage`:: +* Add :c:func:`PyMapping_GetOptionalItem` and + :c:func:`PyMapping_GetOptionalItemString`: variants of + :c:func:`PyObject_GetItem` and :c:func:`PyMapping_GetItemString` which don't + raise :exc:`KeyError` if the key is not found. + These variants are more convenient and faster if the missing key should not + be treated as a failure. + (Contributed by Serhiy Storchaka in :gh:`106307`.) - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params +* Add fixed variants of functions which silently ignore errors: - * ``cgi.parse_multipart()`` can be replaced with the functionality in the - :mod:`email` package (e.g. :class:`email.message.EmailMessage` and - :class:`email.message.Message`) which implements the same MIME RFCs, or - with the :pypi:`multipart` PyPI project. + - :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. + - :c:func:`PyObject_HasAttrStringWithError` replaces :c:func:`PyObject_HasAttrString`. + - :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. + - :c:func:`PyMapping_HasKeyStringWithError` replaces :c:func:`PyMapping_HasKeyString`. - (Contributed by Victor Stinner in :gh:`104773`.) + New functions return not only ``1`` for true and ``0`` for false, but also + ``-1`` for error. - * :mod:`!crypt` module and its private :mod:`!_crypt` extension. - The :mod:`hashlib` module is a potential replacement for certain use cases. - Otherwise, the following PyPI projects can be used: + (Contributed by Serhiy Storchaka in :gh:`108511`.) - * :pypi:`bcrypt`: - Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. - * :pypi:`argon2-cffi`: - The secure Argon2 password hashing algorithm. - * :pypi:`legacycrypt`: - :mod:`ctypes` wrapper to the POSIX crypt library call and associated functionality. - * :pypi:`crypt_r`: - Fork of the :mod:`!crypt` module, wrapper to the :manpage:`crypt_r(3)` library - call and associated functionality. +* If Python is built in :ref:`debug mode ` or :option:`with + assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and + :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. + (Contributed by Victor Stinner in :gh:`106168`.) - (Contributed by Victor Stinner in :gh:`104773`.) +* Add :c:func:`PyModule_Add` function: similar to + :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but + always steals a reference to the value. + (Contributed by Serhiy Storchaka in :gh:`86493`.) - * :mod:`!imghdr`: use the projects :pypi:`filetype`, - :pypi:`puremagic`, or :pypi:`python-magic` instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!mailcap`. - The :mod:`mimetypes` module provides an alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!msilib`. - (Contributed by Zachary Ware in :gh:`104773`.) - - * :mod:`!nis`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!nntplib`: - the :pypi:`nntplib` PyPI project can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!ossaudiodev`: use the - `pygame project `_ for audio playback. - (Contributed by Victor Stinner in :gh:`104780`.) - - * :mod:`!pipes`: use the :mod:`subprocess` module instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!sndhdr`: use the projects :pypi:`filetype`, - :pypi:`puremagic`, or :pypi:`python-magic` instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!spwd`: - the :pypi:`python-pam` project can be used instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!sunau`. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!telnetlib`, use the projects :pypi:`telnetlib3 ` or - :pypi:`Exscript` instead. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!uu`: the :mod:`base64` module is a modern alternative. - (Contributed by Victor Stinner in :gh:`104773`.) - - * :mod:`!xdrlib`. - (Contributed by Victor Stinner in :gh:`104773`.) - -2to3 ----- - -* Remove the ``2to3`` program and the :mod:`!lib2to3` module, - deprecated in Python 3.11. - (Contributed by Victor Stinner in :gh:`104780`.) - -configparser ------------- - -* Remove the undocumented :class:`!configparser.LegacyInterpolation` class, - deprecated in the docstring since Python 3.2, - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104886`.) - -importlib ---------- - -* Remove deprecated :meth:`~object.__getitem__` access for - :class:`!importlib.metadata.EntryPoint` objects. - (Contributed by Jason R. Coombs in :gh:`113175`.) - -locale ------- - -* Remove ``locale.resetlocale()`` function deprecated in Python 3.11: - use ``locale.setlocale(locale.LC_ALL, "")`` instead. - (Contributed by Victor Stinner in :gh:`104783`.) - -logging -------- - -* :mod:`logging`: Remove undocumented and untested ``Logger.warn()`` and - ``LoggerAdapter.warn()`` methods and ``logging.warn()`` function. Deprecated - since Python 3.3, they were aliases to the :meth:`logging.Logger.warning` - method, :meth:`!logging.LoggerAdapter.warning` method and - :func:`logging.warning` function. - (Contributed by Victor Stinner in :gh:`105376`.) - -pathlib -------- - -* Remove support for using :class:`pathlib.Path` objects as context managers. - This functionality was deprecated and made a no-op in Python 3.9. - -re --- - -* Remove undocumented, never working, and deprecated ``re.template`` function - and ``re.TEMPLATE`` flag (and ``re.T`` alias). - (Contributed by Serhiy Storchaka and Nikita Sobolev in :gh:`105687`.) - -tkinter -------- - -* Remove the :mod:`!tkinter.tix` module, deprecated in Python 3.6. The - third-party Tix library which the module wrapped is unmaintained. - (Contributed by Zachary Ware in :gh:`75552`.) - -turtle ------- - -* Remove the :meth:`!turtle.RawTurtle.settiltangle` method, - deprecated in docs since Python 3.1 - and with a deprecation warning since Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104876`.) - -typing ------- - -* Namespaces ``typing.io`` and ``typing.re``, deprecated in Python 3.8, - are now removed. The items in those namespaces can be imported directly - from :mod:`typing`. (Contributed by Sebastian Rittau in :gh:`92871`.) - -* Remove support for the keyword-argument method of creating - :class:`typing.TypedDict` types, deprecated in Python 3.11. - (Contributed by Tomas Roun in :gh:`104786`.) - -unittest --------- - -* Removed the following :mod:`unittest` functions, deprecated in Python 3.11: - - * :func:`!unittest.findTestCases` - * :func:`!unittest.makeSuite` - * :func:`!unittest.getTestCaseNames` - - Use :class:`~unittest.TestLoader` methods instead: - - * :meth:`unittest.TestLoader.loadTestsFromModule` - * :meth:`unittest.TestLoader.loadTestsFromTestCase` - * :meth:`unittest.TestLoader.getTestCaseNames` - - (Contributed by Hugo van Kemenade in :gh:`104835`.) - -* Remove the untested and undocumented :meth:`!unittest.TestProgram.usageExit` - method, deprecated in Python 3.11. - (Contributed by Hugo van Kemenade in :gh:`104992`.) - -urllib ------- - -* Remove *cafile*, *capath* and *cadefault* parameters of the - :func:`urllib.request.urlopen` function, deprecated in Python 3.6: use the - *context* parameter instead. Please use - :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - (Contributed by Victor Stinner in :gh:`105382`.) - -webbrowser ----------- - -* Remove the untested and undocumented :mod:`webbrowser` :class:`!MacOSX` class, - deprecated in Python 3.11. - Use the :class:`!MacOSXOSAScript` class (introduced in Python 3.2) instead. - (Contributed by Hugo van Kemenade in :gh:`104804`.) - -* Remove deprecated ``webbrowser.MacOSXOSAScript._name`` attribute. - Use :attr:`webbrowser.MacOSXOSAScript.name ` - attribute instead. - (Contributed by Nikita Sobolev in :gh:`105546`.) - -CPython bytecode changes -======================== - -* The oparg of ``YIELD_VALUE`` is now ``1`` if the yield is part of a - yield-from or await, and ``0`` otherwise. The oparg of ``RESUME`` was - changed to add a bit indicating whether the except-depth is 1, which - is needed to optimize closing of generators. - (Contributed by Irit Katriel in :gh:`111354`.) - -Porting to Python 3.13 -====================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - -Changes in the Python API -------------------------- - -* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, - :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, - :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and - :c:func:`PySys_GetObject`, which clear all errors which occurred when calling - them, now report them using :func:`sys.unraisablehook`. - You may replace them with other functions as - recommended in the documentation. - (Contributed by Serhiy Storchaka in :gh:`106672`.) - -* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to - retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or - :exc:`KeyError` on Unix platforms where the password database is empty. - -* The :mod:`threading` module now expects the :mod:`!_thread` module to have - an ``_is_main_interpreter`` attribute. It is a function with no - arguments that return ``True`` if the current interpreter is the - main interpreter. - - Any library or application that provides a custom ``_thread`` module - must provide ``_is_main_interpreter()``, just like the module's - other "private" attributes. - (See :gh:`112826`.) - -* :class:`mailbox.Maildir` now ignores files with a leading dot. - (Contributed by Zackery Spytz in :gh:`65559`.) - -* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both - files and directories if a pattern that ends with "``**``" is given, rather - than directories only. Users may add a trailing slash to match only - directories. - -* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was - changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). - The value of the :attr:`!mode` attribute of the readable file-like object - returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. - (Contributed by Serhiy Storchaka in :gh:`115961`.) - -* :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed - to :c:func:`PyUnstable_Code_GetFirstFree`. - (Contributed by Bogdan Romanyuk in :gh:`115781`.) - -* Callbacks registered in the :mod:`tkinter` module now take arguments as - various Python objects (``int``, ``float``, ``bytes``, ``tuple``), - not just ``str``. - To restore the previous behavior set :mod:`!tkinter` module global - :data:`!wantobject` to ``1`` before creating the - :class:`!Tk` object or call the :meth:`!wantobject` - method of the :class:`!Tk` object with argument ``1``. - Calling it with argument ``2`` restores the current default behavior. - (Contributed by Serhiy Storchaka in :gh:`66410`.) - - -Build Changes -============= - -* The :file:`configure` option :option:`--with-system-libmpdec` now defaults - to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python - 3.15. - -* Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate - the :file:`configure` script. - (Contributed by Christian Heimes in :gh:`89886`.) - -* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. - (Contributed by Erlend Aasland in :gh:`105875`.) - -* Python built with :file:`configure` :option:`--with-trace-refs` (tracing - references) is now ABI compatible with the Python release build and - :ref:`debug build `. - (Contributed by Victor Stinner in :gh:`108634`.) - -* Building CPython now requires a compiler with support for the C11 atomic - library, GCC built-in atomic functions, or MSVC interlocked intrinsics. - -* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, - ``termios``, ``winsound``, - ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, - ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` - C extensions are now built with the - :ref:`limited C API `. - (Contributed by Victor Stinner in :gh:`85283`.) - -* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. - (Contributed by Brett Cannon in :gh:`115192`.) - -* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. - (Contributed by Brett Cannon in :gh:`115192`.) - -* Python now bundles the `mimalloc library `__. - It is licensed under the MIT license, see :ref:`mimalloc license `. - The bundled mimalloc has custom changes, see :gh:`113141` for details. - (Contributed by Dino Viehland in :gh:`109914`.) - - -C API Changes -============= - -New Features ------------- - -* You no longer have to define the ``PY_SSIZE_T_CLEAN`` macro before including - :file:`Python.h` when using ``#`` formats in - :ref:`format codes `. - APIs accepting the format codes always use ``Py_ssize_t`` for ``#`` formats. - (Contributed by Inada Naoki in :gh:`104922`.) - -* The *keywords* parameter of :c:func:`PyArg_ParseTupleAndKeywords` and - :c:func:`PyArg_VaParseTupleAndKeywords` now has type :c:expr:`char * const *` - in C and :c:expr:`const char * const *` in C++, instead of :c:expr:`char **`. - It makes these functions compatible with arguments of type - :c:expr:`const char * const *`, :c:expr:`const char **` or - :c:expr:`char * const *` in C++ and :c:expr:`char * const *` in C - without an explicit type cast. - This can be overridden with the :c:macro:`PY_CXX_CONST` macro. - (Contributed by Serhiy Storchaka in :gh:`65210`.) - -* Add :c:func:`PyImport_AddModuleRef`: similar to - :c:func:`PyImport_AddModule`, but return a :term:`strong reference` instead - of a :term:`borrowed reference`. - (Contributed by Victor Stinner in :gh:`105922`.) - -* Add :c:func:`PyWeakref_GetRef` function: similar to - :c:func:`PyWeakref_GetObject` but returns a :term:`strong reference`, or - ``NULL`` if the referent is no longer live. - (Contributed by Victor Stinner in :gh:`105927`.) - -* Add :c:func:`PyObject_GetOptionalAttr` and - :c:func:`PyObject_GetOptionalAttrString`, variants of - :c:func:`PyObject_GetAttr` and :c:func:`PyObject_GetAttrString` which - don't raise :exc:`AttributeError` if the attribute is not found. - These variants are more convenient and faster if the missing attribute - should not be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106521`.) - -* Add :c:func:`PyMapping_GetOptionalItem` and - :c:func:`PyMapping_GetOptionalItemString`: variants of - :c:func:`PyObject_GetItem` and :c:func:`PyMapping_GetItemString` which don't - raise :exc:`KeyError` if the key is not found. - These variants are more convenient and faster if the missing key should not - be treated as a failure. - (Contributed by Serhiy Storchaka in :gh:`106307`.) - -* Add fixed variants of functions which silently ignore errors: - - - :c:func:`PyObject_HasAttrWithError` replaces :c:func:`PyObject_HasAttr`. - - :c:func:`PyObject_HasAttrStringWithError` replaces :c:func:`PyObject_HasAttrString`. - - :c:func:`PyMapping_HasKeyWithError` replaces :c:func:`PyMapping_HasKey`. - - :c:func:`PyMapping_HasKeyStringWithError` replaces :c:func:`PyMapping_HasKeyString`. - - New functions return not only ``1`` for true and ``0`` for false, but also - ``-1`` for error. - - (Contributed by Serhiy Storchaka in :gh:`108511`.) - -* If Python is built in :ref:`debug mode ` or :option:`with - assertions <--with-assertions>`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyList_SET_ITEM` now check the index argument with an assertion. - (Contributed by Victor Stinner in :gh:`106168`.) - -* Add :c:func:`PyModule_Add` function: similar to - :c:func:`PyModule_AddObjectRef` and :c:func:`PyModule_AddObject` but - always steals a reference to the value. - (Contributed by Serhiy Storchaka in :gh:`86493`.) - -* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` - functions: similar to :c:func:`PyDict_GetItemWithError` but returning a - :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, - these functions return -1 on error and so checking ``PyErr_Occurred()`` is - not needed. - (Contributed by Victor Stinner in :gh:`106004`.) +* Add :c:func:`PyDict_GetItemRef` and :c:func:`PyDict_GetItemStringRef` + functions: similar to :c:func:`PyDict_GetItemWithError` but returning a + :term:`strong reference` instead of a :term:`borrowed reference`. Moreover, + these functions return -1 on error and so checking ``PyErr_Occurred()`` is + not needed. + (Contributed by Victor Stinner in :gh:`106004`.) * Added :c:func:`PyDict_SetDefaultRef`, which is similar to :c:func:`PyDict_SetDefault` but returns a :term:`strong reference` instead of @@ -2128,8 +2077,97 @@ New Features by Pablo Galindo in :gh:`93502`.) +Build Changes +============= + +* The :file:`configure` option :option:`--with-system-libmpdec` now defaults + to ``yes``. The bundled copy of ``libmpdecimal`` will be removed in Python + 3.15. + +* Autoconf 2.71 and aclocal 1.16.4 are now required to regenerate + the :file:`configure` script. + (Contributed by Christian Heimes in :gh:`89886`.) + +* SQLite 3.15.2 or newer is required to build the :mod:`sqlite3` extension module. + (Contributed by Erlend Aasland in :gh:`105875`.) + +* Python built with :file:`configure` :option:`--with-trace-refs` (tracing + references) is now ABI compatible with the Python release build and + :ref:`debug build `. + (Contributed by Victor Stinner in :gh:`108634`.) + +* Building CPython now requires a compiler with support for the C11 atomic + library, GCC built-in atomic functions, or MSVC interlocked intrinsics. + +* The ``errno``, ``fcntl``, ``grp``, ``md5``, ``pwd``, ``resource``, + ``termios``, ``winsound``, + ``_ctypes_test``, ``_multiprocessing.posixshmem``, ``_scproxy``, ``_stat``, + ``_statistics``, ``_testconsole``, ``_testimportmultiple`` and ``_uuid`` + C extensions are now built with the :ref:`limited C API `. + (Contributed by Victor Stinner in :gh:`85283`.) + +* ``wasm32-wasi`` is now a :pep:`11` tier 2 platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* ``wasm32-emscripten`` is no longer a :pep:`11` supported platform. + (Contributed by Brett Cannon in :gh:`115192`.) + +* Python now bundles the `mimalloc library `__. + It is licensed under the MIT license; see :ref:`mimalloc license `. + The bundled mimalloc has custom changes, see :gh:`113141` for details. + (Contributed by Dino Viehland in :gh:`109914`.) + + Porting to Python 3.13 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + +Changes in the Python API +------------------------- + +* An :exc:`OSError` is now raised by :func:`getpass.getuser` for any failure to + retrieve a username, instead of :exc:`ImportError` on non-Unix platforms or + :exc:`KeyError` on Unix platforms where the password database is empty. + +* The :mod:`threading` module now expects the :mod:`!_thread` module to have + an ``_is_main_interpreter`` attribute. It is a function with no + arguments that returns ``True`` if the current interpreter is the + main interpreter. + + Any library or application that provides a custom ``_thread`` module + must provide ``_is_main_interpreter()``, just like the module's + other "private" attributes. + (See :gh:`112826`.) + +* :class:`mailbox.Maildir` now ignores files with a leading dot. + (Contributed by Zackery Spytz in :gh:`65559`.) + +* :meth:`pathlib.Path.glob` and :meth:`~pathlib.Path.rglob` now return both + files and directories if a pattern that ends with "``**``" is given, rather + than directories only. Users may add a trailing slash to match only + directories. + +* The value of the :attr:`!mode` attribute of :class:`gzip.GzipFile` was + changed from integer (``1`` or ``2``) to string (``'rb'`` or ``'wb'``). + The value of the :attr:`!mode` attribute of the readable file-like object + returned by :meth:`zipfile.ZipFile.open` was changed from ``'r'`` to ``'rb'``. + (Contributed by Serhiy Storchaka in :gh:`115961`.) + +* Callbacks registered in the :mod:`tkinter` module now take arguments as + various Python objects (``int``, ``float``, ``bytes``, ``tuple``), + not just ``str``. + To restore the previous behavior set :mod:`!tkinter` module global + :data:`!wantobject` to ``1`` before creating the + :class:`!Tk` object or call the :meth:`!wantobject` + method of the :class:`!Tk` object with argument ``1``. + Calling it with argument ``2`` restores the current default behavior. + (Contributed by Serhiy Storchaka in :gh:`66410`.) + + +Changes in the C API +-------------------- * ``Python.h`` no longer includes the ```` standard header. It was included for the ``finite()`` function which is now provided by the @@ -2184,61 +2222,25 @@ Porting to Python 3.13 } Note that ``Py_TRASHCAN_BEGIN`` has a second argument which - should be the deallocation function it is in. - -Deprecated ----------- - -* Passing optional arguments *maxsplit*, *count* and *flags* in module-level - functions :func:`re.split`, :func:`re.sub` and :func:`re.subn` as positional - arguments is now deprecated. - In future Python versions these parameters will be - :ref:`keyword-only `. - (Contributed by Serhiy Storchaka in :gh:`56166`.) - -* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly - the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and - ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. - (Contributed by Victor Stinner in :gh:`105156`.) - -* Deprecate old Python initialization functions: - - * :c:func:`PySys_ResetWarnOptions`: - clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. - * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. - * :c:func:`Py_GetPath`: get :data:`sys.path` instead. - * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. - * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. - * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. - * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or - :envvar:`PYTHONHOME` environment variable instead. - - Functions scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105145`.) - -* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just - an alias to :c:func:`PyImport_ImportModule` since Python 3.3. - Scheduled for removal in Python 3.15. - (Contributed by Victor Stinner in :gh:`105396`.) + should be the deallocation function it is in. The new macros were + added in Python 3.8 and the old macros were deprecated in Python 3.11. + (Contributed by Irit Katriel in :gh:`105111`.) -* Deprecate the :c:func:`PyWeakref_GetObject` and - :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed - reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it - returns a :term:`strong reference`. The `pythoncapi-compat project - `__ can be used to get - :c:func:`PyWeakref_GetRef` on Python 3.12 and older. - (Contributed by Victor Stinner in :gh:`105927`.) +* Functions :c:func:`PyDict_GetItem`, :c:func:`PyDict_GetItemString`, + :c:func:`PyMapping_HasKey`, :c:func:`PyMapping_HasKeyString`, + :c:func:`PyObject_HasAttr`, :c:func:`PyObject_HasAttrString`, and + :c:func:`PySys_GetObject`, which clear all errors which occurred when calling + them, now report them using :func:`sys.unraisablehook`. + You may replace them with other functions as + recommended in the documentation. + (Contributed by Serhiy Storchaka in :gh:`106672`.) -Removed -------- +* :c:func:`!PyCode_GetFirstFree` is an unstable API now and has been renamed + to :c:func:`PyUnstable_Code_GetFirstFree`. + (Contributed by Bogdan Romanyuk in :gh:`115781`.) -* Removed chained :class:`classmethod` descriptors (introduced in - :gh:`63272`). This can no longer be used to wrap other descriptors - such as :class:`property`. The core design of this feature was flawed - and caused a number of downstream problems. To "pass-through" a - :class:`classmethod`, consider using the :attr:`!__wrapped__` - attribute that was added in Python 3.10. (Contributed by Raymond - Hettinger in :gh:`89519`.) +Removed C APIs +-------------- * Remove many APIs (functions, macros, variables) with names prefixed by ``_Py`` or ``_PY`` (considered as private API). If your project is affected @@ -2318,12 +2320,6 @@ Removed Configuration ` instead (:pep:`587`), added to Python 3.8. (Contributed by Victor Stinner in :gh:`105145`.) -* Remove the old trashcan macros ``Py_TRASHCAN_SAFE_BEGIN`` and - ``Py_TRASHCAN_SAFE_END``. They should be replaced by the new macros - ``Py_TRASHCAN_BEGIN`` and ``Py_TRASHCAN_END``. The new macros were - added in Python 3.8 and the old macros were deprecated in Python 3.11. - (Contributed by Irit Katriel in :gh:`105111`.) - * Remove ``PyEval_ThreadsInitialized()`` function, deprecated in Python 3.9. Since Python 3.7, ``Py_Initialize()`` always creates the GIL: calling ``PyEval_InitThreads()`` does nothing and @@ -2360,11 +2356,48 @@ Removed be used instead. (Contributed by Serhiy Storchaka in :gh:`86493`.) +Deprecated C APIs +----------------- + +* Deprecate the old ``Py_UNICODE`` and ``PY_UNICODE_TYPE`` types: use directly + the :c:type:`wchar_t` type instead. Since Python 3.3, ``Py_UNICODE`` and + ``PY_UNICODE_TYPE`` are just aliases to :c:type:`wchar_t`. + (Contributed by Victor Stinner in :gh:`105156`.) + +* Deprecate old Python initialization functions: + + * :c:func:`PySys_ResetWarnOptions`: + clear :data:`sys.warnoptions` and :data:`!warnings.filters` instead. + * :c:func:`Py_GetExecPrefix`: get :data:`sys.exec_prefix` instead. + * :c:func:`Py_GetPath`: get :data:`sys.path` instead. + * :c:func:`Py_GetPrefix`: get :data:`sys.prefix` instead. + * :c:func:`Py_GetProgramFullPath`: get :data:`sys.executable` instead. + * :c:func:`Py_GetProgramName`: get :data:`sys.executable` instead. + * :c:func:`Py_GetPythonHome`: get :c:member:`PyConfig.home` or + :envvar:`PYTHONHOME` environment variable instead. + + Functions scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105145`.) + +* Deprecate the :c:func:`PyImport_ImportModuleNoBlock` function which is just + an alias to :c:func:`PyImport_ImportModule` since Python 3.3. + Scheduled for removal in Python 3.15. + (Contributed by Victor Stinner in :gh:`105396`.) + +* Deprecate the :c:func:`PyWeakref_GetObject` and + :c:func:`PyWeakref_GET_OBJECT` functions, which return a :term:`borrowed + reference`: use the new :c:func:`PyWeakref_GetRef` function instead, it + returns a :term:`strong reference`. The `pythoncapi-compat project + `__ can be used to get + :c:func:`PyWeakref_GetRef` on Python 3.12 and older. + (Contributed by Victor Stinner in :gh:`105927`.) + Pending Removal in Python 3.14 ------------------------------ * Creating immutable types (:c:macro:`Py_TPFLAGS_IMMUTABLETYPE`) with mutable bases using the C API. + * Functions to configure the Python initialization, deprecated in Python 3.11: * ``PySys_SetArgvEx()``: set :c:member:`PyConfig.argv` instead. From 460546529b0959ce9528dec1c5cd836dcc04ad2c Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 7 May 2024 22:01:18 +0100 Subject: [PATCH 215/220] gh-118734: Fixes Windows build when Use_TIER2 is unspecified (#118735) --- .../next/Build/2024-05-07-21-15-47.gh-issue-118734.--GHiS.rst | 2 ++ PCbuild/_testinternalcapi.vcxproj | 2 +- PCbuild/build.bat | 2 +- PCbuild/pythoncore.vcxproj | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-05-07-21-15-47.gh-issue-118734.--GHiS.rst diff --git a/Misc/NEWS.d/next/Build/2024-05-07-21-15-47.gh-issue-118734.--GHiS.rst b/Misc/NEWS.d/next/Build/2024-05-07-21-15-47.gh-issue-118734.--GHiS.rst new file mode 100644 index 000000000000000..40e8e0615fe7964 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-05-07-21-15-47.gh-issue-118734.--GHiS.rst @@ -0,0 +1,2 @@ +Fixes Windows build when invoked directly (not through the :file:`build.bat` +script) without specifying a value for ``UseTIER2``. diff --git a/PCbuild/_testinternalcapi.vcxproj b/PCbuild/_testinternalcapi.vcxproj index d4cd8ad1a46f24a..87db569423de2a2 100644 --- a/PCbuild/_testinternalcapi.vcxproj +++ b/PCbuild/_testinternalcapi.vcxproj @@ -111,7 +111,7 @@ _Py_JIT;%(PreprocessorDefinitions) - _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 13bd895694f176b..6c76f09a0713120 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -68,7 +68,7 @@ set verbose=/nologo /v:m /clp:summary set kill= set do_pgo= set pgo_job=-m test --pgo -set UseTIER2=0 +set UseTIER2= :CheckOpts if "%~1"=="-h" goto Usage diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index b17e782a21421ed..16fb424b11c6a87 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -105,7 +105,7 @@ _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) _Py_JIT;%(PreprocessorDefinitions) - _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) + _Py_TIER2=$(UseTIER2);%(PreprocessorDefinitions) version.lib;ws2_32.lib;pathcch.lib;bcrypt.lib;%(AdditionalDependencies) From 2a85bed89dc14327afe344a57233a37795fde2b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Wed, 8 May 2024 01:40:38 +0200 Subject: [PATCH 216/220] Revert to free macOS runners (#118744) --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f966591c302351..1b2677ff6fe8894 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -208,7 +208,7 @@ jobs: with: config_hash: ${{ needs.check_source.outputs.config_hash }} # macos-14 is M1, macos-13 is Intel - os-matrix: '["macos-14-xlarge", "macos-13-large"]' + os-matrix: '["macos-14", "macos-13"]' build_macos_free_threading: name: 'macOS (free-threading)' @@ -219,7 +219,7 @@ jobs: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: true # macos-14-large is Intel with 12 cores (most parallelism) - os-matrix: '["macos-14-large"]' + os-matrix: '["macos-14"]' build_ubuntu: name: 'Ubuntu' From 2f0a338be66e276a700f86af51d5ef54a138cbfb Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Tue, 7 May 2024 19:09:09 -0500 Subject: [PATCH 217/220] gh-118610: Centralize power caching in `_pylong.py` (#118611) A new `compute_powers()` function computes all and only the powers of the base the various base-conversion functions need, as efficiently as reasonably possible (turns out that invoking `**`is needed at most once). This typically gives a few % speedup, but the primary point is to simplify the base-conversion functions, which no longer need their own, ad hoc, and less efficient power-caching schemes. Co-authored-by: Serhiy Storchaka --- Lib/_pylong.py | 168 ++++++++++++++++++++++++++----------------- Lib/test/test_int.py | 12 ++++ 2 files changed, 113 insertions(+), 67 deletions(-) diff --git a/Lib/_pylong.py b/Lib/_pylong.py index 30bee6fc9ef54ca..4970eb3fa67b2b9 100644 --- a/Lib/_pylong.py +++ b/Lib/_pylong.py @@ -19,6 +19,86 @@ except ImportError: _decimal = None +# A number of functions have this form, where `w` is a desired number of +# digits in base `base`: +# +# def inner(...w...): +# if w <= LIMIT: +# return something +# lo = w >> 1 +# hi = w - lo +# something involving base**lo, inner(...lo...), j, and inner(...hi...) +# figure out largest w needed +# result = inner(w) +# +# They all had some on-the-fly scheme to cache `base**lo` results for reuse. +# Power is costly. +# +# This routine aims to compute all amd only the needed powers in advance, as +# efficiently as reasonably possible. This isn't trivial, and all the +# on-the-fly methods did needless work in many cases. The driving code above +# changes to: +# +# figure out largest w needed +# mycache = compute_powers(w, base, LIMIT) +# result = inner(w) +# +# and `mycache[lo]` replaces `base**lo` in the inner function. +# +# While this does give minor speedups (a few percent at best), the primary +# intent is to simplify the functions using this, by eliminating the need for +# them to craft their own ad-hoc caching schemes. +def compute_powers(w, base, more_than, show=False): + seen = set() + need = set() + ws = {w} + while ws: + w = ws.pop() # any element is fine to use next + if w in seen or w <= more_than: + continue + seen.add(w) + lo = w >> 1 + # only _need_ lo here; some other path may, or may not, need hi + need.add(lo) + ws.add(lo) + if w & 1: + ws.add(lo + 1) + + d = {} + if not need: + return d + it = iter(sorted(need)) + first = next(it) + if show: + print("pow at", first) + d[first] = base ** first + for this in it: + if this - 1 in d: + if show: + print("* base at", this) + d[this] = d[this - 1] * base # cheap + else: + lo = this >> 1 + hi = this - lo + assert lo in d + if show: + print("square at", this) + # Multiplying a bigint by itself (same object!) is about twice + # as fast in CPython. + sq = d[lo] * d[lo] + if hi != lo: + assert hi == lo + 1 + if show: + print(" and * base") + sq *= base + d[this] = sq + return d + +_unbounded_dec_context = decimal.getcontext().copy() +_unbounded_dec_context.prec = decimal.MAX_PREC +_unbounded_dec_context.Emax = decimal.MAX_EMAX +_unbounded_dec_context.Emin = decimal.MIN_EMIN +_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check def int_to_decimal(n): """Asymptotically fast conversion of an 'int' to Decimal.""" @@ -33,57 +113,32 @@ def int_to_decimal(n): # "clever" recursive way. If we want a string representation, we # apply str to _that_. - D = decimal.Decimal - D2 = D(2) - - BITLIM = 128 - - mem = {} - - def w2pow(w): - """Return D(2)**w and store the result. Also possibly save some - intermediate results. In context, these are likely to be reused - across various levels of the conversion to Decimal.""" - if (result := mem.get(w)) is None: - if w <= BITLIM: - result = D2**w - elif w - 1 in mem: - result = (t := mem[w - 1]) + t - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w2pow(w2) * w2pow(w - w2) - mem[w] = result - return result + from decimal import Decimal as D + BITLIM = 200 + # Don't bother caching the "lo" mask in this; the time to compute it is + # tiny compared to the multiply. def inner(n, w): if w <= BITLIM: return D(n) w2 = w >> 1 hi = n >> w2 - lo = n - (hi << w2) - return inner(lo, w2) + inner(hi, w - w2) * w2pow(w2) - - with decimal.localcontext() as ctx: - ctx.prec = decimal.MAX_PREC - ctx.Emax = decimal.MAX_EMAX - ctx.Emin = decimal.MIN_EMIN - ctx.traps[decimal.Inexact] = 1 + lo = n & ((1 << w2) - 1) + return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2] + with decimal.localcontext(_unbounded_dec_context): + nbits = n.bit_length() + w2pow = compute_powers(nbits, D(2), BITLIM) if n < 0: negate = True n = -n else: negate = False - result = inner(n, n.bit_length()) + result = inner(n, nbits) if negate: result = -result return result - def int_to_decimal_string(n): """Asymptotically fast conversion of an 'int' to a decimal string.""" w = n.bit_length() @@ -97,14 +152,13 @@ def int_to_decimal_string(n): # available. This algorithm is asymptotically worse than the algorithm # using the decimal module, but better than the quadratic time # implementation in longobject.c. + + DIGLIM = 1000 def inner(n, w): - if w <= 1000: + if w <= DIGLIM: return str(n) w2 = w >> 1 - d = pow10_cache.get(w2) - if d is None: - d = pow10_cache[w2] = 5**w2 << w2 # 10**i = (5*2)**i = 5**i * 2**i - hi, lo = divmod(n, d) + hi, lo = divmod(n, pow10[w2]) return inner(hi, w - w2) + inner(lo, w2).zfill(w2) # The estimation of the number of decimal digits. @@ -115,7 +169,9 @@ def inner(n, w): # only if the number has way more than 10**15 digits, that exceeds # the 52-bit physical address limit in both Intel64 and AMD64. w = int(w * 0.3010299956639812 + 1) # log10(2) - pow10_cache = {} + pow10 = compute_powers(w, 5, DIGLIM) + for k, v in pow10.items(): + pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k if n < 0: n = -n sign = '-' @@ -128,7 +184,6 @@ def inner(n, w): s = s.lstrip('0') return sign + s - def _str_to_int_inner(s): """Asymptotically fast conversion of a 'str' to an 'int'.""" @@ -144,35 +199,15 @@ def _str_to_int_inner(s): DIGLIM = 2048 - mem = {} - - def w5pow(w): - """Return 5**w and store the result. - Also possibly save some intermediate results. In context, these - are likely to be reused across various levels of the conversion - to 'int'. - """ - if (result := mem.get(w)) is None: - if w <= DIGLIM: - result = 5**w - elif w - 1 in mem: - result = mem[w - 1] * 5 - else: - w2 = w >> 1 - # If w happens to be odd, w-w2 is one larger then w2 - # now. Recurse on the smaller first (w2), so that it's - # in the cache and the larger (w-w2) can be handled by - # the cheaper `w-1 in mem` branch instead. - result = w5pow(w2) * w5pow(w - w2) - mem[w] = result - return result - def inner(a, b): if b - a <= DIGLIM: return int(s[a:b]) mid = (a + b + 1) >> 1 - return inner(mid, b) + ((inner(a, mid) * w5pow(b - mid)) << (b - mid)) + return (inner(mid, b) + + ((inner(a, mid) * w5pow[b - mid]) + << (b - mid))) + w5pow = compute_powers(len(s), 5, DIGLIM) return inner(0, len(s)) @@ -186,7 +221,6 @@ def int_from_string(s): s = s.rstrip().replace('_', '') return _str_to_int_inner(s) - def str_to_int(s): """Asymptotically fast version of decimal string to 'int' conversion.""" # FIXME: this doesn't support the full syntax that int() supports. diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index c8626398b35b896..8959ffb6dcc236f 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -906,6 +906,18 @@ def test_pylong_misbehavior_error_path_from_str( with self.assertRaises(RuntimeError): int(big_value) + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertFalse(sn.startswith('0')) + self.assertEqual(n, int(sn)) + bits <<= 1 if __name__ == "__main__": unittest.main() From e7aec8713f84e90b20c9baae6b7a91d87f327ff0 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 7 May 2024 17:48:05 -0700 Subject: [PATCH 218/220] gh-118746: Fix crash in frame_getlocals and _PyFrame_GetLocals (#118748) We don't know how to create an unoptimized frame with f_locals == NULL, but they are seen in the wild, and this fixes the crash. --- Objects/frameobject.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 26a04cbeea90bf7..d7fcb1925d286c7 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -742,6 +742,15 @@ frame_getlocals(PyFrameObject *f, void *closure) PyCodeObject *co = _PyFrame_GetCode(f->f_frame); if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(f->f_frame)) { + if (f->f_frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + f->f_frame->f_locals = PyDict_New(); + if (f->f_frame->f_locals == NULL) { + return NULL; + } + } return Py_NewRef(f->f_frame->f_locals); } @@ -1937,6 +1946,15 @@ _PyFrame_GetLocals(_PyInterpreterFrame *frame) PyCodeObject *co = _PyFrame_GetCode(frame); if (!(co->co_flags & CO_OPTIMIZED) && !_PyFrame_HasHiddenLocals(frame)) { + if (frame->f_locals == NULL) { + // We found cases when f_locals is NULL for non-optimized code. + // We fill the f_locals with an empty dict to avoid crash until + // we find the root cause. + frame->f_locals = PyDict_New(); + if (frame->f_locals == NULL) { + return NULL; + } + } return Py_NewRef(frame->f_locals); } From fcf52d7ceea3532f0b3475aadd4e1f72db463a1f Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Wed, 8 May 2024 08:59:17 +0300 Subject: [PATCH 219/220] Regen ``Doc/requirements-oldest-sphinx.txt`` (#118736) Regen dependencies --- Doc/requirements-oldest-sphinx.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index 9a5f4f3676e0893..fe45c91501a56a7 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -13,16 +13,16 @@ python-docs-theme>=2022.1 # Sphinx 6.2.1 comes from ``needs_sphinx = '6.2.1'`` in ``Doc/conf.py``. alabaster==0.7.16 -Babel==2.14.0 +Babel==2.15.0 certifi==2024.2.2 charset-normalizer==3.3.2 docutils==0.19 idna==3.7 imagesize==1.4.1 -Jinja2==3.1.3 +Jinja2==3.1.4 MarkupSafe==2.1.5 packaging==24.0 -Pygments==2.17.2 +Pygments==2.18.0 requests==2.31.0 snowballstemmer==2.2.0 Sphinx==6.2.1 From c4f9823be277b2e3de2315526612276626217743 Mon Sep 17 00:00:00 2001 From: trag1c Date: Wed, 8 May 2024 09:06:38 +0200 Subject: [PATCH 220/220] gh-118671: Updated dead ActiveState links (#118730) Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Kirill Podoprigora --- Doc/howto/urllib2.rst | 2 +- Doc/library/collections.rst | 2 +- Doc/library/difflib.rst | 2 +- Doc/library/math.rst | 2 +- Doc/library/random.rst | 2 +- Doc/library/shelve.rst | 2 +- Doc/library/stdtypes.rst | 2 +- Doc/library/sys.rst | 2 +- Doc/whatsnew/3.2.rst | 4 ++-- Lib/test/test_math.py | 2 +- Modules/mathmodule.c | 2 +- Tools/peg_generator/pegen/sccutils.py | 4 ++-- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 7f54a410881514e..33a2a7ea89ea07a 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -594,5 +594,5 @@ This document was reviewed and revised by John Lee. scripts with a localhost server, I have to prevent urllib from using the proxy. .. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. + `_. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index f868799e7f5c105..19b89ad79c74140 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -134,7 +134,7 @@ The class can be used to simulate nested scopes and is useful in templating. :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe - `_ has options to control + `_ has options to control whether writes and other mutations apply only to the first mapping or to any mapping in the chain. diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index d45e46448207a47..6a34f2e4d7f5e17 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -631,7 +631,7 @@ If you want to know how to change the first sequence into the second, use work. * `Simple version control recipe - `_ for a small application + `_ for a small application built with :class:`SequenceMatcher`. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 1475d26486de5fb..ab9236cc7cfb81f 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -134,7 +134,7 @@ Number-theoretic and representation functions For further discussion and two alternative approaches, see the `ASPN cookbook recipes for accurate floating point summation - `_\. + `_\. .. function:: gcd(*integers) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 4584bbc40aa07bc..22a799c706b7e2c 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -55,7 +55,7 @@ from sources provided by the operating system. `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative + `_ for a compatible alternative random number generator with a long period and comparatively simple update operations. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 95c549918870221..1fa614f6584170a 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -86,7 +86,7 @@ Two additional methods are supported: .. seealso:: - `Persistent dictionary recipe `_ + `Persistent dictionary recipe `_ with widely supported storage formats and having the speed of native dictionaries. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 6c13bd015d5691f..8f5f9ee519a5739 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1496,7 +1496,7 @@ objects that compare equal might have different :attr:`~range.start`, .. seealso:: - * The `linspace recipe `_ + * The `linspace recipe `_ shows how to implement a lazy version of range suitable for floating point applications. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 91afa9d58e87e6d..19a396d2300011f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -877,7 +877,7 @@ always available. additional garbage collector overhead if the object is managed by the garbage collector. - See `recursive sizeof recipe `_ + See `recursive sizeof recipe `_ for an example of using :func:`getsizeof` recursively to find the size of containers and all their contents. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 52474517f5facc9..a6b38207b70c933 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -785,8 +785,8 @@ functools (Contributed by Raymond Hettinger and incorporating design ideas from Jim Baker, Miki Tebeka, and Nick Coghlan; see `recipe 498245 - `_\, `recipe 577479 - `_\, :issue:`10586`, and + `_\, `recipe 577479 + `_\, :issue:`10586`, and :issue:`10593`.) * The :func:`functools.wraps` decorator now adds a :attr:`__wrapped__` attribute diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 0e4dbc0b64a4390..b68c442013c09f0 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -657,7 +657,7 @@ def testFsum(self): def msum(iterable): """Full precision summation. Compute sum(iterable) without any intermediate accumulation of error. Based on the 'lsum' function - at http://code.activestate.com/recipes/393090/ + at https://code.activestate.com/recipes/393090-binary-floating-point-summation-accurate-to-full-p/ """ tmant, texp = 0, 0 diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index a3cbfc383761a02..a79694730a8e4e8 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1253,7 +1253,7 @@ FUNC1(tanh, tanh, 0, "Return the hyperbolic tangent of x.") /* Precision summation function as msum() by Raymond Hettinger in - , + , enhanced with the exact partials sum and roundoff from Mark Dickinson's post at . See those links for more details, proofs and other references. diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index 1f0586bb2f7d6db..da4c9331625dd93 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -18,7 +18,7 @@ def strongly_connected_components( exactly once; vertices not part of a SCC are returned as singleton sets. - From http://code.activestate.com/recipes/578507/. + From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ identified: Set[str] = set() stack: List[str] = [] @@ -81,7 +81,7 @@ def topsort( {B, C} {A} - From http://code.activestate.com/recipes/577413/. + From https://code.activestate.com/recipes/577413-topological-sort/history/1/. """ # TODO: Use a faster algorithm? for k, v in data.items():