diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index ce8967337b02f9..efbdcd402cdf67 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -9,8 +9,8 @@ ENV WASMTIME_HOME=/opt/wasmtime ENV WASMTIME_VERSION=7.0.0 ENV WASMTIME_CPU_ARCH=x86_64 -RUN dnf -y --nodocs install git clang xz python3-blurb dnf-plugins-core && \ - dnf -y --nodocs builddep python3 && \ +RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ + dnf -y --nodocs --setopt=install_weak_deps=False builddep python3 && \ dnf -y clean all RUN mkdir ${WASI_SDK_PATH} && \ diff --git a/.gitattributes b/.gitattributes index 13289182400109..4ed95069442f3d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -32,6 +32,10 @@ Lib/test/test_importlib/resources/data01/* noeol Lib/test/test_importlib/resources/namespacedata01/* noeol Lib/test/xmltestdata/* noeol +# Shell scripts should have LF even on Windows because of Cygwin +Lib/venv/scripts/common/activate text eol=lf +Lib/venv/scripts/posix/* text eol=lf + # CRLF files [attr]dos text eol=crlf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fc1bb3388976d5..9149b38d87601c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,7 +5,7 @@ # https://git-scm.com/docs/gitignore#_pattern_format # GitHub -.github/** @ezio-melotti +.github/** @ezio-melotti @hugovk # Build system configure* @erlend-aasland @corona10 @@ -61,11 +61,7 @@ Python/traceback.c @iritkatriel /Tools/build/parse_html5_entities.py @ezio-melotti # Import (including importlib). -# Ignoring importlib.h so as to not get flagged on -# all pull requests that change the emitted -# bytecode. -**/*import*.c @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw -**/*import*.py @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw +**/*import* @brettcannon @encukou @ericsnowcurrently @ncoghlan @warsaw **/*importlib/resources/* @jaraco @warsaw @FFY00 **/importlib/metadata/* @jaraco @warsaw diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4e5328282f1224..df0f107a541614 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,6 +33,7 @@ jobs: check_source: name: 'Check for source changes' runs-on: ubuntu-latest + timeout-minutes: 10 outputs: run_tests: ${{ steps.check.outputs.run_tests }} steps: @@ -63,6 +64,7 @@ jobs: check_generated_files: name: 'Check if generated files are up to date' runs-on: ubuntu-latest + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' steps: @@ -118,6 +120,7 @@ jobs: build_win32: name: 'Windows (x86)' runs-on: windows-latest + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: @@ -126,7 +129,6 @@ jobs: - uses: actions/checkout@v3 - name: Build CPython run: .\PCbuild\build.bat -e -d -p Win32 - timeout-minutes: 30 - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -135,6 +137,7 @@ jobs: build_win_amd64: name: 'Windows (x64)' runs-on: windows-latest + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: @@ -145,7 +148,6 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/msvc.json" - name: Build CPython run: .\PCbuild\build.bat -e -d -p x64 - timeout-minutes: 30 - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests @@ -154,6 +156,7 @@ jobs: build_macos: name: 'macOS' runs-on: macos-latest + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: @@ -184,6 +187,7 @@ jobs: build_ubuntu: name: 'Ubuntu' runs-on: ubuntu-20.04 + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: @@ -241,6 +245,7 @@ jobs: build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ubuntu-20.04 + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' strategy: @@ -290,6 +295,7 @@ jobs: build_asan: name: 'Address sanitizer' runs-on: ubuntu-20.04 + timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: @@ -302,6 +308,10 @@ jobs: run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Set up GCC-10 for ASAN + uses: egor-tensin/setup-gcc@v1 + with: + version: 10 - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index 5f1dcae190efbc..2bed09014e0ff2 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -26,6 +26,7 @@ jobs: build: name: Windows Installer runs-on: windows-latest + timeout-minutes: 60 strategy: matrix: type: [x86, x64, arm64] diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 314a7da647ff70..3101b30231c355 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -36,6 +36,7 @@ jobs: build_doc: name: 'Docs' runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@v3 - name: Register Sphinx problem matcher @@ -80,6 +81,7 @@ jobs: doctest: name: 'Doctest' runs-on: ubuntu-latest + timeout-minutes: 60 steps: - uses: actions/checkout@v3 - name: Register Sphinx problem matcher diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index b2a76ef7d36153..73806c5d6d58af 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -11,6 +11,7 @@ permissions: jobs: notify-new-bugs-announce: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/setup-node@v3 with: diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 99c7a05ae8cab0..7574bfc208ff76 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -13,16 +13,15 @@ jobs: add-to-project: name: Add issues to projects runs-on: ubuntu-latest + timeout-minutes: 10 strategy: matrix: include: # if an issue has any of these labels, it will be added # to the corresponding project - { project: 2, label: "release-blocker, deferred-blocker" } - - { project: 3, label: expert-subinterpreters } - - { project: 29, label: expert-asyncio } - { project: 32, label: sprint } - + steps: - uses: actions/add-to-project@v0.1.0 with: diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml new file mode 100644 index 00000000000000..916bbeb4352734 --- /dev/null +++ b/.github/workflows/require-pr-label.yml @@ -0,0 +1,18 @@ +name: Check labels + +on: + pull_request: + types: [opened, reopened, labeled, unlabeled, synchronize] + +jobs: + label: + name: DO-NOT-MERGE / unresolved review + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: mheap/github-action-required-labels@v4 + with: + mode: exactly + count: 0 + labels: "DO-NOT-MERGE, awaiting changes, awaiting change review" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 07dbcfe31d6563..94676f5ee5fffc 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,10 +12,11 @@ jobs: if: github.repository_owner == 'python' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: "Check PRs" - uses: actions/stale@v7 + uses: actions/stale@v8 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 969515ed287b55..17d841f1f1c54a 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -1,4 +1,4 @@ -name: Verify bundled pip and setuptools +name: Verify bundled wheels on: workflow_dispatch: @@ -23,10 +23,11 @@ concurrency: jobs: verify: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: python-version: '3' - - name: Compare checksums of bundled pip and setuptools to ones published on PyPI + - name: Compare checksum of bundled wheels to the ones published on PyPI run: ./Tools/build/verify_ensurepip_wheels.py diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 7b5d1fac40ed87..69b15296993301 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -232,6 +232,15 @@ Type Objects .. versionadded:: 3.11 +.. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) + + Attempt to assign a version tag to the given type. + + Returns 1 if the type already had a valid version tag or a new one was + assigned, or 0 if a new tag could not be assigned. + + .. versionadded:: 3.12 + Creating Heap-Allocated Types ............................. diff --git a/Doc/conf.py b/Doc/conf.py index 29fb63cbcc8614..42c23bf77c7034 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -68,12 +68,21 @@ # Minimum version of sphinx required needs_sphinx = '3.2' +# Ignore any .rst files in the includes/ directory; +# they're embedded in pages but not rendered individually. # Ignore any .rst files in the venv/ directory. -exclude_patterns = ['venv/*', 'README.rst'] +exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst'] venvdir = os.getenv('VENVDIR') if venvdir is not None: exclude_patterns.append(venvdir + '/*') +nitpick_ignore = [ + # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot + # be resolved, as the method is currently undocumented. For context, see + # https://github.com/python/cpython/pull/103289. + ('py:meth', '_SubParsersAction.add_parser'), +] + # Disable Docutils smartquotes for several translations smartquotes_excludes = { 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], @@ -252,8 +261,31 @@ # Options for the link checker # ---------------------------- -# Ignore certain URLs. -linkcheck_ignore = [r'https://bugs.python.org/(issue)?\d+'] +linkcheck_allowed_redirects = { + # bpo-NNNN -> BPO -> GH Issues + r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': 'https://github.com/python/cpython/issues/\d+', + # GH-NNNN used to refer to pull requests + r'https://github.com/python/cpython/issues/\d+': 'https://github.com/python/cpython/pull/\d+', + # :source:`something` linking files in the repository + r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*' +} + +linkcheck_anchors_ignore = [ + # ignore anchors that start with a '/', e.g. Wikipedia media files: + # https://en.wikipedia.org/wiki/Walrus#/media/File:Pacific_Walrus_-_Bull_(8247646168).jpg + r'\/.*', +] + +linkcheck_ignore = [ + # The crawler gets "Anchor not found" + r'https://developer.apple.com/documentation/.+?#.*', + r'https://devguide.python.org.+?/#.*', + r'https://github.com.+?#.*', + # Robot crawlers not allowed: "403 Client Error: Forbidden" + r'https://support.enthought.com/hc/.*', + # SSLError CertificateError, even though it is valid + r'https://unix.org/version2/whatsnew/lp64_wp.html', +] # Options for extensions diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 21389adedf9c15..d237f8f082d87b 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -129,14 +129,10 @@ involved in creating and publishing a project: * `Uploading the project to the Python Package Index`_ * `The .pypirc file`_ -.. _Project structure: \ - https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects -.. _Building and packaging the project: \ - https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files -.. _Uploading the project to the Python Package Index: \ - https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives -.. _The .pypirc file: \ - https://packaging.python.org/specifications/pypirc/ +.. _Project structure: https://packaging.python.org/tutorials/packaging-projects/#packaging-python-projects +.. _Building and packaging the project: https://packaging.python.org/tutorials/packaging-projects/#creating-the-package-files +.. _Uploading the project to the Python Package Index: https://packaging.python.org/tutorials/packaging-projects/#uploading-the-distribution-archives +.. _The .pypirc file: https://packaging.python.org/specifications/pypirc/ How do I...? diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 80a1387db200c2..56b40acdb69fed 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -337,7 +337,7 @@ Here is an example:: } PyErr_Format(PyExc_AttributeError, - "'%.50s' object has no attribute '%.400s'", + "'%.100s' object has no attribute '%.400s'", tp->tp_name, name); return NULL; } diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index a9cde456575020..597caaa778e1c8 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -780,7 +780,7 @@ socket to :meth:`select.select` to check if it's writable. The :mod:`asyncio` module provides a general purpose single-threaded and concurrent asynchronous library, which can be used for writing non-blocking network code. - The third-party `Twisted `_ library is + The third-party `Twisted `_ library is a popular and feature-rich alternative. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 3d74d550dc345a..53e8cdcae1cd66 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -214,7 +214,7 @@ Glossary A callable is an object that can be called, possibly with a set of arguments (see :term:`argument`), with the following syntax:: - callable(argument1, argument2, ...) + callable(argument1, argument2, argumentN) A :term:`function`, and by extension a :term:`method`, is a callable. An instance of a class that implements the :meth:`~object.__call__` diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index f682587488a227..52e98fa9620194 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -1,10 +1,12 @@ +.. _argparse-tutorial: + ***************** Argparse Tutorial ***************** :author: Tshepang Mbambo -.. _argparse-tutorial: +.. currentmodule:: argparse This tutorial is intended to be a gentle introduction to :mod:`argparse`, the recommended command-line parsing module in the Python standard library. @@ -12,7 +14,7 @@ recommended command-line parsing module in the Python standard library. .. note:: There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for :c:func:`getopt` from the C + :mod:`getopt` (an equivalent for ``getopt()`` from the C language) and the deprecated :mod:`optparse`. Note also that :mod:`argparse` is based on :mod:`optparse`, and therefore very similar in terms of usage. @@ -137,13 +139,13 @@ And running the code: Here is what's happening: -* We've added the :meth:`add_argument` method, which is what we use to specify +* We've added the :meth:`~ArgumentParser.add_argument` method, which is what we use to specify which command-line options the program is willing to accept. In this case, I've named it ``echo`` so that it's in line with its function. * Calling our program now requires us to specify an option. -* The :meth:`parse_args` method actually returns some data from the +* The :meth:`~ArgumentParser.parse_args` method actually returns some data from the options specified, in this case, ``echo``. * The variable is some form of 'magic' that :mod:`argparse` performs for free @@ -256,7 +258,7 @@ Here is what is happening: * To show that the option is actually optional, there is no error when running the program without it. Note that by default, if an optional argument isn't - used, the relevant variable, in this case :attr:`args.verbosity`, is + used, the relevant variable, in this case ``args.verbosity``, is given ``None`` as a value, which is the reason it fails the truth test of the :keyword:`if` statement. @@ -299,7 +301,7 @@ Here is what is happening: We even changed the name of the option to match that idea. Note that we now specify a new keyword, ``action``, and give it the value ``"store_true"``. This means that, if the option is specified, - assign the value ``True`` to :data:`args.verbose`. + assign the value ``True`` to ``args.verbose``. Not specifying it implies ``False``. * It complains when you specify a value, in true spirit of what flags @@ -698,7 +700,7 @@ Conflicting options So far, we have been working with two methods of an :class:`argparse.ArgumentParser` instance. Let's introduce a third one, -:meth:`add_mutually_exclusive_group`. It allows for us to specify options that +:meth:`~ArgumentParser.add_mutually_exclusive_group`. It allows for us to specify options that conflict with each other. Let's also change the rest of the program so that the new functionality makes more sense: we'll introduce the ``--quiet`` option, diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 74710d9b3fc2ed..3688c47f0d6ec9 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1273,11 +1273,14 @@ Using the non-data descriptor protocol, a pure Python version of .. testcode:: + import functools + class StaticMethod: "Emulate PyStaticMethod_Type() in Objects/funcobject.c" def __init__(self, f): self.f = f + functools.update_wrapper(self, f) def __get__(self, obj, objtype=None): return self.f @@ -1285,13 +1288,19 @@ Using the non-data descriptor protocol, a pure Python version of def __call__(self, *args, **kwds): return self.f(*args, **kwds) +The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute +that refers to the underlying function. Also it carries forward +the attributes necessary to make the wrapper look like the wrapped +function: ``__name__``, ``__qualname__``, ``__doc__``, and ``__annotations__``. + .. testcode:: :hide: class E_sim: @StaticMethod - def f(x): - return x * 10 + def f(x: int) -> str: + "Simple function example" + return "!" * x wrapped_ord = StaticMethod(ord) @@ -1299,11 +1308,51 @@ Using the non-data descriptor protocol, a pure Python version of :hide: >>> E_sim.f(3) - 30 + '!!!' >>> E_sim().f(3) - 30 + '!!!' + + >>> sm = vars(E_sim)['f'] + >>> type(sm).__name__ + 'StaticMethod' + >>> f = E_sim.f + >>> type(f).__name__ + 'function' + >>> sm.__name__ + 'f' + >>> f.__name__ + 'f' + >>> sm.__qualname__ + 'E_sim.f' + >>> f.__qualname__ + 'E_sim.f' + >>> sm.__doc__ + 'Simple function example' + >>> f.__doc__ + 'Simple function example' + >>> sm.__annotations__ + {'x': , 'return': } + >>> f.__annotations__ + {'x': , 'return': } + >>> sm.__module__ == f.__module__ + True + >>> sm(3) + '!!!' + >>> f(3) + '!!!' + >>> wrapped_ord('A') 65 + >>> wrapped_ord.__module__ == ord.__module__ + True + >>> wrapped_ord.__wrapped__ == ord + True + >>> wrapped_ord.__name__ == ord.__name__ + True + >>> wrapped_ord.__qualname__ == ord.__qualname__ + True + >>> wrapped_ord.__doc__ == ord.__doc__ + True Class methods @@ -1359,11 +1408,14 @@ Using the non-data descriptor protocol, a pure Python version of .. testcode:: + import functools + class ClassMethod: "Emulate PyClassMethod_Type() in Objects/funcobject.c" def __init__(self, f): self.f = f + functools.update_wrapper(self, f) def __get__(self, obj, cls=None): if cls is None: @@ -1380,8 +1432,9 @@ Using the non-data descriptor protocol, a pure Python version of # Verify the emulation works class T: @ClassMethod - def cm(cls, x, y): - return (cls, x, y) + def cm(cls, x: int, y: str) -> tuple[str, int, str]: + "Class method that returns a tuple" + return (cls.__name__, x, y) @ClassMethod @property @@ -1393,17 +1446,40 @@ Using the non-data descriptor protocol, a pure Python version of :hide: >>> T.cm(11, 22) - (, 11, 22) + ('T', 11, 22) # Also call it from an instance >>> t = T() >>> t.cm(11, 22) - (, 11, 22) + ('T', 11, 22) # Check the alternate path for chained descriptors >>> T.__doc__ "A doc for 'T'" + # Verify that T uses our emulation + >>> type(vars(T)['cm']).__name__ + 'ClassMethod' + + # Verify that update_wrapper() correctly copied attributes + >>> T.cm.__name__ + 'cm' + >>> T.cm.__qualname__ + 'T.cm' + >>> T.cm.__doc__ + 'Class method that returns a tuple' + >>> T.cm.__annotations__ + {'x': , 'y': , 'return': tuple[str, int, str]} + + # Verify that __wrapped__ was added and works correctly + >>> f = vars(T)['cm'].__wrapped__ + >>> type(f).__name__ + 'function' + >>> f.__name__ + 'cm' + >>> f(T, 11, 22) + ('T', 11, 22) + The code path for ``hasattr(type(self.f), '__get__')`` was added in Python 3.9 and makes it possible for :func:`classmethod` to support @@ -1423,6 +1499,12 @@ chained together. In Python 3.11, this functionality was deprecated. >>> G.__doc__ "A doc for 'G'" +The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a +``__wrapped__`` attribute that refers to the underlying function. Also +it carries forward the attributes necessary to make the wrapper look +like the wrapped function: ``__name__``, ``__qualname__``, ``__doc__``, +and ``__annotations__``. + Member objects and __slots__ ---------------------------- diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 9390faded2fb8d..68b75c529e92c7 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -36,8 +36,10 @@ inherits from :class:`Enum` itself. .. note:: Case of Enum Members - Because Enums are used to represent constants we recommend using - UPPER_CASE names for members, and will be using that style in our examples. + Because Enums are used to represent constants, and to help avoid issues + with name clashes between mixin-class methods/attributes and enum names, + we strongly recommend using UPPER_CASE names for members, and will be using + that style in our examples. Depending on the nature of the enum a member's value may or may not be important, but either way that value can be used to get the corresponding @@ -490,6 +492,10 @@ the :meth:`~Enum.__repr__` omits the inherited class' name. For example:: Use the :func:`!dataclass` argument ``repr=False`` to use the standard :func:`repr`. +.. versionchanged:: 3.12 + Only the dataclass fields are shown in the value area, not the dataclass' + name. + Pickling -------- @@ -865,17 +871,19 @@ Some rules: 4. When another data type is mixed in, the :attr:`value` attribute is *not the same* as the enum member itself, although it is equivalent and will compare equal. -5. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's +5. A ``data type`` is a mixin that defines :meth:`__new__`, or a + :class:`~dataclasses.dataclass` +6. %-style formatting: ``%s`` and ``%r`` call the :class:`Enum` class's :meth:`__str__` and :meth:`__repr__` respectively; other codes (such as ``%i`` or ``%h`` for IntEnum) treat the enum member as its mixed-in type. -6. :ref:`Formatted string literals `, :meth:`str.format`, +7. :ref:`Formatted string literals `, :meth:`str.format`, and :func:`format` will use the enum's :meth:`__str__` method. .. note:: Because :class:`IntEnum`, :class:`IntFlag`, and :class:`StrEnum` are designed to be drop-in replacements for existing constants, their - :meth:`__str__` method has been reset to their data types + :meth:`__str__` method has been reset to their data types' :meth:`__str__` method. When to use :meth:`__new__` vs. :meth:`__init__` @@ -990,7 +998,9 @@ but remain normal attributes. Enum members are instances of their enum class, and are normally accessed as ``EnumClass.member``. In certain situations, such as writing custom enum behavior, being able to access one member directly from another is useful, -and is supported. +and is supported; however, in order to avoid name clashes between member names +and attributes/methods from mixed-in classes, upper-case names are strongly +recommended. .. versionchanged:: 3.5 diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 38a651b0f964a6..5cf12cc52bde4e 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -1208,8 +1208,8 @@ General ------- **Structure and Interpretation of Computer Programs**, by Harold Abelson and -Gerald Jay Sussman with Julie Sussman. Full text at -https://mitpress.mit.edu/sicp/. In this classic textbook of computer science, +Gerald Jay Sussman with Julie Sussman. The book can be found at +https://mitpress.mit.edu/sicp. In this classic textbook of computer science, chapters 2 and 3 discuss the use of sequences and streams to organize the data flow inside a program. The book uses Scheme for its examples, but many of the design approaches described in these chapters are applicable to functional-style diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index 2eddb582da7c24..0262054ae2b4a0 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -372,7 +372,7 @@ To save a some tedious error-handling boilerplate code, you can combine these two steps with :c:func:`PyType_GetModuleState`, resulting in:: my_struct *state = (my_struct*)PyType_GetModuleState(type); - if (state === NULL) { + if (state == NULL) { return NULL; } @@ -435,7 +435,7 @@ For example:: PyObject *kwnames) { my_struct *state = (my_struct*)PyType_GetModuleState(defining_class); - if (state === NULL) { + if (state == NULL) { return NULL; } ... // rest of logic @@ -479,7 +479,7 @@ to get the state:: PyObject *module = PyType_GetModuleByDef(Py_TYPE(self), &module_def); my_struct *state = (my_struct*)PyModule_GetState(module); - if (state === NULL) { + if (state == NULL) { return NULL; } diff --git a/Doc/howto/perf_profiling.rst b/Doc/howto/perf_profiling.rst index ad2eb7b4d58aa5..6af5536166f58a 100644 --- a/Doc/howto/perf_profiling.rst +++ b/Doc/howto/perf_profiling.rst @@ -15,9 +15,9 @@ information about the performance of your application. that aid with the analysis of the data that it produces. The main problem with using the ``perf`` profiler with Python applications is that -``perf`` only allows to get information about native symbols, this is, the names of -the functions and procedures written in C. This means that the names and file names -of the Python functions in your code will not appear in the output of the ``perf``. +``perf`` only gets information about native symbols, that is, the names of +functions and procedures written in C. This means that the names and file names +of Python functions in your code will not appear in the output of ``perf``. Since Python 3.12, the interpreter can run in a special mode that allows Python functions to appear in the output of the ``perf`` profiler. When this mode is @@ -28,8 +28,8 @@ relationship between this piece of code and the associated Python function using .. note:: - Support for the ``perf`` profiler is only currently available for Linux on - selected architectures. Check the output of the configure build step or + Support for the ``perf`` profiler is currently only available for Linux on + select architectures. Check the output of the ``configure`` build step or check the output of ``python -m sysconfig | grep HAVE_PERF_TRAMPOLINE`` to see if your system is supported. @@ -52,11 +52,11 @@ For example, consider the following script: if __name__ == "__main__": baz(1000000) -We can run ``perf`` to sample CPU stack traces at 9999 Hertz:: +We can run ``perf`` to sample CPU stack traces at 9999 hertz:: $ perf record -F 9999 -g -o perf.data python my_script.py -Then we can use ``perf`` report to analyze the data: +Then we can use ``perf report`` to analyze the data: .. code-block:: shell-session @@ -97,7 +97,7 @@ Then we can use ``perf`` report to analyze the data: | | | | | |--2.97%--_PyObject_Malloc ... -As you can see here, the Python functions are not shown in the output, only ``_Py_Eval_EvalFrameDefault`` appears +As you can see, the Python functions are not shown in the output, only ``_Py_Eval_EvalFrameDefault`` (the function that evaluates the Python bytecode) shows up. Unfortunately that's not very useful because all Python functions use the same C function to evaluate bytecode so we cannot know which Python function corresponds to which bytecode-evaluating function. @@ -151,7 +151,7 @@ Instead, if we run the same experiment with ``perf`` support enabled we get: How to enable ``perf`` profiling support ---------------------------------------- -``perf`` profiling support can either be enabled from the start using +``perf`` profiling support can be enabled either from the start using the environment variable :envvar:`PYTHONPERFSUPPORT` or the :option:`-X perf <-X>` option, or dynamically using :func:`sys.activate_stack_trampoline` and @@ -192,7 +192,7 @@ Example, using the :mod:`sys` APIs in file :file:`example.py`: How to obtain the best results ------------------------------ -For the best results, Python should be compiled with +For best results, Python should be compiled with ``CFLAGS="-fno-omit-frame-pointer -mno-omit-leaf-frame-pointer"`` as this allows profilers to unwind using only the frame pointer and not on DWARF debug information. This is because as the code that is interposed to allow ``perf`` diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 69af3c3a85c5d6..61ba6bd7224fcc 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -86,7 +86,7 @@ response:: import urllib.request - req = urllib.request.Request('http://www.voidspace.org.uk') + req = urllib.request.Request('http://python.org/') with urllib.request.urlopen(req) as response: the_page = response.read() @@ -458,7 +458,7 @@ To illustrate creating and installing a handler we will use the ``HTTPBasicAuthHandler``. For a more detailed discussion of this subject -- including an explanation of how Basic Authentication works - see the `Basic Authentication Tutorial -`_. +`__. When authentication is required, the server sends a header (as well as the 401 error code) requesting authentication. This specifies the authentication scheme diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index ee68ac58d3de75..33e367f3ccda89 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -67,7 +67,7 @@ default_ Default value used when an argument is not provided dest_ Specify the attribute name used in the result namespace help_ Help message for an argument metavar_ Alternate display name for the argument as shown in help -nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, ``'+'``, or ``argparse.REMAINDER`` +nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` required_ Indicate whether an argument is required or optional ``True`` or ``False`` type_ Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function ====================== =========================================================== ========================================================================================================================== @@ -585,7 +585,7 @@ arguments will never be treated as file references. .. versionchanged:: 3.12 :class:`ArgumentParser` changed encoding and errors to read arguments files - from default (e.g. :func:`locale.getpreferredencoding(False)` and + from default (e.g. :func:`locale.getpreferredencoding(False) ` and ``"strict"``) to :term:`filesystem encoding and error handler`. Arguments file should be encoded in UTF-8 instead of ANSI Codepage on Windows. @@ -1191,7 +1191,7 @@ done downstream after the arguments are parsed. For example, JSON or YAML conversions have complex error cases that require better reporting than can be given by the ``type`` keyword. A :exc:`~json.JSONDecodeError` would not be well formatted and a -:exc:`FileNotFound` exception would not be handled at all. +:exc:`FileNotFoundError` exception would not be handled at all. Even :class:`~argparse.FileType` has its limitations for use with the ``type`` keyword. If one argument uses *FileType* and then a subsequent argument fails, @@ -1445,7 +1445,7 @@ Action classes Action classes implement the Action API, a callable which returns a callable which processes arguments from the command-line. Any object which follows this API may be passed as the ``action`` parameter to -:meth:`add_argument`. +:meth:`~ArgumentParser.add_argument`. .. class:: Action(option_strings, dest, nargs=None, const=None, default=None, \ type=None, choices=None, required=False, help=None, \ @@ -1723,7 +1723,7 @@ Sub-commands :class:`ArgumentParser` supports the creation of such sub-commands with the :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally called with no arguments and returns a special action object. This object - has a single method, :meth:`~ArgumentParser.add_parser`, which takes a + has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a command name and any :class:`ArgumentParser` constructor arguments, and returns an :class:`ArgumentParser` object that can be modified as usual. @@ -1789,7 +1789,7 @@ Sub-commands for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each subparser command, however, can be given by supplying the ``help=`` argument - to :meth:`add_parser` as above.) + to :meth:`~_SubParsersAction.add_parser` as above.) :: @@ -2157,7 +2157,7 @@ the populated namespace and the list of remaining argument strings. .. warning:: :ref:`Prefix matching ` rules apply to - :meth:`parse_known_args`. The parser may consume an option even if it's just + :meth:`~ArgumentParser.parse_known_args`. The parser may consume an option even if it's just a prefix of one of its known options, instead of leaving it in the remaining arguments list. @@ -2218,7 +2218,7 @@ support this parsing style. These parsers do not support all the argparse features, and will raise exceptions if unsupported features are used. In particular, subparsers, -``argparse.REMAINDER``, and mutually exclusive groups that include both +and mutually exclusive groups that include both optionals and positionals are not supported. The following example shows the difference between @@ -2295,3 +2295,17 @@ A partial upgrade path from :mod:`optparse` to :mod:`argparse`: * Replace the OptionParser constructor ``version`` argument with a call to ``parser.add_argument('--version', action='version', version='')``. + +Exceptions +---------- + +.. exception:: ArgumentError + + An error from creating or using an argument (optional or positional). + + The string value of this exception is the message, augmented with + information about the argument that caused it. + +.. exception:: ArgumentTypeError + + Raised when something goes wrong converting a command line string to a type. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 41d09e1e79705c..ba0f909c405a34 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -256,8 +256,9 @@ Creating Tasks .. note:: - :meth:`asyncio.TaskGroup.create_task` is a newer alternative - that allows for convenient waiting for a group of related tasks. + :meth:`asyncio.TaskGroup.create_task` is a new alternative + leveraging structural concurrency; it allows for waiting + for a group of related tasks with strong safety guarantees. .. important:: @@ -340,7 +341,7 @@ Example:: async with asyncio.TaskGroup() as tg: task1 = tg.create_task(some_coro(...)) task2 = tg.create_task(another_coro(...)) - print("Both tasks have completed now.") + print(f"Both tasks have completed now: {task1.result()}, {task2.result()}") The ``async with`` statement will wait for all tasks in the group to finish. While waiting, new tasks may still be added to the group @@ -459,8 +460,12 @@ Running Tasks Concurrently Tasks/Futures to be cancelled. .. note:: - A more modern way to create and run tasks concurrently and - wait for their completion is :class:`asyncio.TaskGroup`. + A new alternative to create and run tasks concurrently and + wait for their completion is :class:`asyncio.TaskGroup`. *TaskGroup* + provides stronger safety guarantees than *gather* for scheduling a nesting of subtasks: + if a task (or a subtask, a task scheduled by a task) + raises an exception, *TaskGroup* will, while *gather* will not, + cancel the remaining scheduled tasks). .. _asyncio_example_gather: @@ -829,6 +834,9 @@ Waiting Primitives Deprecation warning is emitted if not all awaitable objects in the *aws* iterable are Future-like objects and there is no running event loop. + .. versionchanged:: 3.12 + Added support for generators yielding tasks. + Running in Threads ================== diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 1b55868c3aa62f..7cd081d1f54f43 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -304,8 +304,15 @@ Functions and classes provided: This context manager is :ref:`reentrant `. + If the code within the :keyword:`!with` block raises an + :exc:`ExceptionGroup`, suppressed exceptions are removed from the + group. If any exceptions in the group are not suppressed, a group containing them is re-raised. + .. versionadded:: 3.4 + .. versionchanged:: 3.12 + ``suppress`` now supports suppressing exceptions raised as + part of an :exc:`ExceptionGroup`. .. function:: redirect_stdout(new_target) diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 866b180f4bc3b8..2107215c0c1967 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -28,8 +28,8 @@ Such constructors may be factory functions or class instances. .. function:: pickle(type, function, constructor_ob=None) Declares that *function* should be used as a "reduction" function for objects - of type *type*. *function* should return either a string or a tuple - containing two or three elements. See the :attr:`~pickle.Pickler.dispatch_table` + of type *type*. *function* must return either a string or a tuple + containing two or five elements. See the :attr:`~pickle.Pickler.dispatch_table` for more details on the interface of *function*. The *constructor_ob* parameter is a legacy feature and is now ignored, but if diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index f1776554d8b9f2..64baa69be4af31 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -327,7 +327,7 @@ The :mod:`csv` module defines the following constants: Instructs :class:`writer` objects to quote all non-numeric fields. - Instructs the reader to convert all non-quoted fields to type *float*. + Instructs :class:`reader` objects to convert all non-quoted fields to type *float*. .. data:: QUOTE_NONE @@ -337,7 +337,25 @@ The :mod:`csv` module defines the following constants: character. If *escapechar* is not set, the writer will raise :exc:`Error` if any characters that require escaping are encountered. - Instructs :class:`reader` to perform no special processing of quote characters. + Instructs :class:`reader` objects to perform no special processing of quote characters. + +.. data:: QUOTE_NOTNULL + + Instructs :class:`writer` objects to quote all fields which are not + ``None``. This is similar to :data:`QUOTE_ALL`, except that if a + field value is ``None`` an empty (unquoted) string is written. + + Instructs :class:`reader` objects to interpret an empty (unquoted) field as None and + to otherwise behave as :data:`QUOTE_ALL`. + +.. data:: QUOTE_STRINGS + + Instructs :class:`writer` objects to always place quotes around fields + which are strings. This is similar to :data:`QUOTE_NONNUMERIC`, except that if a + field value is ``None`` an empty (unquoted) string is written. + + Instructs :class:`reader` objects to interpret an empty (unquoted) string as ``None`` and + to otherwise behave as :data:`QUOTE_NONNUMERIC`. The :mod:`csv` module defines the following exception: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index d49d702e9e7999..81509c0920bb6e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -390,7 +390,7 @@ regular, non-variadic, function arguments: libc.printf.argtypes = [ctypes.c_char_p] -Because specifying the attribute does inhibit portability it is advised to always +Because specifying the attribute does not inhibit portability it is advised to always specify ``argtypes`` for all variadic functions. diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 5f4dc25bfd7877..85a7d9026d1d30 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -12,8 +12,8 @@ -------------- This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`__init__` and -:meth:`__repr__` to user-defined classes. It was originally described +adding generated :term:`special method`\s such as :meth:`~object.__init__` and +:meth:`~object.__repr__` to user-defined classes. It was originally described in :pep:`557`. The member variables to use in these generated methods are defined @@ -31,7 +31,7 @@ using :pep:`526` type annotations. For example, this code:: def total_cost(self) -> float: return self.unit_price * self.quantity_on_hand -will add, among other things, a :meth:`__init__` that looks like:: +will add, among other things, a :meth:`~object.__init__` that looks like:: def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name @@ -86,86 +86,86 @@ Module contents The parameters to :func:`dataclass` are: - - ``init``: If true (the default), a :meth:`__init__` method will be + - ``init``: If true (the default), a :meth:`~object.__init__` method will be generated. - If the class already defines :meth:`__init__`, this parameter is + If the class already defines :meth:`~object.__init__`, this parameter is ignored. - - ``repr``: If true (the default), a :meth:`__repr__` method will be + - ``repr``: If true (the default), a :meth:`~object.__repr__` method will be generated. The generated repr string will have the class name and the name and repr of each field, in the order they are defined in the class. Fields that are marked as being excluded from the repr are not included. For example: ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - If the class already defines :meth:`__repr__`, this parameter is + If the class already defines :meth:`~object.__repr__`, this parameter is ignored. - - ``eq``: If true (the default), an :meth:`__eq__` method will be + - ``eq``: If true (the default), an :meth:`~object.__eq__` method will be generated. This method compares the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the identical type. - If the class already defines :meth:`__eq__`, this parameter is + If the class already defines :meth:`~object.__eq__`, this parameter is ignored. - - ``order``: If true (the default is ``False``), :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be + - ``order``: If true (the default is ``False``), :meth:`~object.__lt__`, + :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be generated. These compare the class as if it were a tuple of its fields, in order. Both instances in the comparison must be of the identical type. If ``order`` is true and ``eq`` is false, a :exc:`ValueError` is raised. - If the class already defines any of :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then + If the class already defines any of :meth:`~object.__lt__`, + :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, then :exc:`TypeError` is raised. - - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method + - ``unsafe_hash``: If ``False`` (the default), a :meth:`~object.__hash__` method is generated according to how ``eq`` and ``frozen`` are set. - :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are + :meth:`~object.__hash__` is used by built-in :meth:`hash()`, and when objects are added to hashed collections such as dictionaries and sets. Having a - :meth:`__hash__` implies that instances of the class are immutable. + :meth:`~object.__hash__` implies that instances of the class are immutable. Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`__eq__`, and the values of + intent, the existence and behavior of :meth:`~object.__eq__`, and the values of the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. - By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` + By default, :func:`dataclass` will not implicitly add a :meth:`~object.__hash__` method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`__hash__` method. Setting the class + existing explicitly defined :meth:`~object.__hash__` method. Setting the class attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`__hash__` documentation. + described in the :meth:`~object.__hash__` documentation. - If :meth:`__hash__` is not explicitly defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. + If :meth:`~object.__hash__` is not explicitly defined, or if it is set to ``None``, + then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. Although not recommended, you can force :func:`dataclass` to create a - :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case + :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case if your class is logically immutable but can nonetheless be mutated. This is a specialized use case and should be considered carefully. - Here are the rules governing implicit creation of a :meth:`__hash__` - method. Note that you cannot both have an explicit :meth:`__hash__` + Here are the rules governing implicit creation of a :meth:`~object.__hash__` + method. Note that you cannot both have an explicit :meth:`~object.__hash__` method in your dataclass and set ``unsafe_hash=True``; this will result in a :exc:`TypeError`. If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it + generate a :meth:`~object.__hash__` method for you. If ``eq`` is true and + ``frozen`` is false, :meth:`~object.__hash__` will be set to ``None``, marking it unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` + :meth:`~object.__hash__` will be left untouched meaning the :meth:`~object.__hash__` method of the superclass will be used (if the superclass is :class:`object`, this means it will fall back to id-based hashing). - ``frozen``: If true (the default is ``False``), assigning to fields will generate an exception. This emulates read-only frozen instances. If - :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then + :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then :exc:`TypeError` is raised. See the discussion below. - ``match_args``: If true (the default is ``True``), the ``__match_args__`` tuple will be created from the list of - parameters to the generated :meth:`__init__` method (even if - :meth:`__init__` is not generated, see above). If false, or if + parameters to the generated :meth:`~object.__init__` method (even if + :meth:`~object.__init__` is not generated, see above). If false, or if ``__match_args__`` is already defined in the class, then ``__match_args__`` will not be generated. @@ -173,18 +173,18 @@ Module contents - ``kw_only``: If true (the default value is ``False``), then all fields will be marked as keyword-only. If a field is marked as - keyword-only, then the only effect is that the :meth:`__init__` + keyword-only, then the only effect is that the :meth:`~object.__init__` parameter generated from a keyword-only field must be specified - with a keyword when :meth:`__init__` is called. There is no + with a keyword when :meth:`~object.__init__` is called. There is no effect on any other aspect of dataclasses. See the :term:`parameter` glossary entry for details. Also see the :const:`KW_ONLY` section. .. versionadded:: 3.10 - - ``slots``: If true (the default is ``False``), :attr:`__slots__` attribute + - ``slots``: If true (the default is ``False``), :attr:`~object.__slots__` attribute will be generated and new class will be returned instead of the original one. - If :attr:`__slots__` is already defined in the class, then :exc:`TypeError` + If :attr:`~object.__slots__` is already defined in the class, then :exc:`TypeError` is raised. .. versionadded:: 3.10 @@ -215,7 +215,7 @@ Module contents b: int = 0 # assign a default value for 'b' In this example, both ``a`` and ``b`` will be included in the added - :meth:`__init__` method, which will be defined as:: + :meth:`~object.__init__` method, which will be defined as:: def __init__(self, a: int, b: int = 0): @@ -256,13 +256,13 @@ Module contents error to specify both ``default`` and ``default_factory``. - ``init``: If true (the default), this field is included as a - parameter to the generated :meth:`__init__` method. + parameter to the generated :meth:`~object.__init__` method. - ``repr``: If true (the default), this field is included in the - string returned by the generated :meth:`__repr__` method. + string returned by the generated :meth:`~object.__repr__` method. - ``hash``: This can be a bool or ``None``. If true, this field is - included in the generated :meth:`__hash__` method. If ``None`` (the + included in the generated :meth:`~object.__hash__` method. If ``None`` (the default), use the value of ``compare``: this would normally be the expected behavior. A field should be considered in the hash if it's used for comparisons. Setting this value to anything @@ -275,8 +275,8 @@ Module contents is excluded from the hash, it will still be used for comparisons. - ``compare``: If true (the default), this field is included in the - generated equality and comparison methods (:meth:`__eq__`, - :meth:`__gt__`, et al.). + generated equality and comparison methods (:meth:`~object.__eq__`, + :meth:`~object.__gt__`, et al.). - ``metadata``: This can be a mapping or None. None is treated as an empty dict. This value is wrapped in @@ -287,7 +287,7 @@ Module contents namespace in the metadata. - ``kw_only``: If true, this field will be marked as keyword-only. - This is used when the generated :meth:`__init__` method's + This is used when the generated :meth:`~object.__init__` method's parameters are computed. .. versionadded:: 3.10 @@ -435,13 +435,13 @@ Module contents Class, raises :exc:`TypeError`. If values in ``changes`` do not specify fields, raises :exc:`TypeError`. - The newly returned object is created by calling the :meth:`__init__` + The newly returned object is created by calling the :meth:`~object.__init__` method of the dataclass. This ensures that - :meth:`__post_init__`, if present, is also called. + :ref:`__post_init__ `, if present, is also called. Init-only variables without default values, if any exist, must be specified on the call to :func:`replace` so that they can be passed to - :meth:`__init__` and :meth:`__post_init__`. + :meth:`~object.__init__` and :ref:`__post_init__ `. It is an error for ``changes`` to contain any fields that are defined as having ``init=False``. A :exc:`ValueError` will be raised @@ -449,7 +449,7 @@ Module contents Be forewarned about how ``init=False`` fields work during a call to :func:`replace`. They are not copied from the source object, but - rather are initialized in :meth:`__post_init__`, if they're + rather are initialized in :ref:`__post_init__ `, if they're initialized at all. It is expected that ``init=False`` fields will be rarely and judiciously used. If they are used, it might be wise to have alternate class constructors, or perhaps a custom @@ -480,7 +480,7 @@ Module contents :const:`KW_ONLY` is otherwise completely ignored. This includes the name of such a field. By convention, a name of ``_`` is used for a :const:`KW_ONLY` field. Keyword-only fields signify - :meth:`__init__` parameters that must be specified as keywords when + :meth:`~object.__init__` parameters that must be specified as keywords when the class is instantiated. In this example, the fields ``y`` and ``z`` will be marked as keyword-only fields:: @@ -501,20 +501,22 @@ Module contents .. exception:: FrozenInstanceError - Raised when an implicitly defined :meth:`__setattr__` or - :meth:`__delattr__` is called on a dataclass which was defined with + Raised when an implicitly defined :meth:`~object.__setattr__` or + :meth:`~object.__delattr__` is called on a dataclass which was defined with ``frozen=True``. It is a subclass of :exc:`AttributeError`. +.. _post-init-processing: + Post-init processing -------------------- -The generated :meth:`__init__` code will call a method named -:meth:`__post_init__`, if :meth:`__post_init__` is defined on the +The generated :meth:`~object.__init__` code will call a method named +:meth:`!__post_init__`, if :meth:`!__post_init__` is defined on the class. It will normally be called as ``self.__post_init__()``. However, if any ``InitVar`` fields are defined, they will also be -passed to :meth:`__post_init__` in the order they were defined in the -class. If no :meth:`__init__` method is generated, then -:meth:`__post_init__` will not automatically be called. +passed to :meth:`!__post_init__` in the order they were defined in the +class. If no :meth:`~object.__init__` method is generated, then +:meth:`!__post_init__` will not automatically be called. Among other uses, this allows for initializing field values that depend on one or more other fields. For example:: @@ -528,10 +530,10 @@ depend on one or more other fields. For example:: def __post_init__(self): self.c = self.a + self.b -The :meth:`__init__` method generated by :func:`dataclass` does not call base -class :meth:`__init__` methods. If the base class has an :meth:`__init__` method +The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base +class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method that has to be called, it is common to call this method in a -:meth:`__post_init__` method:: +:meth:`!__post_init__` method:: @dataclass class Rectangle: @@ -545,12 +547,12 @@ that has to be called, it is common to call this method in a def __post_init__(self): super().__init__(self.side, self.side) -Note, however, that in general the dataclass-generated :meth:`__init__` methods +Note, however, that in general the dataclass-generated :meth:`~object.__init__` methods don't need to be called, since the derived dataclass will take care of initializing all fields of any base class that is a dataclass itself. See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how +parameters to :meth:`!__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. Class variables @@ -573,8 +575,8 @@ if the type of a field is of type ``dataclasses.InitVar``. If a field is an ``InitVar``, it is considered a pseudo-field called an init-only field. As it is not a true field, it is not returned by the module-level :func:`fields` function. Init-only fields are added as -parameters to the generated :meth:`__init__` method, and are passed to -the optional :meth:`__post_init__` method. They are not otherwise used +parameters to the generated :meth:`~object.__init__` method, and are passed to +the optional :ref:`__post_init__ ` method. They are not otherwise used by dataclasses. For example, suppose a field will be initialized from a database, if a @@ -601,12 +603,12 @@ Frozen instances It is not possible to create truly immutable Python objects. However, by passing ``frozen=True`` to the :meth:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add -:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These +:meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. There is a tiny performance penalty when using ``frozen=True``: -:meth:`__init__` cannot use simple assignment to initialize fields, and -must use :meth:`object.__setattr__`. +:meth:`~object.__init__` cannot use simple assignment to initialize fields, and +must use :meth:`~object.__setattr__`. Inheritance ----------- @@ -634,14 +636,14 @@ example:: The final list of fields is, in order, ``x``, ``y``, ``z``. The final type of ``x`` is ``int``, as specified in class ``C``. -The generated :meth:`__init__` method for ``C`` will look like:: +The generated :meth:`~object.__init__` method for ``C`` will look like:: def __init__(self, x: int = 15, y: int = 0, z: int = 10): -Re-ordering of keyword-only parameters in :meth:`__init__` ----------------------------------------------------------- +Re-ordering of keyword-only parameters in :meth:`~object.__init__` +------------------------------------------------------------------ -After the parameters needed for :meth:`__init__` are computed, any +After the parameters needed for :meth:`~object.__init__` are computed, any keyword-only parameters are moved to come after all regular (non-keyword-only) parameters. This is a requirement of how keyword-only parameters are implemented in Python: they must come @@ -662,7 +664,7 @@ fields, and ``Base.x`` and ``D.z`` are regular fields:: z: int = 10 t: int = field(kw_only=True, default=0) -The generated :meth:`__init__` method for ``D`` will look like:: +The generated :meth:`~object.__init__` method for ``D`` will look like:: def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): @@ -671,7 +673,7 @@ the list of fields: parameters derived from regular fields are followed by parameters derived from keyword-only fields. The relative ordering of keyword-only parameters is maintained in the -re-ordered :meth:`__init__` parameter list. +re-ordered :meth:`~object.__init__` parameter list. Default factory functions @@ -683,10 +685,10 @@ example, to create a new instance of a list, use:: mylist: list = field(default_factory=list) -If a field is excluded from :meth:`__init__` (using ``init=False``) +If a field is excluded from :meth:`~object.__init__` (using ``init=False``) and the field also specifies ``default_factory``, then the default factory function will always be called from the generated -:meth:`__init__` function. This happens because there is no other +:meth:`~object.__init__` function. This happens because there is no other way to give the field an initial value. Mutable default values @@ -714,7 +716,7 @@ Using dataclasses, *if* this code was valid:: @dataclass class D: - x: List = [] + x: list = [] # This code raises ValueError def add(self, element): self.x += element diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 761f5f04b9b288..7889dd7d1c3ef0 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -1043,7 +1043,7 @@ Other constructors, all class methods: Return a :class:`.datetime` corresponding to *date_string*, parsed according to *format*. - This is equivalent to:: + If *format* does not contain microseconds or timezone information, this is equivalent to:: datetime(*(time.strptime(date_string, format)[0:6])) @@ -2510,10 +2510,7 @@ Notes: Because the format depends on the current locale, care should be taken when making assumptions about the output value. Field orderings will vary (for example, "month/day/year" versus "day/month/year"), and the output may - contain Unicode characters encoded using the locale's default encoding (for - example, if the current locale is ``ja_JP``, the default encoding could be - any one of ``eucJP``, ``SJIS``, or ``utf-8``; use :meth:`locale.getlocale` - to determine the current locale's encoding). + contain non-ASCII characters. (2) The :meth:`strptime` method can parse years in the full [1, 9999] range, but diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 6187098a752053..0b4a4973cb4da0 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -926,7 +926,7 @@ Each thread has its own current context which is accessed or changed using the You can also use the :keyword:`with` statement and the :func:`localcontext` function to temporarily change the active context. -.. function:: localcontext(ctx=None, \*\*kwargs) +.. function:: localcontext(ctx=None, **kwargs) Return a context manager that will set the current context for the active thread to a copy of *ctx* on entry to the with-statement and restore the previous context diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 19c08b225ef2bc..3894837127877c 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1036,6 +1036,24 @@ iterations of the loop. pushed to the stack before the attribute or unbound method respectively. +.. opcode:: LOAD_SUPER_ATTR (namei) + + This opcode implements :func:`super` (e.g. ``super().method()`` and + ``super().attr``). It works the same as :opcode:`LOAD_ATTR`, except that + ``namei`` is shifted left by 2 bits instead of 1, and instead of expecting a + single receiver on the stack, it expects three objects (from top of stack + down): ``self`` (the first argument to the current method), ``cls`` (the + class within which the current method was defined), and the global ``super``. + + The low bit of ``namei`` signals to attempt a method load, as with + :opcode:`LOAD_ATTR`. + + The second-low bit of ``namei``, if set, means that this was a two-argument + call to :func:`super` (unset means zero-argument). + + .. versionadded:: 3.12 + + .. opcode:: COMPARE_OP (opname) Performs a Boolean operation. The operation name can be found in diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index c690c837309ea5..582e06261afd72 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -119,7 +119,8 @@ Module Contents :func:`~enum.property` Allows :class:`Enum` members to have attributes without conflicting with - member names. + member names. The ``value`` and ``name`` attributes are implemented this + way. :func:`unique` @@ -169,7 +170,7 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. - .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) + .. method:: EnumType.__call__(cls, value, names=None, \*, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -317,7 +318,7 @@ Data Types >>> PowersOfThree.SECOND.value 9 - .. method:: Enum.__init_subclass__(cls, **kwds) + .. method:: Enum.__init_subclass__(cls, \**kwds) A *classmethod* that is used to further configure subsequent subclasses. By default, does nothing. @@ -696,7 +697,8 @@ Data Types .. attribute:: STRICT - Out-of-range values cause a :exc:`ValueError` to be raised:: + Out-of-range values cause a :exc:`ValueError` to be raised. This is the + default for :class:`Flag`:: >>> from enum import Flag, STRICT, auto >>> class StrictFlag(Flag, boundary=STRICT): @@ -714,7 +716,7 @@ Data Types .. attribute:: CONFORM Out-of-range values have invalid values removed, leaving a valid *Flag* - value. This is the default for :class:`Flag`:: + value:: >>> from enum import Flag, CONFORM, auto >>> class ConformFlag(Flag, boundary=CONFORM): diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 8797485cd05d83..7792e598c1155c 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1681,7 +1681,7 @@ are always available. They are listed here in alphabetical order. class C: @staticmethod - def f(arg1, arg2, ...): ... + def f(arg1, arg2, argN): ... The ``@staticmethod`` form is a function :term:`decorator` -- see :ref:`function` for details. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index d467e50bc7a424..29cbc87bf66d12 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -49,8 +49,13 @@ The :mod:`functools` module defines the following functions: >>> factorial(12) # makes two new recursive calls, the other 10 are cached 479001600 - The cache is threadsafe so the wrapped function can be used in multiple - threads. + The cache is threadsafe so that the wrapped function can be used in + multiple threads. This means that the underlying data structure will + remain coherent during concurrent updates. + + It is possible for the wrapped function to be called more than once if + another thread makes an additional call before the initial call has been + completed and cached. .. versionadded:: 3.9 @@ -118,6 +123,7 @@ The :mod:`functools` module defines the following functions: def stdev(self): return statistics.stdev(self._data) + .. versionadded:: 3.8 .. versionchanged:: 3.12 Prior to Python 3.12, ``cached_property`` included an undocumented lock to @@ -126,8 +132,6 @@ The :mod:`functools` module defines the following functions: per-instance, which could result in unacceptably high lock contention. In Python 3.12+ this locking is removed. - .. versionadded:: 3.8 - .. function:: cmp_to_key(func) @@ -159,8 +163,13 @@ The :mod:`functools` module defines the following functions: *maxsize* most recent calls. It can save time when an expensive or I/O bound function is periodically called with the same arguments. - The cache is threadsafe so the wrapped function can be used in multiple - threads. + The cache is threadsafe so that the wrapped function can be used in + multiple threads. This means that the underlying data structure will + remain coherent during concurrent updates. + + It is possible for the wrapped function to be called more than once if + another thread makes an additional call before the initial call has been + completed and cached. Since a dictionary is used to cache results, the positional and keyword arguments to the function must be :term:`hashable`. @@ -233,7 +242,7 @@ The :mod:`functools` module defines the following functions: @lru_cache(maxsize=32) def get_pep(num): 'Retrieve text of a Python Enhancement Proposal' - resource = 'https://peps.python.org/pep-%04d/' % num + resource = f'https://peps.python.org/pep-{num:04d}' try: with urllib.request.urlopen(resource) as s: return s.read() diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 832ebaf497f37a..6d5c64df1a1f3f 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -206,12 +206,17 @@ The :mod:`gc` module provides the following functions: .. function:: freeze() - Freeze all the objects tracked by gc - move them to a permanent generation - and ignore all the future collections. This can be used before a POSIX - fork() call to make the gc copy-on-write friendly or to speed up collection. - Also collection before a POSIX fork() call may free pages for future - allocation which can cause copy-on-write too so it's advised to disable gc - in parent process and freeze before fork and enable gc in child process. + Freeze all the objects tracked by the garbage collector; move them to a + permanent generation and ignore them in all the future collections. + + If a process will ``fork()`` without ``exec()``, avoiding unnecessary + copy-on-write in child processes will maximize memory sharing and reduce + overall memory usage. This requires both avoiding creation of freed "holes" + in memory pages in the parent process and ensuring that GC collections in + child processes won't touch the ``gc_refs`` counter of long-lived objects + originating in the parent process. To accomplish both, call ``gc.disable()`` + early in the parent process, ``gc.freeze()`` right before ``fork()``, and + ``gc.enable()`` early in child processes. .. versionadded:: 3.7 diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 6e084101995e25..b306d5f55a714f 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -308,6 +308,10 @@ Python module or `Import Package >> packages_distributions() {'importlib_metadata': ['importlib-metadata'], 'yaml': ['PyYAML'], 'jaraco': ['jaraco.classes', 'jaraco.functools'], ...} +Some editable installs, `do not supply top-level names +`_, and thus this +function is not reliable with such installs. + .. versionadded:: 3.10 .. _distributions: diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 70e5b7905f20a9..e57c393a6b370b 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -769,8 +769,8 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``iter_index()`` recipe is being tested to see -whether it proves its worth. +recipes. Currently, the ``sliding_window()`` and ``iter_index()`` recipes +are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from the `more-itertools project `_ found @@ -806,6 +806,23 @@ which incur interpreter overhead. "Return function(0), function(1), ..." return map(function, count(start)) + def repeatfunc(func, times=None, *args): + """Repeat calls to func with specified arguments. + + Example: repeatfunc(random.random) + """ + if times is None: + return starmap(func, repeat(args)) + return starmap(func, repeat(args, times)) + + def flatten(list_of_lists): + "Flatten one level of nesting" + return chain.from_iterable(list_of_lists) + + def ncycles(iterable, n): + "Returns the sequence elements n times" + return chain.from_iterable(repeat(tuple(iterable), n)) + def tail(n, iterable): "Return an iterator over the last n items" # tail(3, 'ABCDEFG') --> E F G @@ -825,70 +842,27 @@ which incur interpreter overhead. "Returns the nth item or a default value" return next(islice(iterable, n, None), default) - def all_equal(iterable): - "Returns True if all the elements are equal to each other" - g = groupby(iterable) - return next(g, True) and not next(g, False) - def quantify(iterable, pred=bool): "Count how many times the predicate is True" return sum(map(pred, iterable)) - def ncycles(iterable, n): - "Returns the sequence elements n times" - return chain.from_iterable(repeat(tuple(iterable), n)) - - def sum_of_squares(it): - "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) -> 1400 - return math.sumprod(*tee(it)) - - def transpose(it): - "Swap the rows and columns of the input." - # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) - return zip(*it, strict=True) - - def matmul(m1, m2): - "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]]) --> (49, 80), (41, 60) - n = len(m2[0]) - return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) - - def convolve(signal, kernel): - # See: https://betterexplained.com/articles/intuitive-convolution/ - # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1, -1]) --> 1st finite difference (1st derivative) - # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative) - kernel = tuple(kernel)[::-1] - n = len(kernel) - window = collections.deque([0], maxlen=n) * n - for x in chain(signal, repeat(0, n-1)): - window.append(x) - yield math.sumprod(kernel, window) + def all_equal(iterable): + "Returns True if all the elements are equal to each other" + g = groupby(iterable) + return next(g, True) and not next(g, False) - def polynomial_from_roots(roots): - """Compute a polynomial's coefficients from its roots. + def first_true(iterable, default=False, pred=None): + """Returns the first true value in the iterable. - (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 - """ - # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] - expansion = [1] - for r in roots: - expansion = convolve(expansion, (1, -r)) - return list(expansion) + If no true value is found, returns *default* - def polynomial_eval(coefficients, x): - """Evaluate a polynomial at a specific value. + If *pred* is not None, returns the first item + for which pred(item) is true. - Computes with better numeric stability than Horner's method. """ - # Evaluate x³ -4x² -17x + 60 at x = 2.5 - # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 - n = len(coefficients) - if n == 0: - return x * 0 # coerce zero to the type of x - powers = map(pow, repeat(x), reversed(range(n))) - return math.sumprod(coefficients, powers) + # first_true([a,b,c], x) --> a or b or c or x + # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x + return next(filter(pred, iterable), default) def iter_index(iterable, value, start=0): "Return indices where a value occurs in a sequence or iterable." @@ -913,44 +887,28 @@ which incur interpreter overhead. except ValueError: pass - def sieve(n): - "Primes less than n" - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 - data = bytearray((0, 1)) * (n // 2) - data[:3] = 0, 0, 0 - limit = math.isqrt(n) + 1 - for p in compress(range(limit), data): - data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - data[2] = 1 - return iter_index(data, 1) if n > 2 else iter([]) - - def factor(n): - "Prime factors of n." - # factor(99) --> 3 3 11 - for prime in sieve(math.isqrt(n) + 1): - while True: - quotient, remainder = divmod(n, prime) - if remainder: - break - yield prime - n = quotient - if n == 1: - return - if n > 1: - yield n + def iter_except(func, exception, first=None): + """ Call a function repeatedly until an exception is raised. - def flatten(list_of_lists): - "Flatten one level of nesting" - return chain.from_iterable(list_of_lists) + Converts a call-until-exception interface to an iterator interface. + Like builtins.iter(func, sentinel) but uses an exception instead + of a sentinel to end the loop. - def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. + Examples: + iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator + iter_except(d.popitem, KeyError) # non-blocking dict iterator + iter_except(d.popleft, IndexError) # non-blocking deque iterator + iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue + iter_except(s.pop, KeyError) # non-blocking set iterator - Example: repeatfunc(random.random) """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) + try: + if first is not None: + yield first() # For database APIs needing an initial cast to db.first() + while True: + yield func() + except exception: + pass def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks" @@ -967,12 +925,6 @@ which incur interpreter overhead. else: raise ValueError('Expected fill, strict, or ignore') - def triplewise(iterable): - "Return overlapping triplets from an iterable" - # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG - for (a, _), (b, c) in pairwise(pairwise(iterable)): - yield a, b, c - def sliding_window(iterable, n): # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG it = iter(iterable) @@ -1003,6 +955,12 @@ which incur interpreter overhead. t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) + def subslices(seq): + "Return all contiguous non-empty subslices of a sequence" + # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D + slices = starmap(slice, combinations(range(len(seq) + 1), 2)) + return map(operator.getitem, repeat(seq), slices) + def before_and_after(predicate, it): """ Variant of takewhile() that allows complete access to the remainder of the iterator. @@ -1032,17 +990,6 @@ which incur interpreter overhead. yield from it return true_iterator(), remainder_iterator() - def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" - # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D - slices = starmap(slice, combinations(range(len(seq) + 1), 2)) - return map(operator.getitem, repeat(seq), slices) - - def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." # unique_everseen('AAAABBBCCDAABBB') --> A B C D @@ -1072,41 +1019,96 @@ which incur interpreter overhead. # unique_justseen('ABBcCAD', str.lower) --> A B c A D return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. +The following recipes have a more mathematical flavor: - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator +.. testcode:: - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() + def powerset(iterable): + "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + s = list(iterable) + return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) + + def sieve(n): + "Primes less than n" + # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 + data = bytearray((0, 1)) * (n // 2) + data[:3] = 0, 0, 0 + limit = math.isqrt(n) + 1 + for p in compress(range(limit), data): + data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) + data[2] = 1 + return iter_index(data, 1) if n > 2 else iter([]) + + def factor(n): + "Prime factors of n." + # factor(99) --> 3 3 11 + for prime in sieve(math.isqrt(n) + 1): while True: - yield func() - except exception: - pass + quotient, remainder = divmod(n, prime) + if remainder: + break + yield prime + n = quotient + if n == 1: + return + if n > 1: + yield n - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. + def sum_of_squares(it): + "Add up the squares of the input values." + # sum_of_squares([10, 20, 30]) -> 1400 + return math.sumprod(*tee(it)) - If no true value is found, returns *default* + def transpose(it): + "Swap the rows and columns of the input." + # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) + return zip(*it, strict=True) - If *pred* is not None, returns the first item - for which pred(item) is true. + def matmul(m1, m2): + "Multiply two matrices." + # matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]]) --> (49, 80), (41, 60) + n = len(m2[0]) + return batched(starmap(math.sumprod, product(m1, transpose(m2))), n) + + def convolve(signal, kernel): + """Linear convolution of two iterables. + Article: https://betterexplained.com/articles/intuitive-convolution/ + Video: https://www.youtube.com/watch?v=KuXjwB4LzSA """ - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) + # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) + # convolve(data, [1, -1]) --> 1st finite difference (1st derivative) + # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative) + kernel = tuple(kernel)[::-1] + n = len(kernel) + padded_signal = chain(repeat(0, n-1), signal, [0] * (n-1)) + for window in sliding_window(padded_signal, n): + yield math.sumprod(kernel, window) + + def polynomial_from_roots(roots): + """Compute a polynomial's coefficients from its roots. + + (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 + """ + # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] + expansion = [1] + for r in roots: + expansion = convolve(expansion, (1, -r)) + return list(expansion) + + def polynomial_eval(coefficients, x): + """Evaluate a polynomial at a specific value. + + Computes with better numeric stability than Horner's method. + """ + # Evaluate x³ -4x² -17x + 60 at x = 2.5 + # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 + n = len(coefficients) + if n == 0: + return x * 0 # coerce zero to the type of x + powers = map(pow, repeat(x), reversed(range(n))) + return math.sumprod(coefficients, powers) def nth_combination(iterable, r, index): "Equivalent to list(combinations(iterable, r))[index]" @@ -1126,6 +1128,7 @@ which incur interpreter overhead. result.append(pool[-1-n]) return tuple(result) + .. doctest:: :hide: @@ -1401,9 +1404,6 @@ which incur interpreter overhead. >>> list(grouper('abcdefg', n=3, incomplete='ignore')) [('a', 'b', 'c'), ('d', 'e', 'f')] - >>> list(triplewise('ABCDEFG')) - [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] - >>> list(sliding_window('ABCDEFG', 4)) [('A', 'B', 'C', 'D'), ('B', 'C', 'D', 'E'), ('C', 'D', 'E', 'F'), ('D', 'E', 'F', 'G')] @@ -1485,3 +1485,45 @@ which incur interpreter overhead. >>> combos = list(combinations(iterable, r)) >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) True + + +.. testcode:: + :hide: + + # Old recipes and their tests which are guaranteed to continue to work. + + def sumprod(vec1, vec2): + "Compute a sum of products." + return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) + + def dotproduct(vec1, vec2): + return sum(map(operator.mul, vec1, vec2)) + + def pad_none(iterable): + """Returns the sequence elements and then returns None indefinitely. + + Useful for emulating the behavior of the built-in map() function. + """ + return chain(iterable, repeat(None)) + + def triplewise(iterable): + "Return overlapping triplets from an iterable" + # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG + for (a, _), (b, c) in pairwise(pairwise(iterable)): + yield a, b, c + + +.. doctest:: + :hide: + + >>> dotproduct([1,2,3], [4,5,6]) + 32 + + >>> sumprod([1,2,3], [4,5,6]) + 32 + + >>> list(islice(pad_none('abc'), 0, 6)) + ['a', 'b', 'c', None, None, None] + + >>> list(triplewise('ABCDEFG')) + [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 2daf2422ebd5b4..250246b5cd9adc 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -253,6 +253,7 @@ otherwise, the context is used to determine what to instantiate. * ``datefmt`` * ``style`` * ``validate`` (since version >=3.8) + * ``defaults`` (since version >=3.12) An optional ``class`` key indicates the name of the formatter's class (as a dotted module and class name). The instantiation @@ -953,16 +954,22 @@ Sections which specify formatter configuration are typified by the following. .. code-block:: ini [formatter_form01] - format=F1 %(asctime)s %(levelname)s %(message)s + format=F1 %(asctime)s %(levelname)s %(message)s %(customfield)s datefmt= style=% validate=True + defaults={'customfield': 'defaultvalue'} class=logging.Formatter The arguments for the formatter configuration are the same as the keys in the dictionary schema :ref:`formatters section `. +The ``defaults`` entry, when :ref:`evaluated ` in the context of +the ``logging`` package's namespace, is a dictionary of default values for +custom formatting fields. If not provided, it defaults to ``None``. + + .. note:: Due to the use of :func:`eval` as described above, there are diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 3e29fed0175e04..5c02d8bc8835bf 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -954,7 +954,16 @@ The canonical way to create an :class:`Option` instance is with the As you can see, most actions involve storing or updating a value somewhere. :mod:`optparse` always creates a special object for this, conventionally called -``options`` (it happens to be an instance of :class:`optparse.Values`). Option +``options``, which is an instance of :class:`optparse.Values`. + +.. class:: Values + + An object holding parsed argument names and values as attributes. + Normally created by calling when calling :meth:`OptionParser.parse_args`, + and can be overridden by a custom subclass passed to the *values* argument of + :meth:`OptionParser.parse_args` (as described in :ref:`optparse-parsing-arguments`). + +Option arguments (and various other values) are stored as attributes of this object, according to the :attr:`~Option.dest` (destination) option attribute. @@ -991,6 +1000,14 @@ one that makes sense for *all* options. Option attributes ^^^^^^^^^^^^^^^^^ +.. class:: Option + + A single command line argument, + with various attributes passed by keyword to the constructor. + Normally created with :meth:`OptionParser.add_option` rather than directly, + and can be overridden by a custom class via the *option_class* argument + to :class:`OptionParser`. + The following option attributes may be passed as keyword arguments to :meth:`OptionParser.add_option`. If you pass an option attribute that is not relevant to a particular option, or fail to pass a required option attribute, @@ -2027,7 +2044,7 @@ Features of note: values.ensure_value(attr, value) If the ``attr`` attribute of ``values`` doesn't exist or is ``None``, then - ensure_value() first sets it to ``value``, and then returns 'value. This is + ensure_value() first sets it to ``value``, and then returns ``value``. This is very handy for actions like ``"extend"``, ``"append"``, and ``"count"``, all of which accumulate data in a variable and expect that variable to be of a certain type (a list for the first two, an integer for the latter). Using @@ -2035,3 +2052,27 @@ Features of note: about setting a default value for the option destinations in question; they can just leave the default as ``None`` and :meth:`ensure_value` will take care of getting it right when it's needed. + +Exceptions +---------- + +.. exception:: OptionError + + Raised if an :class:`Option` instance is created with invalid or + inconsistent arguments. + +.. exception:: OptionConflictError + + Raised if conflicting options are added to an :class:`OptionParser`. + +.. exception:: OptionValueError + + Raised if an invalid option value is encountered on the command line. + +.. exception:: BadOptionError + + Raised if an invalid option is passed on the command line. + +.. exception:: AmbiguousOptionError + + Raised if an ambiguous option is passed on the command line. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7bb501c5946817..50e951c631fa88 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3919,7 +3919,8 @@ to be ignored. the :envvar:`PATH` variable. The other variants, :func:`execl`, :func:`execle`, :func:`execv`, and :func:`execve`, will not use the :envvar:`PATH` variable to locate the executable; *path* must contain an appropriate absolute or relative - path. + path. Relative paths must include at least one slash, even on Windows, as + plain names will not be resolved. For :func:`execle`, :func:`execlpe`, :func:`execve`, and :func:`execvpe` (note that these all end in "e"), the *env* parameter must be a mapping which is diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index d80c5eebbf27a7..5bc48a6d5f77fd 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -36,73 +36,91 @@ extension interface uses the modules :mod:`bdb` and :mod:`cmd`. Module :mod:`traceback` Standard interface to extract, format and print stack traces of Python programs. -The debugger's prompt is ``(Pdb)``. Typical usage to run a program under control -of the debugger is:: +The typical usage to break into the debugger is to insert:: - >>> import pdb - >>> import mymodule - >>> pdb.run('mymodule.test()') - > (0)?() - (Pdb) continue - > (1)?() + import pdb; pdb.set_trace() + +Or:: + + breakpoint() + +at the location you want to break into the debugger, and then run the program. +You can then step through the code following this statement, and continue +running without the debugger using the :pdbcmd:`continue` command. + +.. versionadded:: 3.7 + The built-in :func:`breakpoint()`, when called with defaults, can be used + instead of ``import pdb; pdb.set_trace()``. + +:: + + def double(x): + breakpoint() + return x * 2 + val = 3 + print(f"{val} * 2 is {double(val)}") + +The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: + + > ...(3)double() + -> return x * 2 + (Pdb) p x + 3 (Pdb) continue - NameError: 'spam' - > (1)?() - (Pdb) + 3 * 2 is 6 .. versionchanged:: 3.3 Tab-completion via the :mod:`readline` module is available for commands and command arguments, e.g. the current global and local names are offered as arguments of the ``p`` command. -:file:`pdb.py` can also be invoked as a script to debug other scripts. For + +You can also invoke :mod:`pdb` from the command line to debug other scripts. For example:: python -m pdb myscript.py -When invoked as a script, pdb will automatically enter post-mortem debugging if +When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post-mortem debugging (or after normal exit of the program), pdb will restart the program. Automatic restarting preserves pdb's state (such as breakpoints) and in most cases is more useful than quitting the debugger upon program's exit. .. versionadded:: 3.2 - :file:`pdb.py` now accepts a ``-c`` option that executes commands as if given + ``-c`` option is introduced to execute commands as if given in a :file:`.pdbrc` file, see :ref:`debugger-commands`. .. versionadded:: 3.7 - :file:`pdb.py` now accepts a ``-m`` option that execute modules similar to the way + ``-m`` option is introduced to execute modules similar to the way ``python -m`` does. As with a script, the debugger will pause execution just before the first line of the module. +Typical usage to execute a statement under control of the debugger is:: -The typical usage to break into the debugger is to insert:: - - import pdb; pdb.set_trace() - -at the location you want to break into the debugger, and then run the program. -You can then step through the code following this statement, and continue -running without the debugger using the :pdbcmd:`continue` command. - -.. versionadded:: 3.7 - The built-in :func:`breakpoint()`, when called with defaults, can be used - instead of ``import pdb; pdb.set_trace()``. + >>> import pdb + >>> def f(x): + ... print(1 / x) + >>> pdb.run("f(2)") + > (1)() + (Pdb) continue + 0.5 + >>> The typical usage to inspect a crashed program is:: >>> import pdb - >>> import mymodule - >>> mymodule.test() + >>> def f(x): + ... print(1 / x) + ... + >>> f(0) Traceback (most recent call last): File "", line 1, in - File "./mymodule.py", line 4, in test - test2() - File "./mymodule.py", line 3, in test2 - print(spam) - NameError: spam + File "", line 2, in f + ZeroDivisionError: division by zero >>> pdb.pm() - > ./mymodule.py(3)test2() - -> print(spam) + > (2)f() + (Pdb) p x + 0 (Pdb) @@ -275,7 +293,7 @@ can be overridden by the local file. .. pdbcommand:: w(here) - Print a stack trace, with the most recent frame at the bottom. An arrow + Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) indicates the current frame, which determines the context of most commands. .. pdbcommand:: d(own) [count] @@ -315,14 +333,14 @@ can be overridden by the local file. With a space separated list of breakpoint numbers, clear those breakpoints. Without argument, clear all breaks (but first ask confirmation). -.. pdbcommand:: disable [bpnumber ...] +.. pdbcommand:: disable bpnumber [bpnumber ...] Disable the breakpoints given as a space separated list of breakpoint numbers. Disabling a breakpoint means it cannot cause the program to stop execution, but unlike clearing a breakpoint, it remains in the list of breakpoints and can be (re-)enabled. -.. pdbcommand:: enable [bpnumber ...] +.. pdbcommand:: enable bpnumber [bpnumber ...] Enable the breakpoints specified. @@ -442,7 +460,7 @@ can be overridden by the local file. .. pdbcommand:: a(rgs) - Print the argument list of the current function. + Print the arguments of the current function and their current values. .. pdbcommand:: p expression @@ -476,6 +494,50 @@ can be overridden by the local file. Without *expression*, list all display expressions for the current frame. + .. note:: + + Display evaluates *expression* and compares to the result of the previous + evaluation of *expression*, so when the result is mutable, display may not + be able to pick up the changes. + + Example:: + + lst = [] + breakpoint() + pass + lst.append(1) + print(lst) + + Display won't realize ``lst`` has been changed because the result of evaluation + is modified in place by ``lst.append(1)`` before being compared:: + + > example.py(3)() + -> pass + (Pdb) display lst + display lst: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + (Pdb) + + You can do some tricks with copy mechanism to make it work:: + + > example.py(3)() + -> pass + (Pdb) display lst[:] + display lst[:]: [] + (Pdb) n + > example.py(4)() + -> lst.append(1) + (Pdb) n + > example.py(5)() + -> print(lst) + display lst[:]: [1] [old: []] + (Pdb) + .. versionadded:: 3.2 .. pdbcommand:: undisplay [expression] @@ -552,7 +614,7 @@ can be overridden by the local file. .. pdbcommand:: retval - Print the return value for the last return of a function. + Print the return value for the last return of the current function. .. rubric:: Footnotes diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 788a02dcb8922f..64e617b82b48bc 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -25,9 +25,9 @@ support. from pkgutil import extend_path __path__ = extend_path(__path__, __name__) - This will add to the package's ``__path__`` all subdirectories of directories - on :data:`sys.path` named after the package. This is useful if one wants to - distribute different parts of a single logical package as multiple + For each directory on :data:`sys.path` that has a subdirectory that matches the + package name, add the subdirectory to the package's :attr:`__path__`. This is useful + if one wants to distribute different parts of a single logical package as multiple directories. It also looks for :file:`\*.pkg` files beginning where ``*`` matches the @@ -82,7 +82,7 @@ support. This is a backwards compatibility wrapper around :func:`importlib.util.find_spec` that converts most failures to :exc:`ImportError` and only returns the loader rather than the full - :class:`ModuleSpec`. + :class:`importlib.machinery.ModuleSpec`. .. versionchanged:: 3.3 Updated to be based directly on :mod:`importlib` rather than relying diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index 4d485d25b54020..8fb0eca8df74d8 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -19,7 +19,7 @@ function. Readline keybindings may be configured via an initialization file, typically ``.inputrc`` in your home directory. See `Readline Init File -`_ +`_ in the GNU Readline manual for information about the format and allowable constructs of that file, and the capabilities of the Readline library in general. diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index a051c65b97b05e..04215d31ba10ca 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -36,7 +36,7 @@ scheduler: Example:: >>> import sched, time - >>> s = sched.scheduler(time.time, time.sleep) + >>> s = sched.scheduler(time.monotonic, time.sleep) >>> def print_time(a='default'): ... print("From print_time", time.time(), a) ... diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 373cc7d6072031..7f408be2336824 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -662,7 +662,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Remove the archive format *name* from the list of supported formats. -.. function:: unpack_archive(filename[, extract_dir[, format]]) +.. function:: unpack_archive(filename[, extract_dir[, format[, filter]]]) Unpack an archive. *filename* is the full path of the archive. @@ -676,6 +676,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. registered for that extension. In case none is found, a :exc:`ValueError` is raised. + The keyword-only *filter* argument is passed to the underlying unpacking + function. For zip files, *filter* is not accepted. + For tar files, it is recommended to set it to ``'data'``, + unless using features specific to tar and UNIX-like filesystems. + (See :ref:`tarfile-extraction-filter` for details.) + The ``'data'`` filter will become the default for tar files + in Python 3.14. + .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive .. warning:: @@ -688,6 +696,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. .. versionchanged:: 3.7 Accepts a :term:`path-like object` for *filename* and *extract_dir*. + .. versionchanged:: 3.12 + Added the *filter* argument. + .. function:: register_unpack_format(name, extensions, function[, extra_args[, description]]) Registers an unpack format. *name* is the name of the format and @@ -695,11 +706,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. ``.zip`` for Zip files. *function* is the callable that will be used to unpack archives. The - callable will receive the path of the archive, followed by the directory - the archive must be extracted to. - - When provided, *extra_args* is a sequence of ``(name, value)`` tuples that - will be passed as keywords arguments to the callable. + callable will receive: + + - the path of the archive, as a positional argument; + - the directory the archive must be extracted to, as a positional argument; + - possibly a *filter* keyword argument, if it was given to + :func:`unpack_archive`; + - additional keyword arguments, specified by *extra_args* as a sequence + of ``(name, value)`` tuples. *description* can be provided to describe the format, and will be returned by the :func:`get_unpack_formats` function. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index ceb962e860042d..d65e9fe81acf8b 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -140,9 +140,16 @@ server is the address family. ForkingUDPServer ThreadingTCPServer ThreadingUDPServer + ForkingUnixStreamServer + ForkingUnixDatagramServer + ThreadingUnixStreamServer + ThreadingUnixDatagramServer These classes are pre-defined using the mix-in classes. +.. versionadded:: 3.12 + The ``ForkingUnixStreamServer`` and ``ForkingUnixDatagramServer`` classes + were added. To implement a service, you must derive a class from :class:`BaseRequestHandler` and redefine its :meth:`~BaseRequestHandler.handle` method. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index bcfc6e5cfce611..2360472b31f175 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1605,8 +1605,8 @@ expression support in the :mod:`re` module). converts it to ``"ss"``. The casefolding algorithm is - `described in section 3.13 of the Unicode Standard - `__. + `described in section 3.13 'Default Case Folding' of the Unicode Standard + `__. .. versionadded:: 3.3 @@ -1768,8 +1768,9 @@ expression support in the :mod:`re` module). one character, ``False`` otherwise. Alphabetic characters are those characters defined in the Unicode character database as "Letter", i.e., those with general category property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different - from the `Alphabetic property defined in the Unicode Standard - `_. + from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and + Ideographic' of the Unicode Standard + `_. .. method:: str.isascii() @@ -1904,8 +1905,8 @@ expression support in the :mod:`re` module). lowercase. The lowercasing algorithm used is - `described in section 3.13 of the Unicode Standard - `__. + `described in section 3.13 'Default Case Folding' of the Unicode Standard + `__. .. method:: str.lstrip([chars]) @@ -2250,8 +2251,8 @@ expression support in the :mod:`re` module). titlecase). The uppercasing algorithm used is - `described in section 3.13 of the Unicode Standard - `__. + `described in section 3.13 'Default Case Folding' of the Unicode Standard + `__. .. method:: str.zfill(width) @@ -3714,12 +3715,15 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. - ``len(view)`` is equal to the length of :class:`~memoryview.tolist`. - If ``view.ndim = 0``, the length is 1. If ``view.ndim = 1``, the length - is equal to the number of elements in the view. For higher dimensions, - the length is equal to the length of the nested list representation of - the view. The :class:`~memoryview.itemsize` attribute will give you the - number of bytes in a single element. + ``len(view)`` is equal to the length of :class:`~memoryview.tolist`, which + is the nested list representation of the view. If ``view.ndim = 1``, + this is equal to the number of elements in the view. + + .. versionchanged:: 3.12 + If ``view.ndim == 0``, ``len(view)`` now raises :exc:`TypeError` instead of returning 1. + + The :class:`~memoryview.itemsize` attribute will give you the number of + bytes in a single element. A :class:`memoryview` supports slicing and indexing to expose its data. One-dimensional slicing will result in a subview:: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 5ada827328188d..26b3f5000634f5 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -235,7 +235,7 @@ dictionary keys (e.g., the strings ``'10'`` or ``':-]'``) within a format string The *arg_name* can be followed by any number of index or attribute expressions. An expression of the form ``'.name'`` selects the named attribute using :func:`getattr`, while an expression of the form ``'[index]'`` -does an index lookup using :func:`__getitem__`. +does an index lookup using :meth:`~object.__getitem__`. .. versionchanged:: 3.1 The positional argument specifiers can be omitted for :meth:`str.format`, @@ -254,10 +254,10 @@ Some simple format string examples:: "Units destroyed: {players[0]}" # First element of keyword argument 'players'. The *conversion* field causes a type coercion before formatting. Normally, the -job of formatting a value is done by the :meth:`__format__` method of the value +job of formatting a value is done by the :meth:`~object.__format__` method of the value itself. However, in some cases it is desirable to force a type to be formatted as a string, overriding its own definition of formatting. By converting the -value to a string before calling :meth:`__format__`, the normal formatting logic +value to a string before calling :meth:`~object.__format__`, the normal formatting logic is bypassed. Three conversion flags are currently supported: ``'!s'`` which calls :func:`str` diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 2b5a82e0107fb6..53dfbf827260c9 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -919,9 +919,12 @@ Reassigning them to new values is unsupported: .. attribute:: Popen.returncode - The child return code, set by :meth:`poll` and :meth:`wait` (and indirectly - by :meth:`communicate`). A ``None`` value indicates that the process - hasn't terminated yet. + The child return code. Initially ``None``, :attr:`returncode` is set by + a call to the :meth:`poll`, :meth:`wait`, or :meth:`communicate` methods + if they detect that the process has terminated. + + A ``None`` value indicates that the process hadn't yet terminated at the + time of the last method call. A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 00721efd1cf65e..7324f3113e0a08 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -220,6 +220,10 @@ always available. .. audit-event:: sys._current_exceptions "" sys._current_exceptions + .. versionchanged:: 3.12 + Each value in the dictionary is now a single exception instance, rather + than a 3-tuple as returned from ``sys.exc_info()``. + .. function:: breakpointhook() This hook function is called by built-in :func:`breakpoint`. By default, @@ -666,6 +670,13 @@ always available. .. versionadded:: 3.4 +.. function:: getunicodeinternedsize() + + Return the number of unicode objects that have been interned. + + .. versionadded:: 3.12 + + .. function:: getandroidapilevel() Return the build time API version of Android as an integer. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 741d40da152101..891af1bcf7edff 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -36,6 +36,13 @@ Some facts and figures: .. versionchanged:: 3.3 Added support for :mod:`lzma` compression. +.. versionchanged:: 3.12 + Archives are extracted using a :ref:`filter `, + which makes it possible to either limit surprising/dangerous features, + or to acknowledge that they are expected and the archive is fully trusted. + By default, archives are fully trusted, but this default is deprecated + and slated to change in Python 3.14. + .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) @@ -209,6 +216,38 @@ The :mod:`tarfile` module defines the following exceptions: Is raised by :meth:`TarInfo.frombuf` if the buffer it gets is invalid. +.. exception:: FilterError + + Base class for members :ref:`refused ` by + filters. + + .. attribute:: tarinfo + + Information about the member that the filter refused to extract, + as :ref:`TarInfo `. + +.. exception:: AbsolutePathError + + Raised to refuse extracting a member with an absolute path. + +.. exception:: OutsideDestinationError + + Raised to refuse extracting a member outside the destination directory. + +.. exception:: SpecialFileError + + Raised to refuse extracting a special file (e.g. a device or pipe). + +.. exception:: AbsoluteLinkError + + Raised to refuse extracting a symbolic link with an absolute path. + +.. exception:: LinkOutsideDestinationError + + Raised to refuse extracting a symbolic link pointing outside the destination + directory. + + The following constants are available at the module level: .. data:: ENCODING @@ -319,11 +358,8 @@ be finalized; only the internally used file object will be closed. See the *debug* can be set from ``0`` (no debug messages) up to ``3`` (all debug messages). The messages are written to ``sys.stderr``. - If *errorlevel* is ``0``, all errors are ignored when using :meth:`TarFile.extract`. - Nevertheless, they appear as error messages in the debug output, when debugging - is enabled. If ``1``, all *fatal* errors are raised as :exc:`OSError` - exceptions. If ``2``, all *non-fatal* errors are raised as :exc:`TarError` - exceptions as well. + *errorlevel* controls how extraction errors are handled, + see :attr:`the corresponding attribute <~TarFile.errorlevel>`. The *encoding* and *errors* arguments define the character encoding to be used for reading or writing the archive and how conversion errors are going @@ -390,7 +426,7 @@ be finalized; only the internally used file object will be closed. See the available. -.. method:: TarFile.extractall(path=".", members=None, *, numeric_owner=False) +.. method:: TarFile.extractall(path=".", members=None, *, numeric_owner=False, filter=None) Extract all members from the archive to the current working directory or directory *path*. If optional *members* is given, it must be a subset of the @@ -404,6 +440,12 @@ be finalized; only the internally used file object will be closed. See the are used to set the owner/group for the extracted files. Otherwise, the named values from the tarfile are used. + The *filter* argument specifies how ``members`` are modified or rejected + before extraction. + See :ref:`tarfile-extraction-filter` for details. + It is recommended to set this explicitly depending on which *tar* features + you need to support. + .. warning:: Never extract archives from untrusted sources without prior inspection. @@ -411,14 +453,20 @@ be finalized; only the internally used file object will be closed. See the that have absolute filenames starting with ``"/"`` or filenames with two dots ``".."``. + Set ``filter='data'`` to prevent the most dangerous security issues, + and read the :ref:`tarfile-extraction-filter` section for details. + .. versionchanged:: 3.5 Added the *numeric_owner* parameter. .. versionchanged:: 3.6 The *path* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.12 + Added the *filter* parameter. -.. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False) + +.. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False, filter=None) Extract a member from the archive to the current working directory, using its full name. Its file information is extracted as accurately as possible. *member* @@ -426,9 +474,8 @@ be finalized; only the internally used file object will be closed. See the directory using *path*. *path* may be a :term:`path-like object`. File attributes (owner, mtime, mode) are set unless *set_attrs* is false. - If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile - are used to set the owner/group for the extracted files. Otherwise, the named - values from the tarfile are used. + The *numeric_owner* and *filter* arguments are the same as + for :meth:`extractall`. .. note:: @@ -439,6 +486,9 @@ be finalized; only the internally used file object will be closed. See the See the warning for :meth:`extractall`. + Set ``filter='data'`` to prevent the most dangerous security issues, + and read the :ref:`tarfile-extraction-filter` section for details. + .. versionchanged:: 3.2 Added the *set_attrs* parameter. @@ -448,6 +498,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.6 The *path* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.12 + Added the *filter* parameter. + .. method:: TarFile.extractfile(member) @@ -460,6 +513,55 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. +.. attribute:: TarFile.errorlevel + :type: int + + If *errorlevel* is ``0``, errors are ignored when using :meth:`TarFile.extract` + and :meth:`TarFile.extractall`. + Nevertheless, they appear as error messages in the debug output when + *debug* is greater than 0. + If ``1`` (the default), all *fatal* errors are raised as :exc:`OSError` or + :exc:`FilterError` exceptions. If ``2``, all *non-fatal* errors are raised + as :exc:`TarError` exceptions as well. + + Some exceptions, e.g. ones caused by wrong argument types or data + corruption, are always raised. + + Custom :ref:`extraction filters ` + should raise :exc:`FilterError` for *fatal* errors + and :exc:`ExtractError` for *non-fatal* ones. + + Note that when an exception is raised, the archive may be partially + extracted. It is the user’s responsibility to clean up. + +.. attribute:: TarFile.extraction_filter + + .. versionadded:: 3.12 + + The :ref:`extraction filter ` used + as a default for the *filter* argument of :meth:`~TarFile.extract` + and :meth:`~TarFile.extractall`. + + The attribute may be ``None`` or a callable. + String names are not allowed for this attribute, unlike the *filter* + argument to :meth:`~TarFile.extract`. + + If ``extraction_filter`` is ``None`` (the default), + calling an extraction method without a *filter* argument will raise a + ``DeprecationWarning``, + and fall back to the :func:`fully_trusted ` filter, + whose dangerous behavior matches previous versions of Python. + + In Python 3.14+, leaving ``extraction_filter=None`` will cause + extraction methods to use the :func:`data ` filter by default. + + The attribute may be set on instances or overridden in subclasses. + It also is possible to set it on the ``TarFile`` class itself to set a + global default, although, since it affects all uses of *tarfile*, + it is best practice to only do so in top-level applications or + :mod:`site configuration `. + To set a global default this way, a filter function needs to be wrapped in + :func:`staticmethod()` to prevent injection of a ``self`` argument. .. method:: TarFile.add(name, arcname=None, recursive=True, *, filter=None) @@ -535,8 +637,23 @@ permissions, owner etc.), it provides some useful methods to determine its type. It does *not* contain the file's data itself. :class:`TarInfo` objects are returned by :class:`TarFile`'s methods -:meth:`getmember`, :meth:`getmembers` and :meth:`gettarinfo`. +:meth:`~TarFile.getmember`, :meth:`~TarFile.getmembers` and +:meth:`~TarFile.gettarinfo`. +Modifying the objects returned by :meth:`~!TarFile.getmember` or +:meth:`~!TarFile.getmembers` will affect all subsequent +operations on the archive. +For cases where this is unwanted, you can use :mod:`copy.copy() ` or +call the :meth:`~TarInfo.replace` method to create a modified copy in one step. + +Several attributes can be set to ``None`` to indicate that a piece of metadata +is unused or unknown. +Different :class:`TarInfo` methods handle ``None`` differently: + +- The :meth:`~TarFile.extract` or :meth:`~TarFile.extractall` methods will + ignore the corresponding metadata, leaving it set to a default. +- :meth:`~TarFile.addfile` will fail. +- :meth:`~TarFile.list` will print a placeholder string. .. class:: TarInfo(name="") @@ -569,24 +686,39 @@ A ``TarInfo`` object has the following public data attributes: .. attribute:: TarInfo.name + :type: str Name of the archive member. .. attribute:: TarInfo.size + :type: int Size in bytes. .. attribute:: TarInfo.mtime + :type: int | float - Time of last modification. + Time of last modification in seconds since the :ref:`epoch `, + as in :attr:`os.stat_result.st_mtime`. + + .. versionchanged:: 3.12 + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.mode + :type: int - Permission bits. + Permission bits, as for :func:`os.chmod`. + .. versionchanged:: 3.12 + + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.type @@ -598,35 +730,76 @@ A ``TarInfo`` object has the following public data attributes: .. attribute:: TarInfo.linkname + :type: str Name of the target file name, which is only present in :class:`TarInfo` objects of type :const:`LNKTYPE` and :const:`SYMTYPE`. .. attribute:: TarInfo.uid + :type: int User ID of the user who originally stored this member. + .. versionchanged:: 3.12 + + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.gid + :type: int Group ID of the user who originally stored this member. + .. versionchanged:: 3.12 + + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.uname + :type: str User name. + .. versionchanged:: 3.12 + + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.gname + :type: str Group name. + .. versionchanged:: 3.12 + + Can be set to ``None`` for :meth:`~TarFile.extract` and + :meth:`~TarFile.extractall`, causing extraction to skip applying this + attribute. .. attribute:: TarInfo.pax_headers + :type: dict A dictionary containing key-value pairs of an associated pax extended header. +.. method:: TarInfo.replace(name=..., mtime=..., mode=..., linkname=..., + uid=..., gid=..., uname=..., gname=..., + deep=True) + + .. versionadded:: 3.12 + + Return a *new* copy of the :class:`!TarInfo` object with the given attributes + changed. For example, to return a ``TarInfo`` with the group name set to + ``'staff'``, use:: + + new_tarinfo = old_tarinfo.replace(gname='staff') + + By default, a deep copy is made. + If *deep* is false, the copy is shallow, i.e. ``pax_headers`` + and any custom attributes are shared with the original ``TarInfo`` object. A :class:`TarInfo` object also provides some convenient query methods: @@ -676,9 +849,258 @@ A :class:`TarInfo` object also provides some convenient query methods: Return :const:`True` if it is one of character device, block device or FIFO. +.. _tarfile-extraction-filter: + +Extraction filters +------------------ + +.. versionadded:: 3.12 + +The *tar* format is designed to capture all details of a UNIX-like filesystem, +which makes it very powerful. +Unfortunately, the features make it easy to create tar files that have +unintended -- and possibly malicious -- effects when extracted. +For example, extracting a tar file can overwrite arbitrary files in various +ways (e.g. by using absolute paths, ``..`` path components, or symlinks that +affect later members). + +In most cases, the full functionality is not needed. +Therefore, *tarfile* supports extraction filters: a mechanism to limit +functionality, and thus mitigate some of the security issues. + +.. seealso:: + + :pep:`706` + Contains further motivation and rationale behind the design. + +The *filter* argument to :meth:`TarFile.extract` or :meth:`~TarFile.extractall` +can be: + +* the string ``'fully_trusted'``: Honor all metadata as specified in the + archive. + Should be used if the user trusts the archive completely, or implements + their own complex verification. + +* the string ``'tar'``: Honor most *tar*-specific features (i.e. features of + UNIX-like filesystems), but block features that are very likely to be + surprising or malicious. See :func:`tar_filter` for details. + +* the string ``'data'``: Ignore or block most features specific to UNIX-like + filesystems. Intended for extracting cross-platform data archives. + See :func:`data_filter` for details. + +* ``None`` (default): Use :attr:`TarFile.extraction_filter`. + + If that is also ``None`` (the default), raise a ``DeprecationWarning``, + and fall back to the ``'fully_trusted'`` filter, whose dangerous behavior + matches previous versions of Python. + + In Python 3.14, the ``'data'`` filter will become the default instead. + It's possible to switch earlier; see :attr:`TarFile.extraction_filter`. + +* A callable which will be called for each extracted member with a + :ref:`TarInfo ` describing the member and the destination + path to where the archive is extracted (i.e. the same path is used for all + members):: + + filter(/, member: TarInfo, path: str) -> TarInfo | None + + The callable is called just before each member is extracted, so it can + take the current state of the disk into account. + It can: + + - return a :class:`TarInfo` object which will be used instead of the metadata + in the archive, or + - return ``None``, in which case the member will be skipped, or + - raise an exception to abort the operation or skip the member, + depending on :attr:`~TarFile.errorlevel`. + Note that when extraction is aborted, :meth:`~TarFile.extractall` may leave + the archive partially extracted. It does not attempt to clean up. + +Default named filters +~~~~~~~~~~~~~~~~~~~~~ + +The pre-defined, named filters are available as functions, so they can be +reused in custom filters: + +.. function:: fully_trusted_filter(/, member, path) + + Return *member* unchanged. + + This implements the ``'fully_trusted'`` filter. + +.. function:: tar_filter(/, member, path) + + Implements the ``'tar'`` filter. + + - Strip leading slashes (``/`` and :attr:`os.sep`) from filenames. + - :ref:`Refuse ` to extract files with absolute + paths (in case the name is absolute + even after stripping slashes, e.g. ``C:/foo`` on Windows). + This raises :class:`~tarfile.AbsolutePathError`. + - :ref:`Refuse ` to extract files whose absolute + path (after following symlinks) would end up outside the destination. + This raises :class:`~tarfile.OutsideDestinationError`. + - Clear high mode bits (setuid, setgid, sticky) and group/other write bits + (:attr:`~stat.S_IWGRP`|:attr:`~stat.S_IWOTH`). + + Return the modified ``TarInfo`` member. + +.. function:: data_filter(/, member, path) + + Implements the ``'data'`` filter. + In addition to what ``tar_filter`` does: + + - :ref:`Refuse ` to extract links (hard or soft) + that link to absolute paths, or ones that link outside the destination. + + This raises :class:`~tarfile.AbsoluteLinkError` or + :class:`~tarfile.LinkOutsideDestinationError`. + + Note that such files are refused even on platforms that do not support + symbolic links. + + - :ref:`Refuse ` to extract device files + (including pipes). + This raises :class:`~tarfile.SpecialFileError`. + + - For regular files, including hard links: + + - Set the owner read and write permissions + (:attr:`~stat.S_IRUSR`|:attr:`~stat.S_IWUSR`). + - Remove the group & other executable permission + (:attr:`~stat.S_IXGRP`|:attr:`~stat.S_IXOTH`) + if the owner doesn’t have it (:attr:`~stat.S_IXUSR`). + + - For other files (directories), set ``mode`` to ``None``, so + that extraction methods skip applying permission bits. + - Set user and group info (``uid``, ``gid``, ``uname``, ``gname``) + to ``None``, so that extraction methods skip setting it. + + Return the modified ``TarInfo`` member. + + +.. _tarfile-extraction-refuse: + +Filter errors +~~~~~~~~~~~~~ + +When a filter refuses to extract a file, it will raise an appropriate exception, +a subclass of :class:`~tarfile.FilterError`. +This will abort the extraction if :attr:`TarFile.errorlevel` is 1 or more. +With ``errorlevel=0`` the error will be logged and the member will be skipped, +but extraction will continue. + + +Hints for further verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Even with ``filter='data'``, *tarfile* is not suited for extracting untrusted +files without prior inspection. +Among other issues, the pre-defined filters do not prevent denial-of-service +attacks. Users should do additional checks. + +Here is an incomplete list of things to consider: + +* Extract to a :func:`new temporary directory ` + to prevent e.g. exploiting pre-existing links, and to make it easier to + clean up after a failed extraction. +* When working with untrusted data, use external (e.g. OS-level) limits on + disk, memory and CPU usage. +* Check filenames against an allow-list of characters + (to filter out control characters, confusables, foreign path separators, + etc.). +* Check that filenames have expected extensions (discouraging files that + execute when you “click on them”, or extension-less files like Windows special device names). +* Limit the number of extracted files, total size of extracted data, + filename length (including symlink length), and size of individual files. +* Check for files that would be shadowed on case-insensitive filesystems. + +Also note that: + +* Tar files may contain multiple versions of the same file. + Later ones are expected to overwrite any earlier ones. + This feature is crucial to allow updating tape archives, but can be abused + maliciously. +* *tarfile* does not protect against issues with “live” data, + e.g. an attacker tinkering with the destination (or source) directory while + extraction (or archiving) is in progress. + + +Supporting older Python versions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Extraction filters were added to Python 3.12, but may be backported to older +versions as security updates. +To check whether the feature is available, use e.g. +``hasattr(tarfile, 'data_filter')`` rather than checking the Python version. + +The following examples show how to support Python versions with and without +the feature. +Note that setting ``extraction_filter`` will affect any subsequent operations. + +* Fully trusted archive:: + + my_tarfile.extraction_filter = (lambda member, path: member) + my_tarfile.extractall() + +* Use the ``'data'`` filter if available, but revert to Python 3.11 behavior + (``'fully_trusted'``) if this feature is not available:: + + my_tarfile.extraction_filter = getattr(tarfile, 'data_filter', + (lambda member, path: member)) + my_tarfile.extractall() + +* Use the ``'data'`` filter; *fail* if it is not available:: + + my_tarfile.extractall(filter=tarfile.data_filter) + + or:: + + my_tarfile.extraction_filter = tarfile.data_filter + my_tarfile.extractall() + +* Use the ``'data'`` filter; *warn* if it is not available:: + + if hasattr(tarfile, 'data_filter'): + my_tarfile.extractall(filter='data') + else: + # remove this when no longer needed + warn_the_user('Extracting may be unsafe; consider updating Python') + my_tarfile.extractall() + + +Stateful extraction filter example +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +While *tarfile*'s extraction methods take a simple *filter* callable, +custom filters may be more complex objects with an internal state. +It may be useful to write these as context managers, to be used like this:: + + with StatefulFilter() as filter_func: + tar.extractall(path, filter=filter_func) + +Such a filter can be written as, for example:: + + class StatefulFilter: + def __init__(self): + self.file_count = 0 + + def __enter__(self): + return self + + def __call__(self, member, path): + self.file_count += 1 + return member + + def __exit__(self, *exc_info): + print(f'{self.file_count} files extracted') + + .. _tarfile-commandline: .. program:: tarfile + Command-Line Interface ---------------------- @@ -748,6 +1170,13 @@ Command-line options Verbose output. +.. cmdoption:: --filter + + Specifies the *filter* for ``--extract``. + See :ref:`tarfile-extraction-filter` for details. + Only string names are accepted (that is, ``fully_trusted``, ``tar``, + and ``data``). + .. _tar-examples: Examples @@ -757,7 +1186,7 @@ How to extract an entire tar archive to the current working directory:: import tarfile tar = tarfile.open("sample.tar.gz") - tar.extractall() + tar.extractall(filter='data') tar.close() How to extract a subset of a tar archive with :meth:`TarFile.extractall` using diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index 61358eb76925b2..fd4c294613fd31 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -292,6 +292,9 @@ The module defines the following user-callable items: .. versionchanged:: 3.6 The *dir* parameter now accepts a :term:`path-like object`. + .. versionchanged:: 3.12 + :func:`mkdtemp` now always returns an absolute path, even if *dir* is relative. + .. function:: gettempdir() diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc index 2739d5bfc1dfa2..3b345099bf54b5 100644 --- a/Doc/library/token-list.inc +++ b/Doc/library/token-list.inc @@ -201,6 +201,10 @@ Token value for ``":="``. +.. data:: EXCLAMATION + + Token value for ``"!"``. + .. data:: OP .. data:: AWAIT @@ -213,6 +217,12 @@ .. data:: SOFT_KEYWORD +.. data:: FSTRING_START + +.. data:: FSTRING_MIDDLE + +.. data:: FSTRING_END + .. data:: ERRORTOKEN .. data:: N_TOKENS diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 747ba58bb225d4..a15fb5cfa49473 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -75,13 +75,53 @@ Dynamic Type Creation This function looks for items in *bases* that are not instances of :class:`type`, and returns a tuple where each such object that has - an ``__mro_entries__`` method is replaced with an unpacked result of + an :meth:`~object.__mro_entries__` method is replaced with an unpacked result of calling this method. If a *bases* item is an instance of :class:`type`, - or it doesn't have an ``__mro_entries__`` method, then it is included in + or it doesn't have an :meth:`!__mro_entries__` method, then it is included in the return tuple unchanged. .. versionadded:: 3.7 +.. function:: get_original_bases(cls, /) + + Return the tuple of objects originally given as the bases of *cls* before + the :meth:`~object.__mro_entries__` method has been called on any bases + (following the mechanisms laid out in :pep:`560`). This is useful for + introspecting :ref:`Generics `. + + For classes that have an ``__orig_bases__`` attribute, this + function returns the value of ``cls.__orig_bases__``. + For classes without the ``__orig_bases__`` attribute, ``cls.__bases__`` is + returned. + + Examples:: + + from typing import TypeVar, Generic, NamedTuple, TypedDict + + T = TypeVar("T") + class Foo(Generic[T]): ... + class Bar(Foo[int], float): ... + class Baz(list[str]): ... + Eggs = NamedTuple("Eggs", [("a", int), ("b", str)]) + Spam = TypedDict("Spam", {"a": int, "b": str}) + + assert Bar.__bases__ == (Foo, float) + assert get_original_bases(Bar) == (Foo[int], float) + + assert Baz.__bases__ == (list,) + assert get_original_bases(Baz) == (list[str],) + + assert Eggs.__bases__ == (tuple,) + assert get_original_bases(Eggs) == (NamedTuple,) + + assert Spam.__bases__ == (dict,) + assert get_original_bases(Spam) == (TypedDict,) + + assert int.__bases__ == (object,) + assert get_original_bases(int) == (object,) + + .. versionadded:: 3.12 + .. seealso:: :pep:`560` - Core support for typing module and generic types @@ -311,6 +351,13 @@ Standard names are defined for the following types: .. versionchanged:: 3.9.2 This type can now be subclassed. + .. seealso:: + + :ref:`Generic Alias Types` + In-depth documentation on instances of :class:`!types.GenericAlias` + + :pep:`585` - Type Hinting Generics In Standard Collections + Introducing the :class:`!types.GenericAlias` class .. class:: UnionType diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 03ff259bbdf75f..15bab7775eadd8 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -423,7 +423,7 @@ to this is that a list of types can be used to substitute a :class:`ParamSpec`:: >>> class Z(Generic[T, P]): ... ... >>> Z[int, [dict, float]] - __main__.Z[int, (, )] + __main__.Z[int, [dict, float]] Furthermore, a generic with only one parameter specification variable will accept @@ -434,9 +434,9 @@ to the former, so the following are equivalent:: >>> class X(Generic[P]): ... ... >>> X[int, str] - __main__.X[(, )] + __main__.X[[int, str]] >>> X[[int, str]] - __main__.X[(, )] + __main__.X[[int, str]] Do note that generics with :class:`ParamSpec` may not have correct ``__parameters__`` after substitution in some cases because they diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index d1a977fd7da6a5..a7c74cfa4fb477 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2191,10 +2191,6 @@ Loading and running tests .. versionadded:: 3.12 Added *durations* keyword argument. - .. versionchanged:: 3.12 - Subclasses should accept ``**kwargs`` to ensure compatibility as the - interface changes. - .. data:: defaultTestLoader Instance of the :class:`TestLoader` class intended to be shared. If no diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 240ab139838db9..52bf99e5bb0f67 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -284,11 +284,14 @@ creation according to their needs, the :class:`EnvBuilder` class. .. method:: upgrade_dependencies(context) - Upgrades the core venv dependency packages (currently ``pip`` and - ``setuptools``) in the environment. This is done by shelling out to the + Upgrades the core venv dependency packages (currently ``pip``) + in the environment. This is done by shelling out to the ``pip`` executable in the environment. .. versionadded:: 3.9 + .. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. .. method:: post_setup(context) diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index e2a085d6e98e67..6f4826cb065c64 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -7,7 +7,7 @@ .. moduleauthor:: James C. Ahlstrom .. sectionauthor:: James C. Ahlstrom -**Source code:** :source:`Lib/zipfile.py` +**Source code:** :source:`Lib/zipfile/` -------------- diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index ed9ebaba784bc2..55431f1951e50d 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -2089,16 +2089,25 @@ Resolving MRO entries .. method:: object.__mro_entries__(self, bases) If a base that appears in a class definition is not an instance of - :class:`type`, then an ``__mro_entries__`` method is searched on the base. - If an ``__mro_entries__`` method is found, the base is substituted with the - result of a call to ``__mro_entries__`` when creating the class. - The method is called with the original bases tuple, and must return a tuple + :class:`type`, then an :meth:`!__mro_entries__` method is searched on the base. + If an :meth:`!__mro_entries__` method is found, the base is substituted with the + result of a call to :meth:`!__mro_entries__` when creating the class. + The method is called with the original bases tuple + passed to the *bases* parameter, and must return a tuple of classes that will be used instead of the base. The returned tuple may be empty: in these cases, the original base is ignored. .. seealso:: - :pep:`560` - Core support for typing module and generic types + :func:`types.resolve_bases` + Dynamically resolve bases that are not instances of :class:`type`. + + :func:`types.get_original_bases` + Retrieve a class's "original bases" prior to modifications by + :meth:`~object.__mro_entries__`. + + :pep:`560` + Core support for typing module and generic types. Determining the appropriate metaclass diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index f6fe8df97810ff..f3350174f931aa 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -59,7 +59,6 @@ Doc/faq/gui.rst Doc/faq/library.rst Doc/faq/programming.rst Doc/glossary.rst -Doc/howto/argparse.rst Doc/howto/curses.rst Doc/howto/descriptor.rst Doc/howto/enum.rst @@ -78,7 +77,6 @@ Doc/library/__future__.rst Doc/library/_thread.rst Doc/library/abc.rst Doc/library/aifc.rst -Doc/library/argparse.rst Doc/library/ast.rst Doc/library/asyncio-dev.rst Doc/library/asyncio-eventloop.rst @@ -113,7 +111,6 @@ Doc/library/csv.rst Doc/library/ctypes.rst Doc/library/curses.ascii.rst Doc/library/curses.rst -Doc/library/dataclasses.rst Doc/library/datetime.rst Doc/library/dbm.rst Doc/library/decimal.rst @@ -180,7 +177,6 @@ Doc/library/os.rst Doc/library/ossaudiodev.rst Doc/library/pickle.rst Doc/library/pickletools.rst -Doc/library/pkgutil.rst Doc/library/platform.rst Doc/library/plistlib.rst Doc/library/poplib.rst diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 2a4d070ec057df..b35e8454fa2a1a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -495,7 +495,8 @@ Miscellaneous options Reserved for various implementation-specific options. CPython currently defines the following possible values: - * ``-X faulthandler`` to enable :mod:`faulthandler`; + * ``-X faulthandler`` to enable :mod:`faulthandler`. + See also :envvar:`PYTHONFAULTHANDLER`. * ``-X showrefcount`` to output the total reference count and number of used memory blocks when the program finishes or after each statement in the interactive interpreter. This only works on :ref:`debug builds @@ -503,8 +504,9 @@ Miscellaneous options * ``-X tracemalloc`` to start tracing Python memory allocations using the :mod:`tracemalloc` module. By default, only the most recent frame is stored in a traceback of a trace. Use ``-X tracemalloc=NFRAME`` to start - tracing with a traceback limit of *NFRAME* frames. See the - :func:`tracemalloc.start` for more information. + tracing with a traceback limit of *NFRAME* frames. + See :func:`tracemalloc.start` and :envvar:`PYTHONTRACEMALLOC` + for more information. * ``-X int_max_str_digits`` configures the :ref:`integer string conversion length limitation `. See also :envvar:`PYTHONINTMAXSTRDIGITS`. @@ -519,6 +521,7 @@ Miscellaneous options * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` (even when it would otherwise activate automatically). + See also :envvar:`PYTHONUTF8`. * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel tree rooted at the given directory instead of to the code tree. See also :envvar:`PYTHONPYCACHEPREFIX`. @@ -861,7 +864,9 @@ conflict. Python memory allocations using the :mod:`tracemalloc` module. The value of the variable is the maximum number of frames stored in a traceback of a trace. For example, ``PYTHONTRACEMALLOC=1`` stores only the most recent - frame. See the :func:`tracemalloc.start` for more information. + frame. + See the :func:`tracemalloc.start` function for more information. + This is equivalent to setting the :option:`-X` ``tracemalloc`` option. .. versionadded:: 3.4 @@ -869,8 +874,8 @@ conflict. .. envvar:: PYTHONPROFILEIMPORTTIME If this environment variable is set to a non-empty string, Python will - show how long each import takes. This is exactly equivalent to setting - ``-X importtime`` on the command line. + show how long each import takes. + This is equivalent to setting the :option:`-X` ``importtime`` option. .. versionadded:: 3.7 @@ -1012,6 +1017,7 @@ conflict. If this environment variable is set to a non-empty string, enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by default. + This is equivalent to setting the :option:`-X` ``dev`` option. .. versionadded:: 3.7 diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 067ff4cce5e48d..0044eb07f56eec 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -54,13 +54,6 @@ On FreeBSD and OpenBSD pkg_add ftp://ftp.openbsd.org/pub/OpenBSD/4.2/packages/i386/python-2.5.1p2.tgz -On OpenSolaris --------------- - -You can get Python from `OpenCSW `_. Various versions -of Python are available and can be installed with e.g. ``pkgutil -i python27``. - - .. _building-python-on-unix: Building Python diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 43ee6b7807d57e..2fc90126482268 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -61,12 +61,16 @@ The command, if run with ``-h``, will show the available options:: environment (pip is bootstrapped by default) --prompt PROMPT Provides an alternative prompt prefix for this environment. - --upgrade-deps Upgrade core dependencies: pip setuptools to the + --upgrade-deps Upgrade core dependencies (pip) to the latest version in PyPI Once an environment has been created, you may wish to activate it, e.g. by sourcing an activate script in its bin directory. +.. versionchanged:: 3.12 + + ``setuptools`` is no longer a core venv dependency. + .. versionchanged:: 3.9 Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI @@ -104,4 +108,3 @@ invoked to bootstrap ``pip`` into the virtual environment. Multiple paths can be given to ``venv``, in which case an identical virtual environment will be created, according to the given options, at each provided path. - diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 1c4e41c0e0e239..380950eb507ffb 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -470,7 +470,7 @@ user's system, including environment variables, system registry settings, and installed packages. The standard library is included as pre-compiled and optimized ``.pyc`` files in a ZIP, and ``python3.dll``, ``python37.dll``, ``python.exe`` and ``pythonw.exe`` are all provided. Tcl/tk (including all -dependants, such as Idle), pip and the Python documentation are not included. +dependents, such as Idle), pip and the Python documentation are not included. .. note:: diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 34f2656f765c7d..4ee2aacb108a36 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -172,7 +172,7 @@ this edition of "What's New in Python" links to the bug/patch item for each change. Hosting of the Python bug tracker is kindly provided by -`Upfront Systems `__ +`Upfront Systems `__ of Stellenbosch, South Africa. Martin von Löwis put a lot of effort into importing existing bugs and patches from SourceForge; his scripts for this import operation are at diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 810a2cd2537c34..36afcb163f1afc 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -2104,7 +2104,7 @@ Changes to Python's build process and to the C API include: * The latest release of the GNU Debugger, GDB 7, can be `scripted using Python - `__. + `__. When you begin debugging an executable program P, GDB will look for a file named ``P-gdb.py`` and automatically read it. Dave Malcolm contributed a :file:`python-gdb.py` that adds a number of diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 66e40ef7326cc0..9a32f985c6a05a 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -137,6 +137,13 @@ New Features (Design by Pablo Galindo. Contributed by Pablo Galindo and Christian Heimes with contributions from Gregory P. Smith [Google] and Mark Shannon in :gh:`96123`.) +* The extraction methods in :mod:`tarfile`, and :func:`shutil.unpack_archive`, + have a new a *filter* argument that allows limiting tar features than may be + surprising or dangerous, such as creating files outside the destination + directory. + See :ref:`tarfile-extraction-filter` for details. + In Python 3.14, the default will switch to ``'data'``. + (Contributed by Petr Viktorin in :pep:`706`.) Other Language Changes @@ -192,6 +199,10 @@ Other Language Changes * :class:`slice` objects are now hashable, allowing them to be used as dict keys and set items. (Contributed by Will Bradshaw and Furkan Onder in :gh:`101264`.) +* Exceptions raised in a typeobject's ``__set_name__`` method are no longer + wrapped by a :exc:`RuntimeError`. Context information is added to the + exception as a :pep:`678` note. (Contributed by Irit Katriel in :gh:`77757`.) + New Modules =========== @@ -210,6 +221,11 @@ array asyncio ------- +* The performance of writing to sockets in :mod:`asyncio` has been + significantly improved. ``asyncio`` now avoids unnecessary copying when + writing to sockets and uses :meth:`~socket.socket.sendmsg` if the platform + supports it. (Contributed by Kumar Aditya in :gh:`91166`.) + * On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default if :func:`os.pidfd_open` is available and functional instead of :class:`~asyncio.ThreadedChildWatcher`. @@ -241,9 +257,17 @@ asyncio :mod:`asyncio` does not support legacy generator-based coroutines. (Contributed by Kumar Aditya in :gh:`102748`.) -* :func:`asyncio.wait` now accepts generators yielding tasks. +* :func:`asyncio.wait` and :func:`asyncio.as_completed` now accepts generators + yielding tasks. (Contributed by Kumar Aditya in :gh:`78530`.) +csv +--- + +* Add :data:`~csv.QUOTE_NOTNULL` and :data:`~csv.QUOTE_STRINGS` flags to + provide finer grained control of ``None`` and empty strings by + :class:`~csv.writer` objects. + inspect ------- @@ -292,6 +316,13 @@ fractions * Objects of type :class:`fractions.Fraction` now support float-style formatting. (Contributed by Mark Dickinson in :gh:`100161`.) +itertools +--------- + +* Added :class:`itertools.batched()` for collecting into even-sized + tuples where the last batch may be shorter than the rest. + (Contributed by Raymond Hettinger in :gh:`98363`.) + math ---- @@ -383,6 +414,13 @@ threading profiling functions in all running threads in addition to the calling one. (Contributed by Pablo Galindo in :gh:`93503`.) +types +----- + +* Add :func:`types.get_original_bases` to allow for further introspection of + :ref:`user-defined-generics` when subclassed. (Contributed by + James Hilton-Balfe and Alex Waygood in :gh:`101827`.) + unicodedata ----------- @@ -419,8 +457,10 @@ uuid tempfile -------- -The :class:`tempfile.NamedTemporaryFile` function has a new optional parameter -*delete_on_close* (Contributed by Evgeny Zorin in :gh:`58451`.) +* The :class:`tempfile.NamedTemporaryFile` function has a new optional parameter + *delete_on_close* (Contributed by Evgeny Zorin in :gh:`58451`.) +* :func:`tempfile.mkdtemp` now always returns an absolute path, even if the + argument provided to the *dir* parameter is a relative path. .. _whatsnew-typing-py312: @@ -494,6 +534,10 @@ sys :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback`. (Contributed by Irit Katriel in :gh:`102778`.) +* :func:`sys._current_exceptions` now returns a mapping from thread-id to an + exception instance, rather than to a ``(typ, exc, tb)`` tuple. + (Contributed by Irit Katriel in :gh:`103176`.) + Optimizations ============= @@ -595,6 +639,10 @@ Deprecated * The *onerror* argument of :func:`shutil.rmtree` is deprecated as will be removed in Python 3.14. Use *onexc* instead. (Contributed by Irit Katriel in :gh:`102828`.) +* Extracting tar archives without specifying *filter* is deprecated until + Python 3.14, when ``'data'`` filter will become the default. + See :ref:`tarfile-extraction-filter` for details. + Pending Removal in Python 3.13 ------------------------------ @@ -711,6 +759,24 @@ Removed project can be installed: it still provides ``distutils``. (Contributed by Victor Stinner in :gh:`92584`.) +* Remove the bundled setuptools wheel from :mod:`ensurepip`, + and stop installing setuptools in environments created by :mod:`venv`. + + ``pip (>= 22.1)`` does not require setuptools to be installed in the + environment. ``setuptools``-based (and ``distutils``-based) packages + can still be used with ``pip install``, since pip will provide + ``setuptools`` in the build environment it uses for building a + package. + + ``easy_install``, ``pkg_resources``, ``setuptools`` and ``distutils`` + are no longer provided by default in environments created with + ``venv`` or bootstrapped with ``ensurepip``, since they are part of + the ``setuptools`` package. For projects relying on these at runtime, + the ``setuptools`` project should be declared as a dependency and + installed separately (typically, using pip). + + (Contributed by Pradyun Gedam in :gh:`95299`.) + * Removed many old deprecated :mod:`unittest` features: - A number of :class:`~unittest.TestCase` method aliases: @@ -935,6 +1001,14 @@ Changes in the Python API synchronization is needed, implement locking within the cached property getter function or around multi-threaded access points. +* :func:`sys._current_exceptions` now returns a mapping from thread-id to an + exception instance, rather than to a ``(typ, exc, tb)`` tuple. + (Contributed by Irit Katriel in :gh:`103176`.) + +* When extracting tar files using :mod:`tarfile` or + :func:`shutil.unpack_archive`, pass the *filter* argument to limit features + that may be surprising or dangerous. + See :ref:`tarfile-extraction-filter` for details. Build Changes ============= @@ -1079,6 +1153,24 @@ New Features to replace the legacy-api :c:func:`!PyErr_Display`. (Contributed by Irit Katriel in :gh:`102755`). +* :pep:`683`: Introduced Immortal Objects to Python which allows objects + to bypass reference counts and introduced changes to the C-API: + + - ``_Py_IMMORTAL_REFCNT``: The reference count that defines an object + as immortal. + - ``_Py_IsImmortal`` Checks if an object has the immortal reference count. + - ``PyObject_HEAD_INIT`` This will now initialize reference count to + ``_Py_IMMORTAL_REFCNT`` when used with ``Py_BUILD_CORE``. + - ``SSTATE_INTERNED_IMMORTAL`` An identifier for interned unicode objects + that are immortal. + - ``SSTATE_INTERNED_IMMORTAL_STATIC`` An identifier for interned unicode + objects that are immortal and static + - ``sys.getunicodeinternedsize`` This returns the total number of unicode + objects that have been interned. This is now needed for refleak.py to + correctly track reference counts and allocated blocks + + (Contributed by Eddie Elizondo in :gh:`84436`.) + Porting to Python 3.12 ---------------------- @@ -1243,8 +1335,7 @@ Removed * :c:func:`!PyUnicode_GetSize` * :c:func:`!PyUnicode_GET_DATA_SIZE` -* Remove the ``PyUnicode_InternImmortal()`` function and the - ``SSTATE_INTERNED_IMMORTAL`` macro. +* Remove the ``PyUnicode_InternImmortal()`` function macro. (Contributed by Victor Stinner in :gh:`85858`.) * Remove ``Jython`` compatibility hacks from several stdlib modules and tests. diff --git a/Grammar/Tokens b/Grammar/Tokens index 1f3e3b09913653..096876fdd130f8 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -53,6 +53,7 @@ ATEQUAL '@=' RARROW '->' ELLIPSIS '...' COLONEQUAL ':=' +EXCLAMATION '!' OP AWAIT @@ -60,6 +61,9 @@ ASYNC TYPE_IGNORE TYPE_COMMENT SOFT_KEYWORD +FSTRING_START +FSTRING_MIDDLE +FSTRING_END ERRORTOKEN # These aren't used by the C tokenizer but are needed for tokenize.py diff --git a/Grammar/python.gram b/Grammar/python.gram index 6ab09de4fc149a..0851c7c8a8b8ba 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -195,7 +195,7 @@ yield_stmt[stmt_ty]: y=yield_expr { _PyAST_Expr(y, EXTRA) } assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _PyAST_Assert(a, b, EXTRA) } -import_stmt[stmt_ty]: +import_stmt[stmt_ty]: | invalid_import | import_name | import_from @@ -416,8 +416,8 @@ try_stmt[stmt_ty]: | invalid_try_stmt | 'try' &&':' b=block f=finally_block { _PyAST_Try(b, NULL, NULL, f, EXTRA) } | 'try' &&':' b=block ex[asdl_excepthandler_seq*]=except_block+ el=[else_block] f=[finally_block] { _PyAST_Try(b, ex, el, f, EXTRA) } - | 'try' &&':' b=block ex[asdl_excepthandler_seq*]=except_star_block+ el=[else_block] f=[finally_block] { - CHECK_VERSION(stmt_ty, 11, "Exception groups are", + | 'try' &&':' b=block ex[asdl_excepthandler_seq*]=except_star_block+ el=[else_block] f=[finally_block] { + CHECK_VERSION(stmt_ty, 11, "Exception groups are", _PyAST_TryStar(b, ex, el, f, EXTRA)) } @@ -830,7 +830,7 @@ atom[expr_ty]: | 'True' { _PyAST_Constant(Py_True, NULL, EXTRA) } | 'False' { _PyAST_Constant(Py_False, NULL, EXTRA) } | 'None' { _PyAST_Constant(Py_None, NULL, EXTRA) } - | &STRING strings + | &(STRING|FSTRING_START) strings | NUMBER | &'(' (tuple | group | genexp) | &'[' (list | listcomp) @@ -900,7 +900,26 @@ lambda_param[arg_ty]: a=NAME { _PyAST_arg(a->v.Name.id, NULL, NULL, EXTRA) } # LITERALS # ======== -strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) } +fstring_middle[expr_ty]: + | fstring_replacement_field + | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } +fstring_replacement_field[expr_ty]: + | '{' a=(yield_expr | star_expressions) debug_expr="="? conversion=[fstring_conversion] format=[fstring_full_format_spec] '}' { + _PyPegen_formatted_value(p, a, debug_expr, conversion, format, EXTRA) + } + | invalid_replacement_field +fstring_conversion[expr_ty]: + | conv_token="!" conv=NAME { _PyPegen_check_fstring_conversion(p, conv_token, conv) } +fstring_full_format_spec[expr_ty]: + | ':' spec=fstring_format_spec* { spec ? _PyAST_JoinedStr((asdl_expr_seq*)spec, EXTRA) : NULL } +fstring_format_spec[expr_ty]: + | t=FSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } + | fstring_replacement_field +fstring[expr_ty]: + | a=FSTRING_START b=fstring_middle* c=FSTRING_END { _PyPegen_joined_str(p, a, (asdl_expr_seq*)b, c) } + +string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) } +strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } @@ -1141,6 +1160,8 @@ invalid_expression: _PyPegen_check_legacy_stmt(p, a) ? NULL : p->tokens[p->mark-1]->level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "invalid syntax. Perhaps you forgot a comma?") } | a=disjunction 'if' b=disjunction !('else'|':') { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "expected 'else' after 'if' expression") } + | a='lambda' [lambda_params] b=':' &(FSTRING_MIDDLE | fstring_replacement_field) { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "f-string: lambda expressions are not allowed without parentheses") } invalid_named_expression(memo): | a=expression ':=' expression { @@ -1264,7 +1285,7 @@ invalid_group: invalid_import: | a='import' dotted_name 'from' dotted_name { RAISE_SYNTAX_ERROR_STARTING_FROM(a, "Did you mean to use 'from ... import ...' instead?") } - + invalid_import_from_targets: | import_from_as_names ',' NEWLINE { RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } @@ -1358,3 +1379,24 @@ invalid_kvpair: | expression a=':' &('}'|',') {RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "expression expected after dictionary key and ':'") } invalid_starred_expression: | a='*' expression '=' b=expression { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "cannot assign to iterable argument unpacking") } +invalid_replacement_field: + | '{' a='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '='") } + | '{' a='!' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '!'") } + | '{' a=':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before ':'") } + | '{' a='}' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "f-string: valid expression required before '}'") } + | '{' !(yield_expr | star_expressions) { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting a valid expression after '{'")} + | '{' (yield_expr | star_expressions) !('=' | '!' | ':' | '}') { + PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '=', or '!', or ':', or '}'") } + | '{' (yield_expr | star_expressions) '=' !('!' | ':' | '}') { + PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '!', or ':', or '}'") } + | '{' (yield_expr | star_expressions) '='? invalid_conversion_character + | '{' (yield_expr | star_expressions) '='? ['!' NAME] !(':' | '}') { + PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting ':' or '}'") } + | '{' (yield_expr | star_expressions) '='? ['!' NAME] ':' fstring_format_spec* !'}' { + PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}', or format specs") } + | '{' (yield_expr | star_expressions) '='? ['!' NAME] !'}' { + PyErr_Occurred() ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: expecting '}'") } + +invalid_conversion_character: + | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: missing conversion character") } + | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("f-string: invalid conversion character") } diff --git a/Include/boolobject.h b/Include/boolobject.h index ca21fbfad8e827..976fa35201d035 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -11,8 +11,7 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type) -/* Py_False and Py_True are the only two bools in existence. -Don't forget to apply Py_INCREF() when returning either!!! */ +/* Py_False and Py_True are the only two bools in existence. */ /* Don't use these directly */ PyAPI_DATA(PyLongObject) _Py_FalseStruct; @@ -31,8 +30,8 @@ PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) /* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_NewRef(Py_True) -#define Py_RETURN_FALSE return Py_NewRef(Py_False) +#define Py_RETURN_TRUE return Py_True +#define Py_RETURN_FALSE return Py_False /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/cpython/code.h b/Include/cpython/code.h index abcf1250603dfe..6bead361c79245 100644 --- a/Include/cpython/code.h +++ b/Include/cpython/code.h @@ -3,10 +3,22 @@ #ifndef Py_LIMITED_API #ifndef Py_CODE_H #define Py_CODE_H + #ifdef __cplusplus extern "C" { #endif + +/* Count of all "real" monitoring events (not derived from other events) */ +#define PY_MONITORING_UNGROUPED_EVENTS 14 +/* Count of all monitoring events */ +#define PY_MONITORING_EVENTS 16 + +/* Table of which tools are active for each monitored event. */ +typedef struct _Py_Monitors { + uint8_t tools[PY_MONITORING_UNGROUPED_EVENTS]; +} _Py_Monitors; + /* Each instruction in a code object is a fixed-width value, * currently 2 bytes: 1-byte opcode + 1-byte oparg. The EXTENDED_ARG * opcode allows for larger values but the current limit is 3 uses @@ -56,6 +68,35 @@ typedef struct { PyObject *_co_freevars; } _PyCoCached; +/* Ancilliary data structure used for instrumentation. + Line instrumentation creates an array of + these. One entry per code unit.*/ +typedef struct { + uint8_t original_opcode; + int8_t line_delta; +} _PyCoLineInstrumentationData; + +/* Main data structure used for instrumentation. + * This is allocated when needed for instrumentation + */ +typedef struct { + /* Monitoring specific to this code object */ + _Py_Monitors local_monitors; + /* Monitoring that is active on this code object */ + _Py_Monitors active_monitors; + /* The tools that are to be notified for events for the matching code unit */ + uint8_t *tools; + /* Information to support line events */ + _PyCoLineInstrumentationData *lines; + /* The tools that are to be notified for line events for the matching code unit */ + uint8_t *line_tools; + /* Information to support instruction events */ + /* The underlying instructions, which can themselves be instrumented */ + uint8_t *per_instruction_opcodes; + /* The tools that are to be notified for instruction events for the matching code unit */ + uint8_t *per_instruction_tools; +} _PyCoMonitoringData; + // To avoid repeating ourselves in deepfreeze.py, all PyCodeObject members are // defined in this macro: #define _PyCode_DEF(SIZE) { \ @@ -87,7 +128,6 @@ typedef struct { PyObject *co_exceptiontable; /* Byte string encoding exception handling \ table */ \ int co_flags; /* CO_..., see below */ \ - short _co_linearray_entry_size; /* Size of each entry in _co_linearray */ \ \ /* The rest are not so impactful on performance. */ \ int co_argcount; /* #arguments, except *args */ \ @@ -114,8 +154,9 @@ typedef struct { PyObject *co_linetable; /* bytes object that holds location info */ \ PyObject *co_weakreflist; /* to support weakrefs to code objects */ \ _PyCoCached *_co_cached; /* cached co_* attributes */ \ + uint64_t _co_instrumentation_version; /* current instrumentation version */ \ + _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \ int _co_firsttraceable; /* index of first traceable instruction */ \ - char *_co_linearray; /* array of line offsets */ \ /* Scratch space for extra data relating to the code object. \ Type is a void* to keep the format private in codeobject.c to force \ people to go through the proper APIs. */ \ diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 8bc681b1a93f5c..79c1023baa9a0f 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -245,6 +245,8 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, /* --- PyInterpreterConfig ------------------------------------ */ typedef struct { + // XXX "allow_object_sharing"? "own_objects"? + int use_main_obmalloc; int allow_fork; int allow_exec; int allow_threads; @@ -254,6 +256,7 @@ typedef struct { #define _PyInterpreterConfig_INIT \ { \ + .use_main_obmalloc = 0, \ .allow_fork = 0, \ .allow_exec = 0, \ .allow_threads = 1, \ @@ -263,6 +266,7 @@ typedef struct { #define _PyInterpreterConfig_LEGACY_INIT \ { \ + .use_main_obmalloc = 1, \ .allow_fork = 1, \ .allow_exec = 1, \ .allow_threads = 1, \ diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 98cc51cd7fee49..ce4d13cd9c28fe 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -564,3 +564,10 @@ PyAPI_FUNC(int) PyType_AddWatcher(PyType_WatchCallback callback); PyAPI_FUNC(int) PyType_ClearWatcher(int watcher_id); PyAPI_FUNC(int) PyType_Watch(int watcher_id, PyObject *type); PyAPI_FUNC(int) PyType_Unwatch(int watcher_id, PyObject *type); + +/* Attempt to assign a version tag to the given type. + * + * Returns 1 if the type already had a valid version tag or a new one was + * assigned, or 0 if a new tag could not be assigned. + */ +PyAPI_FUNC(int) PyUnstable_Type_AssignVersionTag(PyTypeObject *type); diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 3efb241e8237e7..f33c72d4cf4d2a 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -11,6 +11,10 @@ is available in a given context. For example, forking the process might not be allowed in the current interpreter (i.e. os.fork() would fail). */ +/* Set if the interpreter share obmalloc runtime state + with the main interpreter. */ +#define Py_RTFLAGS_USE_MAIN_OBMALLOC (1UL << 5) + /* Set if import should check a module for subinterpreter support. */ #define Py_RTFLAGS_MULTI_INTERP_EXTENSIONS (1UL << 8) @@ -58,12 +62,6 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_C_RETURN 6 #define PyTrace_OPCODE 7 - -typedef struct { - PyCodeObject *code; // The code object for the bounds. May be NULL. - PyCodeAddressRange bounds; // Only valid if code != NULL. -} PyTraceInfo; - // Internal structure: you should not use it directly, but use public functions // like PyThreadState_EnterTracing() and PyThreadState_LeaveTracing(). typedef struct _PyCFrame { @@ -77,7 +75,6 @@ typedef struct _PyCFrame { * discipline and make sure that instances of this struct cannot * accessed outside of their lifetime. */ - uint8_t use_tracing; // 0 or 255 (or'ed into opcode, hence 8-bit type) /* Pointer to the currently executing frame (it can be NULL) */ struct _PyInterpreterFrame *current_frame; struct _PyCFrame *previous; @@ -157,7 +154,7 @@ struct _ts { This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ int tracing; - int tracing_what; /* The event currently being traced, if any. */ + int what_event; /* The event currently being monitored, if any. */ /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ @@ -228,8 +225,6 @@ struct _ts { /* Unique thread state id. */ uint64_t id; - PyTraceInfo trace_info; - _PyStackChunk *datastack_chunk; PyObject **datastack_top; PyObject **datastack_limit; diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 75a74ffa2f9dff..3394726dfffd72 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -98,9 +98,16 @@ typedef struct { Py_ssize_t length; /* Number of code points in the string */ Py_hash_t hash; /* Hash value; -1 if not set */ struct { - /* If interned is set, the two references from the - dictionary to this object are *not* counted in ob_refcnt. */ - unsigned int interned:1; + /* If interned is non-zero, the two references from the + dictionary to this object are *not* counted in ob_refcnt. + The possible values here are: + 0: Not Interned + 1: Interned + 2: Interned and Immortal + 3: Interned, Immortal, and Static + This categorization allows the runtime to determine the right + cleanup mechanism at runtime shutdown. */ + unsigned int interned:2; /* Character size: - PyUnicode_1BYTE_KIND (1): @@ -135,7 +142,7 @@ typedef struct { unsigned int ascii:1; /* Padding to ensure that PyUnicode_DATA() is always aligned to 4 bytes (see issue #19537 on m68k). */ - unsigned int :26; + unsigned int :25; } state; } PyASCIIObject; @@ -183,6 +190,8 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( /* Interning state. */ #define SSTATE_NOT_INTERNED 0 #define SSTATE_INTERNED_MORTAL 1 +#define SSTATE_INTERNED_IMMORTAL 2 +#define SSTATE_INTERNED_IMMORTAL_STATIC 3 /* Use only if you know it's a string */ static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) { diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index faf1be585e17ef..7d5d5e03de9e41 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -51,6 +51,15 @@ typedef struct { #define INLINE_CACHE_ENTRIES_BINARY_SUBSCR CACHE_ENTRIES(_PyBinarySubscrCache) +typedef struct { + uint16_t counter; + uint16_t class_version[2]; + uint16_t self_type_version[2]; + uint16_t method[4]; +} _PySuperAttrCache; + +#define INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR CACHE_ENTRIES(_PySuperAttrCache) + typedef struct { uint16_t counter; uint16_t version[2]; @@ -217,6 +226,8 @@ extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); /* Specialization functions */ +extern void _Py_Specialize_LoadSuperAttr(PyObject *global_super, PyObject *class, PyObject *self, + _Py_CODEUNIT *instr, PyObject *name, int load_method); extern void _Py_Specialize_LoadAttr(PyObject *owner, _Py_CODEUNIT *instr, PyObject *name); extern void _Py_Specialize_StoreAttr(PyObject *owner, _Py_CODEUNIT *instr, @@ -441,32 +452,6 @@ adaptive_counter_backoff(uint16_t counter) { /* Line array cache for tracing */ -extern int _PyCode_CreateLineArray(PyCodeObject *co); - -static inline int -_PyCode_InitLineArray(PyCodeObject *co) -{ - if (co->_co_linearray) { - return 0; - } - return _PyCode_CreateLineArray(co); -} - -static inline int -_PyCode_LineNumberFromArray(PyCodeObject *co, int index) -{ - assert(co->_co_linearray != NULL); - assert(index >= 0); - assert(index < Py_SIZE(co)); - if (co->_co_linearray_entry_size == 2) { - return ((int16_t *)co->_co_linearray)[index]; - } - else { - assert(co->_co_linearray_entry_size == 4); - return ((int32_t *)co->_co_linearray)[index]; - } -} - typedef struct _PyShimCodeDef { const uint8_t *code; int codelen; @@ -500,6 +485,10 @@ extern uint32_t _Py_next_func_version; #define COMPARISON_NOT_EQUALS (COMPARISON_UNORDERED | COMPARISON_LESS_THAN | COMPARISON_GREATER_THAN) +extern int _Py_Instrument(PyCodeObject *co, PyInterpreterState *interp); + +extern int _Py_GetBaseOpcode(PyCodeObject *code, int offset); + #ifdef __cplusplus } diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index 6a1e02c0b895d7..f85240c48a89b0 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -33,6 +33,45 @@ extern int _PyAST_Optimize( struct _arena *arena, _PyASTOptimizeState *state); + +typedef struct { + int i_opcode; + int i_oparg; + _PyCompilerSrcLocation i_loc; +} _PyCompilerInstruction; + +typedef struct { + _PyCompilerInstruction *s_instrs; + int s_allocated; + int s_used; + + int *s_labelmap; /* label id --> instr offset */ + int s_labelmap_size; + int s_next_free_label; /* next free label id */ +} _PyCompile_InstructionSequence; + +typedef struct { + PyObject *u_name; + PyObject *u_qualname; /* dot-separated qualified name (lazy) */ + + /* The following fields are dicts that map objects to + the index of them in co_XXX. The index is used as + the argument for opcodes that refer to those collections. + */ + PyObject *u_consts; /* all constants */ + PyObject *u_names; /* all names */ + PyObject *u_varnames; /* local variables */ + PyObject *u_cellvars; /* cell variables */ + PyObject *u_freevars; /* free variables */ + + Py_ssize_t u_argcount; /* number of arguments for block */ + Py_ssize_t u_posonlyargcount; /* number of positional only arguments for block */ + Py_ssize_t u_kwonlyargcount; /* number of keyword only arguments for block */ + + int u_firstlineno; /* the first lineno of the block */ +} _PyCompile_CodeUnitMetadata; + + /* Utility for a number of growing arrays used in the compiler */ int _PyCompile_EnsureArrayLargeEnough( int idx, diff --git a/Include/internal/pycore_flowgraph.h b/Include/internal/pycore_flowgraph.h index 7c0b8fe980c3c7..f470dad3aaa459 100644 --- a/Include/internal/pycore_flowgraph.h +++ b/Include/internal/pycore_flowgraph.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_opcode_utils.h" +#include "pycore_compile.h" static const _PyCompilerSrcLocation NO_LOCATION = {-1, -1, -1, -1}; @@ -33,7 +34,8 @@ typedef struct { typedef struct _PyCfgBasicblock_ { /* Each basicblock in a compilation unit is linked via b_list in the reverse order that the block are allocated. b_list points to the next - block, not to be confused with b_next, which is next by control flow. */ + block in this list, not to be confused with b_next, which is next by + control flow. */ struct _PyCfgBasicblock_ *b_list; /* The label of this block if it is a jump target, -1 otherwise */ _PyCfgJumpTargetLabel b_label; @@ -91,10 +93,9 @@ void _PyCfgBuilder_Fini(_PyCfgBuilder *g); _PyCfgInstruction* _PyCfg_BasicblockLastInstr(const _PyCfgBasicblock *b); int _PyCfg_OptimizeCodeUnit(_PyCfgBuilder *g, PyObject *consts, PyObject *const_cache, - int code_flags, int nlocals, int nparams); + int code_flags, int nlocals, int nparams, int firstlineno); int _PyCfg_Stackdepth(_PyCfgBasicblock *entryblock, int code_flags); void _PyCfg_ConvertExceptionHandlersToNops(_PyCfgBasicblock *entryblock); -int _PyCfg_ResolveLineNumbers(_PyCfgBuilder *g, int firstlineno); int _PyCfg_ResolveJumps(_PyCfgBuilder *g); int _PyCfg_InstrSize(_PyCfgInstruction *instruction); @@ -110,6 +111,10 @@ basicblock_nofallthrough(const _PyCfgBasicblock *b) { #define BB_NO_FALLTHROUGH(B) (basicblock_nofallthrough(B)) #define BB_HAS_FALLTHROUGH(B) (!basicblock_nofallthrough(B)) +PyCodeObject * +_PyAssemble_MakeCodeObject(_PyCompile_CodeUnitMetadata *u, PyObject *const_cache, + PyObject *consts, int maxdepth, _PyCfgBasicblock *entryblock, + int nlocalsplus, int code_flags, PyObject *filename); #ifdef __cplusplus } diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 5806cf05f174a9..20d48d20362571 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -19,6 +19,7 @@ struct _frame { struct _PyInterpreterFrame *f_frame; /* points to the frame data */ PyObject *f_trace; /* Trace function */ int f_lineno; /* Current line number. Only valid if non-zero */ + int f_last_traced_line; /* The last line traced for this frame */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */ @@ -60,7 +61,13 @@ typedef struct _PyInterpreterFrame { // over, or (in the case of a newly-created frame) a totally invalid value: _Py_CODEUNIT *prev_instr; int stacktop; /* Offset of TOS from localsplus */ - uint16_t yield_offset; + /* The return_offset determines where a `RETURN` should go in the caller, + * relative to `prev_instr`. + * It is only meaningful to the callee, + * so it needs to be set in any CALL (to a Python function) + * or SEND (to a coroutine or generator). + * If there is no callee, then it is meaningless. */ + uint16_t return_offset; char owner; /* Locals and stack */ PyObject *localsplus[1]; @@ -120,7 +127,7 @@ _PyFrame_Initialize( frame->stacktop = code->co_nlocalsplus; frame->frame_obj = NULL; frame->prev_instr = _PyCode_CODE(code) - 1; - frame->yield_offset = 0; + frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { @@ -137,10 +144,16 @@ _PyFrame_GetLocalsArray(_PyInterpreterFrame *frame) return frame->localsplus; } +/* Fetches the stack pointer, and sets stacktop to -1. + Having stacktop <= 0 ensures that invalid + values are not visible to the cycle GC. + We choose -1 rather than 0 to assist debugging. */ static inline PyObject** _PyFrame_GetStackPointer(_PyInterpreterFrame *frame) { - return frame->localsplus+frame->stacktop; + PyObject **sp = frame->localsplus + frame->stacktop; + frame->stacktop = -1; + return sp; } static inline void diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 6474586ec4ab93..9afd78fdca8fa1 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -8,15 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_object.h" // _PyObject_IMMORTAL_REFCNT - #ifdef Py_DEBUG static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { - if (Py_REFCNT(obj) < _PyObject_IMMORTAL_REFCNT) { + if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { _PyObject_ASSERT_FAILED_MSG(obj, "immortal object has less refcnt than expected " - "_PyObject_IMMORTAL_REFCNT"); + "_Py_IMMORTAL_REFCNT"); } } #endif diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 7a78a91aa617e6..0a9f24efbdb908 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -19,6 +19,8 @@ struct _import_runtime_state { used exclusively for when the extensions dict is access/modified from an arbitrary thread. */ PyThreadState main_tstate; + /* A lock to guard the dict. */ + PyThread_type_lock mutex; /* A dict mapping (filename, name) to PyModuleDef for modules. Only legacy (single-phase init) extension modules are added and only if they support multiple initialization (m_size >- 0) diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h new file mode 100644 index 00000000000000..e94d8755546efd --- /dev/null +++ b/Include/internal/pycore_instruments.h @@ -0,0 +1,107 @@ + +#ifndef Py_INTERNAL_INSTRUMENT_H +#define Py_INTERNAL_INSTRUMENT_H + + +#include "pycore_bitutils.h" // _Py_popcount32 +#include "pycore_frame.h" + +#include "cpython/code.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define PY_MONITORING_TOOL_IDS 8 + +/* Local events. + * These require bytecode instrumentation */ + +#define PY_MONITORING_EVENT_PY_START 0 +#define PY_MONITORING_EVENT_PY_RESUME 1 +#define PY_MONITORING_EVENT_PY_RETURN 2 +#define PY_MONITORING_EVENT_PY_YIELD 3 +#define PY_MONITORING_EVENT_CALL 4 +#define PY_MONITORING_EVENT_LINE 5 +#define PY_MONITORING_EVENT_INSTRUCTION 6 +#define PY_MONITORING_EVENT_JUMP 7 +#define PY_MONITORING_EVENT_BRANCH 8 +#define PY_MONITORING_EVENT_STOP_ITERATION 9 + +#define PY_MONITORING_INSTRUMENTED_EVENTS 10 + +/* Other events, mainly exceptions */ + +#define PY_MONITORING_EVENT_RAISE 10 +#define PY_MONITORING_EVENT_EXCEPTION_HANDLED 11 +#define PY_MONITORING_EVENT_PY_UNWIND 12 +#define PY_MONITORING_EVENT_PY_THROW 13 + + +/* Ancilliary events */ + +#define PY_MONITORING_EVENT_C_RETURN 14 +#define PY_MONITORING_EVENT_C_RAISE 15 + + +typedef uint32_t _PyMonitoringEventSet; + +/* Tool IDs */ + +/* These are defined in PEP 669 for convenience to avoid clashes */ +#define PY_MONITORING_DEBUGGER_ID 0 +#define PY_MONITORING_COVERAGE_ID 1 +#define PY_MONITORING_PROFILER_ID 2 +#define PY_MONITORING_OPTIMIZER_ID 5 + +/* Internal IDs used to suuport sys.setprofile() and sys.settrace() */ +#define PY_MONITORING_SYS_PROFILE_ID 6 +#define PY_MONITORING_SYS_TRACE_ID 7 + + +PyObject *_PyMonitoring_RegisterCallback(int tool_id, int event_id, PyObject *obj); + +int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events); + +extern int +_Py_call_instrumentation(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern int +_Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, + _Py_CODEUNIT *instr); + +extern int +_Py_call_instrumentation_instruction( + PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr); + +int +_Py_call_instrumentation_jump( + PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target); + +extern int +_Py_call_instrumentation_arg(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg); + +extern int +_Py_call_instrumentation_2args(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); + +extern void +_Py_call_instrumentation_exc0(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); + +extern void +_Py_call_instrumentation_exc2(PyThreadState *tstate, int event, + _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); + +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, int index); + +extern PyObject _PyInstrumentation_MISSING; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_INSTRUMENT_H */ diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index d64a68cd2da54e..7276ce35ba68f0 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -23,10 +23,12 @@ extern "C" { #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_genobject.h" // struct _Py_async_gen_state #include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_import.h" // struct _import_state +#include "pycore_instruments.h" // PY_MONITORING_EVENTS #include "pycore_list.h" // struct _Py_list_state -#include "pycore_global_objects.h" // struct _Py_interp_static_objects #include "pycore_object_state.h" // struct _py_object_state +#include "pycore_obmalloc.h" // struct obmalloc_state #include "pycore_tuple.h" // struct _Py_tuple_state #include "pycore_typeobject.h" // struct type_cache #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -37,7 +39,6 @@ struct _Py_long_state { int max_str_digits; }; - /* interpreter state */ /* PyInterpreterState holds the global state for one of the runtime's @@ -49,6 +50,9 @@ struct _is { PyInterpreterState *next; + uint64_t monitoring_version; + uint64_t last_restart_version; + struct pythreads { uint64_t next_unique_id; /* The linked list of threads, newest first. */ @@ -79,6 +83,8 @@ struct _is { int _initialized; int finalizing; + struct _obmalloc_state obmalloc; + struct _ceval_state ceval; struct _gc_runtime_state gc; @@ -148,6 +154,15 @@ struct _is { struct callable_cache callable_cache; PyCodeObject *interpreter_trampoline; + _Py_Monitors monitors; + bool f_opcode_trace_set; + bool sys_profile_initialized; + bool sys_trace_initialized; + Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ + Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ + PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][PY_MONITORING_EVENTS]; + PyObject *monitoring_tool_names[PY_MONITORING_TOOL_IDS]; + struct _Py_interp_cached_objects cached_objects; struct _Py_interp_static_objects static_objects; diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 137a0465d5ec60..fe86581e81f6b5 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -245,7 +245,7 @@ _PyLong_FlipSign(PyLongObject *op) { #define _PyLong_DIGIT_INIT(val) \ { \ - .ob_base = _PyObject_IMMORTAL_INIT(&PyLong_Type), \ + .ob_base = _PyObject_HEAD_INIT(&PyLong_Type) \ .long_value = { \ .lv_tag = TAG_FROM_SIGN_AND_SIZE( \ (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b3d496ed6fc240..2ca047846e0935 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,21 +14,25 @@ extern "C" { #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _PyRuntime -/* This value provides *effective* immortality, meaning the object should never - be deallocated (until runtime finalization). See PEP 683 for more details about - immortality, as well as a proposed mechanism for proper immortality. */ -#define _PyObject_IMMORTAL_REFCNT 999999999 - -#define _PyObject_IMMORTAL_INIT(type) \ - { \ - .ob_refcnt = _PyObject_IMMORTAL_REFCNT, \ - .ob_type = (type), \ - } -#define _PyVarObject_IMMORTAL_INIT(type, size) \ - { \ - .ob_base = _PyObject_IMMORTAL_INIT(type), \ - .ob_size = size, \ - } +/* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid + designated initializer conflicts in C++20. If we use the deinition in + object.h, we will be mixing designated and non-designated initializers in + pycore objects which is forbiddent in C++20. However, if we then use + designated initializers in object.h then Extensions without designated break. + Furthermore, we can't use designated initializers in Extensions since these + are not supported pre-C++20. Thus, keeping an internal copy here is the most + backwards compatible solution */ +#define _PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + .ob_refcnt = _Py_IMMORTAL_REFCNT, \ + .ob_type = (type) \ + }, +#define _PyVarObject_HEAD_INIT(type, size) \ + { \ + .ob_base = _PyObject_HEAD_INIT(type) \ + .ob_size = size \ + }, PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, @@ -61,9 +65,20 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) +static inline void _Py_SetImmortal(PyObject *op) +{ + if (op) { + op->ob_refcnt = _Py_IMMORTAL_REFCNT; + } +} +#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) + static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { + if (_Py_IsImmortal(op)) { + return; + } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); @@ -82,6 +97,9 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { + if (_Py_IsImmortal(op)) { + return; + } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index a5c7f4528f9126..ca2a0419b4f038 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -657,8 +657,12 @@ struct _obmalloc_usage { #endif /* WITH_PYMALLOC_RADIX_TREE */ -struct _obmalloc_state { +struct _obmalloc_global_state { int dump_debug_stats; + Py_ssize_t interpreter_leaks; +}; + +struct _obmalloc_state { struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; struct _obmalloc_usage usage; @@ -675,7 +679,11 @@ void _PyObject_VirtualFree(void *, size_t size); /* This function returns the number of allocated memory blocks, regardless of size */ -PyAPI_FUNC(Py_ssize_t) _Py_GetAllocatedBlocks(void); +extern Py_ssize_t _Py_GetGlobalAllocatedBlocks(void); +#define _Py_GetAllocatedBlocks() \ + _Py_GetGlobalAllocatedBlocks() +extern Py_ssize_t _PyInterpreterState_GetAllocatedBlocks(PyInterpreterState *); +extern void _PyInterpreterState_FinalizeAllocatedBlocks(PyInterpreterState *); #ifdef WITH_PYMALLOC diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index c9f197e72de9f5..8ee72ff2d4126f 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -54,9 +54,13 @@ extern "C" { # error "NB_SMALL_SIZE_CLASSES should be less than 64" #endif -#define _obmalloc_state_INIT(obmalloc) \ +#define _obmalloc_global_state_INIT \ { \ .dump_debug_stats = -1, \ + } + +#define _obmalloc_state_INIT(obmalloc) \ + { \ .pools = { \ .used = _obmalloc_pools_INIT(obmalloc.pools), \ }, \ diff --git a/Include/internal/pycore_opcode.h b/Include/internal/pycore_opcode.h index 81ca91465950b7..a82885463ab2e9 100644 --- a/Include/internal/pycore_opcode.h +++ b/Include/internal/pycore_opcode.h @@ -12,8 +12,6 @@ extern "C" { #include "opcode.h" -extern const uint32_t _PyOpcode_RelativeJump[9]; - extern const uint32_t _PyOpcode_Jump[9]; extern const uint8_t _PyOpcode_Caches[256]; @@ -21,17 +19,6 @@ extern const uint8_t _PyOpcode_Caches[256]; extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_TABLES -const uint32_t _PyOpcode_RelativeJump[9] = { - 0U, - 0U, - 536870912U, - 135020544U, - 4163U, - 0U, - 0U, - 0U, - 48U, -}; const uint32_t _PyOpcode_Jump[9] = { 0U, 0U, @@ -55,6 +42,7 @@ const uint8_t _PyOpcode_Caches[256] = { [LOAD_GLOBAL] = 4, [BINARY_OP] = 1, [SEND] = 1, + [LOAD_SUPER_ATTR] = 9, [CALL] = 3, }; @@ -125,6 +113,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [DICT_UPDATE] = DICT_UPDATE, [END_ASYNC_FOR] = END_ASYNC_FOR, [END_FOR] = END_FOR, + [END_SEND] = END_SEND, [EXTENDED_ARG] = EXTENDED_ARG, [FORMAT_VALUE] = FORMAT_VALUE, [FOR_ITER] = FOR_ITER, @@ -140,6 +129,23 @@ const uint8_t _PyOpcode_Deopt[256] = { [GET_YIELD_FROM_ITER] = GET_YIELD_FROM_ITER, [IMPORT_FROM] = IMPORT_FROM, [IMPORT_NAME] = IMPORT_NAME, + [INSTRUMENTED_CALL] = INSTRUMENTED_CALL, + [INSTRUMENTED_CALL_FUNCTION_EX] = INSTRUMENTED_CALL_FUNCTION_EX, + [INSTRUMENTED_END_FOR] = INSTRUMENTED_END_FOR, + [INSTRUMENTED_END_SEND] = INSTRUMENTED_END_SEND, + [INSTRUMENTED_FOR_ITER] = INSTRUMENTED_FOR_ITER, + [INSTRUMENTED_INSTRUCTION] = INSTRUMENTED_INSTRUCTION, + [INSTRUMENTED_JUMP_BACKWARD] = INSTRUMENTED_JUMP_BACKWARD, + [INSTRUMENTED_JUMP_FORWARD] = INSTRUMENTED_JUMP_FORWARD, + [INSTRUMENTED_LINE] = INSTRUMENTED_LINE, + [INSTRUMENTED_POP_JUMP_IF_FALSE] = INSTRUMENTED_POP_JUMP_IF_FALSE, + [INSTRUMENTED_POP_JUMP_IF_NONE] = INSTRUMENTED_POP_JUMP_IF_NONE, + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = INSTRUMENTED_POP_JUMP_IF_NOT_NONE, + [INSTRUMENTED_POP_JUMP_IF_TRUE] = INSTRUMENTED_POP_JUMP_IF_TRUE, + [INSTRUMENTED_RESUME] = INSTRUMENTED_RESUME, + [INSTRUMENTED_RETURN_CONST] = INSTRUMENTED_RETURN_CONST, + [INSTRUMENTED_RETURN_VALUE] = INSTRUMENTED_RETURN_VALUE, + [INSTRUMENTED_YIELD_VALUE] = INSTRUMENTED_YIELD_VALUE, [INTERPRETER_EXIT] = INTERPRETER_EXIT, [IS_OP] = IS_OP, [JUMP_BACKWARD] = JUMP_BACKWARD, @@ -174,6 +180,8 @@ const uint8_t _PyOpcode_Deopt[256] = { [LOAD_GLOBAL_BUILTIN] = LOAD_GLOBAL, [LOAD_GLOBAL_MODULE] = LOAD_GLOBAL, [LOAD_NAME] = LOAD_NAME, + [LOAD_SUPER_ATTR] = LOAD_SUPER_ATTR, + [LOAD_SUPER_ATTR_METHOD] = LOAD_SUPER_ATTR, [MAKE_CELL] = MAKE_CELL, [MAKE_FUNCTION] = MAKE_FUNCTION, [MAP_ADD] = MAP_ADD, @@ -192,6 +200,7 @@ const uint8_t _PyOpcode_Deopt[256] = { [PUSH_NULL] = PUSH_NULL, [RAISE_VARARGS] = RAISE_VARARGS, [RERAISE] = RERAISE, + [RESERVED] = RESERVED, [RESUME] = RESUME, [RETURN_CONST] = RETURN_CONST, [RETURN_GENERATOR] = RETURN_GENERATOR, @@ -230,23 +239,25 @@ const uint8_t _PyOpcode_Deopt[256] = { #endif // NEED_OPCODE_TABLES #ifdef Py_DEBUG -static const char *const _PyOpcode_OpName[263] = { +static const char *const _PyOpcode_OpName[266] = { [CACHE] = "CACHE", [POP_TOP] = "POP_TOP", [PUSH_NULL] = "PUSH_NULL", [INTERPRETER_EXIT] = "INTERPRETER_EXIT", [END_FOR] = "END_FOR", + [END_SEND] = "END_SEND", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", [BINARY_OP_ADD_UNICODE] = "BINARY_OP_ADD_UNICODE", - [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [NOP] = "NOP", - [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", + [BINARY_OP_INPLACE_ADD_UNICODE] = "BINARY_OP_INPLACE_ADD_UNICODE", [UNARY_NEGATIVE] = "UNARY_NEGATIVE", [UNARY_NOT] = "UNARY_NOT", + [BINARY_OP_MULTIPLY_FLOAT] = "BINARY_OP_MULTIPLY_FLOAT", [BINARY_OP_MULTIPLY_INT] = "BINARY_OP_MULTIPLY_INT", - [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", [UNARY_INVERT] = "UNARY_INVERT", + [BINARY_OP_SUBTRACT_FLOAT] = "BINARY_OP_SUBTRACT_FLOAT", + [RESERVED] = "RESERVED", [BINARY_OP_SUBTRACT_INT] = "BINARY_OP_SUBTRACT_INT", [BINARY_SUBSCR_DICT] = "BINARY_SUBSCR_DICT", [BINARY_SUBSCR_GETITEM] = "BINARY_SUBSCR_GETITEM", @@ -254,21 +265,21 @@ static const char *const _PyOpcode_OpName[263] = { [BINARY_SUBSCR_TUPLE_INT] = "BINARY_SUBSCR_TUPLE_INT", [CALL_PY_EXACT_ARGS] = "CALL_PY_EXACT_ARGS", [CALL_PY_WITH_DEFAULTS] = "CALL_PY_WITH_DEFAULTS", - [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", - [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [BINARY_SUBSCR] = "BINARY_SUBSCR", [BINARY_SLICE] = "BINARY_SLICE", [STORE_SLICE] = "STORE_SLICE", - [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", - [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_BOUND_METHOD_EXACT_ARGS] = "CALL_BOUND_METHOD_EXACT_ARGS", + [CALL_BUILTIN_CLASS] = "CALL_BUILTIN_CLASS", [GET_LEN] = "GET_LEN", [MATCH_MAPPING] = "MATCH_MAPPING", [MATCH_SEQUENCE] = "MATCH_SEQUENCE", [MATCH_KEYS] = "MATCH_KEYS", - [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", + [CALL_BUILTIN_FAST_WITH_KEYWORDS] = "CALL_BUILTIN_FAST_WITH_KEYWORDS", [PUSH_EXC_INFO] = "PUSH_EXC_INFO", [CHECK_EXC_MATCH] = "CHECK_EXC_MATCH", [CHECK_EG_MATCH] = "CHECK_EG_MATCH", + [CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS] = "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + [CALL_NO_KW_BUILTIN_FAST] = "CALL_NO_KW_BUILTIN_FAST", [CALL_NO_KW_BUILTIN_O] = "CALL_NO_KW_BUILTIN_O", [CALL_NO_KW_ISINSTANCE] = "CALL_NO_KW_ISINSTANCE", [CALL_NO_KW_LEN] = "CALL_NO_KW_LEN", @@ -278,8 +289,6 @@ static const char *const _PyOpcode_OpName[263] = { [CALL_NO_KW_METHOD_DESCRIPTOR_O] = "CALL_NO_KW_METHOD_DESCRIPTOR_O", [CALL_NO_KW_STR_1] = "CALL_NO_KW_STR_1", [CALL_NO_KW_TUPLE_1] = "CALL_NO_KW_TUPLE_1", - [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", - [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [WITH_EXCEPT_START] = "WITH_EXCEPT_START", [GET_AITER] = "GET_AITER", [GET_ANEXT] = "GET_ANEXT", @@ -287,39 +296,39 @@ static const char *const _PyOpcode_OpName[263] = { [BEFORE_WITH] = "BEFORE_WITH", [END_ASYNC_FOR] = "END_ASYNC_FOR", [CLEANUP_THROW] = "CLEANUP_THROW", + [CALL_NO_KW_TYPE_1] = "CALL_NO_KW_TYPE_1", + [COMPARE_OP_FLOAT] = "COMPARE_OP_FLOAT", [COMPARE_OP_INT] = "COMPARE_OP_INT", [COMPARE_OP_STR] = "COMPARE_OP_STR", - [FOR_ITER_LIST] = "FOR_ITER_LIST", - [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [STORE_SUBSCR] = "STORE_SUBSCR", [DELETE_SUBSCR] = "DELETE_SUBSCR", + [FOR_ITER_LIST] = "FOR_ITER_LIST", + [FOR_ITER_TUPLE] = "FOR_ITER_TUPLE", [FOR_ITER_RANGE] = "FOR_ITER_RANGE", [FOR_ITER_GEN] = "FOR_ITER_GEN", + [LOAD_SUPER_ATTR_METHOD] = "LOAD_SUPER_ATTR_METHOD", [LOAD_ATTR_CLASS] = "LOAD_ATTR_CLASS", + [GET_ITER] = "GET_ITER", + [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", [LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN] = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_ATTR_INSTANCE_VALUE] = "LOAD_ATTR_INSTANCE_VALUE", [LOAD_ATTR_MODULE] = "LOAD_ATTR_MODULE", - [GET_ITER] = "GET_ITER", - [GET_YIELD_FROM_ITER] = "GET_YIELD_FROM_ITER", + [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", + [RETURN_GENERATOR] = "RETURN_GENERATOR", [LOAD_ATTR_PROPERTY] = "LOAD_ATTR_PROPERTY", - [LOAD_BUILD_CLASS] = "LOAD_BUILD_CLASS", [LOAD_ATTR_SLOT] = "LOAD_ATTR_SLOT", [LOAD_ATTR_WITH_HINT] = "LOAD_ATTR_WITH_HINT", - [LOAD_ASSERTION_ERROR] = "LOAD_ASSERTION_ERROR", - [RETURN_GENERATOR] = "RETURN_GENERATOR", [LOAD_ATTR_METHOD_LAZY_DICT] = "LOAD_ATTR_METHOD_LAZY_DICT", [LOAD_ATTR_METHOD_NO_DICT] = "LOAD_ATTR_METHOD_NO_DICT", [LOAD_ATTR_METHOD_WITH_VALUES] = "LOAD_ATTR_METHOD_WITH_VALUES", [LOAD_CONST__LOAD_FAST] = "LOAD_CONST__LOAD_FAST", + [RETURN_VALUE] = "RETURN_VALUE", [LOAD_FAST__LOAD_CONST] = "LOAD_FAST__LOAD_CONST", + [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", [LOAD_FAST__LOAD_FAST] = "LOAD_FAST__LOAD_FAST", [LOAD_GLOBAL_BUILTIN] = "LOAD_GLOBAL_BUILTIN", - [RETURN_VALUE] = "RETURN_VALUE", [LOAD_GLOBAL_MODULE] = "LOAD_GLOBAL_MODULE", - [SETUP_ANNOTATIONS] = "SETUP_ANNOTATIONS", - [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", - [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", - [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [POP_EXCEPT] = "POP_EXCEPT", [STORE_NAME] = "STORE_NAME", [DELETE_NAME] = "DELETE_NAME", @@ -342,9 +351,9 @@ static const char *const _PyOpcode_OpName[263] = { [IMPORT_NAME] = "IMPORT_NAME", [IMPORT_FROM] = "IMPORT_FROM", [JUMP_FORWARD] = "JUMP_FORWARD", - [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", - [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", - [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", + [STORE_ATTR_INSTANCE_VALUE] = "STORE_ATTR_INSTANCE_VALUE", + [STORE_ATTR_SLOT] = "STORE_ATTR_SLOT", + [STORE_ATTR_WITH_HINT] = "STORE_ATTR_WITH_HINT", [POP_JUMP_IF_FALSE] = "POP_JUMP_IF_FALSE", [POP_JUMP_IF_TRUE] = "POP_JUMP_IF_TRUE", [LOAD_GLOBAL] = "LOAD_GLOBAL", @@ -372,9 +381,9 @@ static const char *const _PyOpcode_OpName[263] = { [STORE_DEREF] = "STORE_DEREF", [DELETE_DEREF] = "DELETE_DEREF", [JUMP_BACKWARD] = "JUMP_BACKWARD", - [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [LOAD_SUPER_ATTR] = "LOAD_SUPER_ATTR", [CALL_FUNCTION_EX] = "CALL_FUNCTION_EX", - [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [STORE_FAST__LOAD_FAST] = "STORE_FAST__LOAD_FAST", [EXTENDED_ARG] = "EXTENDED_ARG", [LIST_APPEND] = "LIST_APPEND", [SET_ADD] = "SET_ADD", @@ -384,20 +393,20 @@ static const char *const _PyOpcode_OpName[263] = { [YIELD_VALUE] = "YIELD_VALUE", [RESUME] = "RESUME", [MATCH_CLASS] = "MATCH_CLASS", - [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", - [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", + [STORE_FAST__STORE_FAST] = "STORE_FAST__STORE_FAST", + [STORE_SUBSCR_DICT] = "STORE_SUBSCR_DICT", [FORMAT_VALUE] = "FORMAT_VALUE", [BUILD_CONST_KEY_MAP] = "BUILD_CONST_KEY_MAP", [BUILD_STRING] = "BUILD_STRING", - [SEND_GEN] = "SEND_GEN", - [159] = "<159>", - [160] = "<160>", - [161] = "<161>", + [STORE_SUBSCR_LIST_INT] = "STORE_SUBSCR_LIST_INT", + [UNPACK_SEQUENCE_LIST] = "UNPACK_SEQUENCE_LIST", + [UNPACK_SEQUENCE_TUPLE] = "UNPACK_SEQUENCE_TUPLE", + [UNPACK_SEQUENCE_TWO_TUPLE] = "UNPACK_SEQUENCE_TWO_TUPLE", [LIST_EXTEND] = "LIST_EXTEND", [SET_UPDATE] = "SET_UPDATE", [DICT_MERGE] = "DICT_MERGE", [DICT_UPDATE] = "DICT_UPDATE", - [166] = "<166>", + [SEND_GEN] = "SEND_GEN", [167] = "<167>", [168] = "<168>", [169] = "<169>", @@ -469,24 +478,24 @@ static const char *const _PyOpcode_OpName[263] = { [235] = "<235>", [236] = "<236>", [237] = "<237>", - [238] = "<238>", - [239] = "<239>", - [240] = "<240>", - [241] = "<241>", - [242] = "<242>", - [243] = "<243>", - [244] = "<244>", - [245] = "<245>", - [246] = "<246>", - [247] = "<247>", - [248] = "<248>", - [249] = "<249>", - [250] = "<250>", - [251] = "<251>", - [252] = "<252>", - [253] = "<253>", - [254] = "<254>", - [DO_TRACING] = "DO_TRACING", + [INSTRUMENTED_POP_JUMP_IF_NONE] = "INSTRUMENTED_POP_JUMP_IF_NONE", + [INSTRUMENTED_POP_JUMP_IF_NOT_NONE] = "INSTRUMENTED_POP_JUMP_IF_NOT_NONE", + [INSTRUMENTED_RESUME] = "INSTRUMENTED_RESUME", + [INSTRUMENTED_CALL] = "INSTRUMENTED_CALL", + [INSTRUMENTED_RETURN_VALUE] = "INSTRUMENTED_RETURN_VALUE", + [INSTRUMENTED_YIELD_VALUE] = "INSTRUMENTED_YIELD_VALUE", + [INSTRUMENTED_CALL_FUNCTION_EX] = "INSTRUMENTED_CALL_FUNCTION_EX", + [INSTRUMENTED_JUMP_FORWARD] = "INSTRUMENTED_JUMP_FORWARD", + [INSTRUMENTED_JUMP_BACKWARD] = "INSTRUMENTED_JUMP_BACKWARD", + [INSTRUMENTED_RETURN_CONST] = "INSTRUMENTED_RETURN_CONST", + [INSTRUMENTED_FOR_ITER] = "INSTRUMENTED_FOR_ITER", + [INSTRUMENTED_POP_JUMP_IF_FALSE] = "INSTRUMENTED_POP_JUMP_IF_FALSE", + [INSTRUMENTED_POP_JUMP_IF_TRUE] = "INSTRUMENTED_POP_JUMP_IF_TRUE", + [INSTRUMENTED_END_FOR] = "INSTRUMENTED_END_FOR", + [INSTRUMENTED_END_SEND] = "INSTRUMENTED_END_SEND", + [INSTRUMENTED_INSTRUCTION] = "INSTRUMENTED_INSTRUCTION", + [INSTRUMENTED_LINE] = "INSTRUMENTED_LINE", + [255] = "<255>", [SETUP_FINALLY] = "SETUP_FINALLY", [SETUP_CLEANUP] = "SETUP_CLEANUP", [SETUP_WITH] = "SETUP_WITH", @@ -494,14 +503,13 @@ static const char *const _PyOpcode_OpName[263] = { [JUMP] = "JUMP", [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LOAD_METHOD] = "LOAD_METHOD", + [LOAD_SUPER_METHOD] = "LOAD_SUPER_METHOD", + [LOAD_ZERO_SUPER_METHOD] = "LOAD_ZERO_SUPER_METHOD", + [LOAD_ZERO_SUPER_ATTR] = "LOAD_ZERO_SUPER_ATTR", }; #endif #define EXTRA_CASES \ - case 159: \ - case 160: \ - case 161: \ - case 166: \ case 167: \ case 168: \ case 169: \ @@ -569,23 +577,7 @@ static const char *const _PyOpcode_OpName[263] = { case 235: \ case 236: \ case 237: \ - case 238: \ - case 239: \ - case 240: \ - case 241: \ - case 242: \ - case 243: \ - case 244: \ - case 245: \ - case 246: \ - case 247: \ - case 248: \ - case 249: \ - case 250: \ - case 251: \ - case 252: \ - case 253: \ - case 254: \ + case 255: \ ; #ifdef __cplusplus diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 96bb4d743bf29f..1d5ff988290bd4 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -8,7 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_opcode.h" // _PyOpcode_RelativeJump +#include "pycore_opcode.h" // _PyOpcode_Jump #define MAX_REAL_OPCODE 254 @@ -85,9 +85,6 @@ is_bit_set_in_table(const uint32_t *table, int bitindex) { #undef LOG_BITS_PER_INT #undef MASK_LOW_LOG_BITS -#define IS_RELATIVE_JUMP(opcode) (is_bit_set_in_table(_PyOpcode_RelativeJump, opcode)) - - #ifdef __cplusplus } diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 1bb4a9aa103898..4620a269644917 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -109,6 +109,8 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); +void _PyErr_FormatNote(const char *format, ...); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index a899e848bb8b3c..f96261a650dac7 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -64,6 +64,7 @@ extern void _PyAtExit_Fini(PyInterpreterState *interp); extern void _PyThread_FiniType(PyInterpreterState *interp); extern void _Py_Deepfreeze_Fini(void); extern void _PyArg_Fini(void); +extern void _Py_FinalizeAllocatedBlocks(_PyRuntimeState *); extern PyStatus _PyGILState_Init(PyInterpreterState *interp); extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index b5408622d9d4b2..180ea676bc22eb 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -33,6 +33,13 @@ _Py_IsMainInterpreter(PyInterpreterState *interp) return (interp == _PyInterpreterState_Main()); } +static inline int +_Py_IsMainInterpreterFinalizing(PyInterpreterState *interp) +{ + return (_PyRuntimeState_GetFinalizing(interp->runtime) != NULL && + interp == &interp->runtime->_main_interpreter); +} + static inline const PyConfig * _Py_GetMainConfig(void) @@ -64,17 +71,14 @@ _Py_ThreadCanHandlePendingCalls(void) /* Variable and macro for in-line access to current thread and interpreter state */ -static inline PyThreadState* -_PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) -{ - return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->tstate_current); -} +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +extern _Py_thread_local PyThreadState *_Py_tss_tstate; +#endif +PyAPI_DATA(PyThreadState *) _PyThreadState_GetCurrent(void); /* Get the current Python thread state. - Efficient macro reading directly the 'tstate_current' atomic - variable. The macro is unsafe: it does not check for error and it can - return NULL. + This function is unsafe: it does not check for error and it can return NULL. The caller must hold the GIL. @@ -82,9 +86,20 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) static inline PyThreadState* _PyThreadState_GET(void) { - return _PyRuntimeState_GetThreadState(&_PyRuntime); +#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) + return _Py_tss_tstate; +#else + return _PyThreadState_GetCurrent(); +#endif } +static inline PyThreadState* +_PyRuntimeState_GetThreadState(_PyRuntimeState *Py_UNUSED(runtime)) +{ + return _PyThreadState_GET(); +} + + static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) { @@ -133,16 +148,6 @@ extern void _PyThreadState_BindDetached(PyThreadState *); extern void _PyThreadState_UnbindDetached(PyThreadState *); -static inline void -_PyThreadState_UpdateTracingState(PyThreadState *tstate) -{ - bool use_tracing = - (tstate->tracing == 0) && - (tstate->c_tracefunc != NULL || tstate->c_profilefunc != NULL); - tstate->cframe->use_tracing = (use_tracing ? 255 : 0); -} - - /* Other */ PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap( diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index 3ebe49926edda6..d1b165d0ab9c38 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -21,10 +21,10 @@ extern "C" { #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_pythread.h" // struct _pythread_runtime_state -#include "pycore_obmalloc.h" // struct obmalloc_state #include "pycore_signal.h" // struct _signals_runtime_state #include "pycore_time.h" // struct _time_runtime_state #include "pycore_tracemalloc.h" // struct _tracemalloc_runtime_state +#include "pycore_typeobject.h" // struct types_runtime_state #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids struct _getargs_runtime_state { @@ -87,7 +87,7 @@ typedef struct pyruntimestate { _Py_atomic_address _finalizing; struct _pymem_allocators allocators; - struct _obmalloc_state obmalloc; + struct _obmalloc_global_state obmalloc; struct pyhash_runtime_state pyhash_state; struct _time_runtime_state time; struct _pythread_runtime_state threads; @@ -119,9 +119,6 @@ typedef struct pyruntimestate { unsigned long main_thread; - /* Assuming the current thread holds the GIL, this is the - PyThreadState for the current thread. */ - _Py_atomic_address tstate_current; /* Used for the thread state bound to the current thread. */ Py_tss_t autoTSSkey; @@ -153,13 +150,7 @@ typedef struct pyruntimestate { struct _py_object_runtime_state object_state; struct _Py_float_runtime_state float_state; struct _Py_unicode_runtime_state unicode_state; - - struct { - /* Used to set PyTypeObject.tp_version_tag */ - // bpo-42745: next_version_tag remains shared by all interpreters - // because of static types. - unsigned int next_version_tag; - } types; + struct _types_runtime_state types; /* All the objects that are shared by the runtime's interpreters. */ struct _Py_static_objects static_objects; diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 5b09a45e41cd84..a48461c0742872 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -29,7 +29,7 @@ extern PyTypeObject _PyExc_MemoryError; _pymem_allocators_debug_INIT, \ _pymem_allocators_obj_arena_INIT, \ }, \ - .obmalloc = _obmalloc_state_INIT(runtime.obmalloc), \ + .obmalloc = _obmalloc_global_state_INIT, \ .pyhash_state = pyhash_state_INIT, \ .signals = _signals_RUNTIME_INIT, \ .interpreters = { \ @@ -76,13 +76,13 @@ extern PyTypeObject _PyExc_MemoryError; .latin1 = _Py_str_latin1_INIT, \ }, \ .tuple_empty = { \ - .ob_base = _PyVarObject_IMMORTAL_INIT(&PyTuple_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0) \ }, \ .hamt_bitmap_node_empty = { \ - .ob_base = _PyVarObject_IMMORTAL_INIT(&_PyHamt_BitmapNode_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0) \ }, \ .context_token_missing = { \ - .ob_base = _PyObject_IMMORTAL_INIT(&_PyContextTokenMissing_Type), \ + .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type) \ }, \ }, \ }, \ @@ -93,6 +93,7 @@ extern PyTypeObject _PyExc_MemoryError; { \ .id_refcount = -1, \ .imports = IMPORTS_INIT, \ + .obmalloc = _obmalloc_state_INIT(INTERP.obmalloc), \ .ceval = { \ .recursion_limit = Py_DEFAULT_RECURSION_LIMIT, \ }, \ @@ -112,15 +113,18 @@ extern PyTypeObject _PyExc_MemoryError; .func_state = { \ .next_version = 1, \ }, \ + .types = { \ + .next_version_tag = _Py_TYPE_BASE_VERSION_TAG, \ + }, \ .static_objects = { \ .singletons = { \ ._not_used = 1, \ .hamt_empty = { \ - .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \ + .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type) \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ .last_resort_memory_error = { \ - _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \ + _PyObject_HEAD_INIT(&_PyExc_MemoryError) \ }, \ }, \ }, \ @@ -138,7 +142,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyBytes_SIMPLE_INIT(CH, LEN) \ { \ - _PyVarObject_IMMORTAL_INIT(&PyBytes_Type, (LEN)), \ + _PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)) \ .ob_shash = -1, \ .ob_sval = { (CH) }, \ } @@ -149,7 +153,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyUnicode_ASCII_BASE_INIT(LITERAL, ASCII) \ { \ - .ob_base = _PyObject_IMMORTAL_INIT(&PyUnicode_Type), \ + .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type) \ .length = sizeof(LITERAL) - 1, \ .hash = -1, \ .state = { \ diff --git a/Include/internal/pycore_token.h b/Include/internal/pycore_token.h index 95459ab9f7d004..b9df8766736adf 100644 --- a/Include/internal/pycore_token.h +++ b/Include/internal/pycore_token.h @@ -67,14 +67,18 @@ extern "C" { #define RARROW 51 #define ELLIPSIS 52 #define COLONEQUAL 53 -#define OP 54 -#define AWAIT 55 -#define ASYNC 56 -#define TYPE_IGNORE 57 -#define TYPE_COMMENT 58 -#define SOFT_KEYWORD 59 -#define ERRORTOKEN 60 -#define N_TOKENS 64 +#define EXCLAMATION 54 +#define OP 55 +#define AWAIT 56 +#define ASYNC 57 +#define TYPE_IGNORE 58 +#define TYPE_COMMENT 59 +#define SOFT_KEYWORD 60 +#define FSTRING_START 61 +#define FSTRING_MIDDLE 62 +#define FSTRING_END 63 +#define ERRORTOKEN 64 +#define N_TOKENS 68 #define NT_OFFSET 256 /* Special definitions for cooperation with parser */ @@ -86,6 +90,8 @@ extern "C" { (x) == NEWLINE || \ (x) == INDENT || \ (x) == DEDENT) +#define ISSTRINGLIT(x) ((x) == STRING || \ + (x) == FSTRING_MIDDLE) // Symbols exported for test_peg_generator diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index cc5ce2875101ea..76253fd5fd864c 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -11,22 +11,17 @@ extern "C" { #endif -/* runtime lifecycle */ +/* state */ -extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); -extern void _PyTypes_FiniTypes(PyInterpreterState *); -extern void _PyTypes_Fini(PyInterpreterState *); - - -/* other API */ - -/* Length of array of slotdef pointers used to store slots with the - same __name__. There should be at most MAX_EQUIV-1 slotdef entries with - the same __name__, for any __name__. Since that's a static property, it is - appropriate to declare fixed-size arrays for this. */ -#define MAX_EQUIV 10 +#define _Py_TYPE_BASE_VERSION_TAG (2<<16) +#define _Py_MAX_GLOBAL_TYPE_VERSION_TAG (_Py_TYPE_BASE_VERSION_TAG - 1) -typedef struct wrapperbase pytype_slotdef; +struct _types_runtime_state { + /* Used to set PyTypeObject.tp_version_tag for core static types. */ + // bpo-42745: next_version_tag remains shared by all interpreters + // because of static types. + unsigned int next_version_tag; +}; // Type attribute lookup cache: speed up attribute and method lookups, @@ -57,6 +52,36 @@ typedef struct { PyObject *tp_weaklist; } static_builtin_state; +struct types_state { + /* Used to set PyTypeObject.tp_version_tag. + It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1, + where all those lower numbers are used for core static types. */ + unsigned int next_version_tag; + + struct type_cache type_cache; + size_t num_builtins_initialized; + static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; +}; + + +/* runtime lifecycle */ + +extern PyStatus _PyTypes_InitTypes(PyInterpreterState *); +extern void _PyTypes_FiniTypes(PyInterpreterState *); +extern void _PyTypes_Fini(PyInterpreterState *); + + +/* other API */ + +/* Length of array of slotdef pointers used to store slots with the + same __name__. There should be at most MAX_EQUIV-1 slotdef entries with + the same __name__, for any __name__. Since that's a static property, it is + appropriate to declare fixed-size arrays for this. */ +#define MAX_EQUIV 10 + +typedef struct wrapperbase pytype_slotdef; + + static inline PyObject ** _PyStaticType_GET_WEAKREFS_LISTPTR(static_builtin_state *state) { @@ -78,12 +103,6 @@ _PyType_GetModuleState(PyTypeObject *type) return mod->md_state; } -struct types_state { - struct type_cache type_cache; - size_t num_builtins_initialized; - static_builtin_state builtins[_Py_MAX_STATIC_BUILTIN_TYPES]; -}; - extern int _PyStaticType_InitBuiltin(PyTypeObject *type); extern static_builtin_state * _PyStaticType_GetState(PyTypeObject *); @@ -98,6 +117,11 @@ _Py_type_getattro(PyTypeObject *type, PyObject *name); PyObject *_Py_slot_tp_getattro(PyObject *self, PyObject *name); PyObject *_Py_slot_tp_getattr_hook(PyObject *self, PyObject *name); +PyObject * +_PySuper_Lookup(PyTypeObject *su_type, PyObject *su_obj, PyObject *name, int *meth_found); +PyObject * +_PySuper_LookupDescr(PyTypeObject *su_type, PyObject *su_obj, PyObject *name); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index ff97b9a623d210..1bb0f366e78163 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI void _PyUnicode_ExactDealloc(PyObject *op); +Py_ssize_t _PyUnicode_InternedSize(void); /* runtime lifecycle */ diff --git a/Include/object.h b/Include/object.h index 2943a6066818cd..66c3df0d7f780a 100644 --- a/Include/object.h +++ b/Include/object.h @@ -78,12 +78,76 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; -#define PyObject_HEAD_INIT(type) \ - { _PyObject_EXTRA_INIT \ - 1, (type) }, +/* +Immortalization: + +The following indicates the immortalization strategy depending on the amount +of available bits in the reference count field. All strategies are backwards +compatible but the specific reference count value or immortalization check +might change depending on the specializations for the underlying system. + +Proper deallocation of immortal instances requires distinguishing between +statically allocated immortal instances vs those promoted by the runtime to be +immortal. The latter should be the only instances that require +cleanup during runtime finalization. +*/ + +#if SIZEOF_VOID_P > 4 +/* +In 64+ bit systems, an object will be marked as immortal by setting all of the +lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF + +Using the lower 32 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases will use saturated arithmetic, taking advantage of +having all the lower 32 bits set, which will avoid the reference count to go +beyond the refcount limit. Immortality checks for reference count decreases will +be done by checking the bit sign flag in the lower 32 bits. +*/ +#define _Py_IMMORTAL_REFCNT UINT_MAX + +#else +/* +In 32 bit systems, an object will be marked as immortal by setting all of the +lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF -#define PyVarObject_HEAD_INIT(type, size) \ - { PyObject_HEAD_INIT(type) (size) }, +Using the lower 30 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases and decreases will first go through an immortality +check by comparing the reference count field to the immortality reference count. +*/ +#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) +#endif + +// Make all internal uses of PyObject_HEAD_INIT immortal while preserving the +// C-API expectation that the refcnt will be set to 1. +#ifdef Py_BUILD_CORE +#define PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + { _Py_IMMORTAL_REFCNT }, \ + (type) \ + }, +#else +#define PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + { 1 }, \ + (type) \ + }, +#endif /* Py_BUILD_CORE */ + +#define PyVarObject_HEAD_INIT(type, size) \ + { \ + PyObject_HEAD_INIT(type) \ + (size) \ + }, /* PyObject_VAR_HEAD defines the initial segment of all variable-size * container objects. These end with a declaration of an array with 1 @@ -101,7 +165,12 @@ whose size is determined when the object is allocated. */ struct _object { _PyObject_HEAD_EXTRA - Py_ssize_t ob_refcnt; + union { + Py_ssize_t ob_refcnt; +#if SIZEOF_VOID_P > 4 + PY_UINT32_T ob_refcnt_split[2]; +#endif + }; PyTypeObject *ob_type; }; @@ -152,6 +221,15 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif +static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) +{ +#if SIZEOF_VOID_P > 4 + return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; +#else + return op->ob_refcnt == _Py_IMMORTAL_REFCNT; +#endif +} +#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; @@ -162,6 +240,13 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { + // This immortal check is for code that is unaware of immortal objects. + // The runtime tracks these objects and we should avoid as much + // as possible having extensions inadvertently change the refcnt + // of an immortalized object. + if (_Py_IsImmortal(ob)) { + return; + } ob->ob_refcnt = refcnt; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 @@ -524,19 +609,33 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); -static inline void Py_INCREF(PyObject *op) +static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 // Stable ABI for Python 3.10 built in debug mode. _Py_IncRef(op); #else - _Py_INCREF_STAT_INC(); // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. +#if SIZEOF_VOID_P > 4 + // Portable saturated add, branching on the carry flag and set low bits + PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + PY_UINT32_T new_refcnt = cur_refcnt + 1; + if (new_refcnt == 0) { + return; + } + op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; +#else + // Explicitly check immortality against the immortal value + if (_Py_IsImmortal(op)) { + return; + } + op->ob_refcnt++; +#endif + _Py_INCREF_STAT_INC(); #ifdef Py_REF_DEBUG _Py_INC_REFTOTAL(); -#endif // Py_REF_DEBUG - op->ob_refcnt++; +#endif #endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 @@ -553,6 +652,9 @@ static inline void Py_DECREF(PyObject *op) { #elif defined(Py_REF_DEBUG) static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { + if (_Py_IsImmortal(op)) { + return; + } _Py_DECREF_STAT_INC(); _Py_DEC_REFTOTAL(); if (--op->ob_refcnt != 0) { @@ -567,11 +669,14 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) #else -static inline void Py_DECREF(PyObject *op) +static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) { - _Py_DECREF_STAT_INC(); // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); if (--op->ob_refcnt == 0) { _Py_Dealloc(op); } @@ -721,7 +826,7 @@ PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) /* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_NewRef(Py_None) +#define Py_RETURN_NONE return Py_None /* Py_NotImplemented is a singleton used to signal that an operation is @@ -731,7 +836,7 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) /* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented /* Rich comparison opcodes */ #define Py_LT 0 diff --git a/Include/opcode.h b/Include/opcode.h index 0ff84dc5a551a0..37a9e9bffa4cb7 100644 --- a/Include/opcode.h +++ b/Include/opcode.h @@ -13,10 +13,12 @@ extern "C" { #define PUSH_NULL 2 #define INTERPRETER_EXIT 3 #define END_FOR 4 +#define END_SEND 5 #define NOP 9 #define UNARY_NEGATIVE 11 #define UNARY_NOT 12 #define UNARY_INVERT 15 +#define RESERVED 17 #define BINARY_SUBSCR 25 #define BINARY_SLICE 26 #define STORE_SLICE 27 @@ -93,6 +95,7 @@ extern "C" { #define STORE_DEREF 138 #define DELETE_DEREF 139 #define JUMP_BACKWARD 140 +#define LOAD_SUPER_ATTR 141 #define CALL_FUNCTION_EX 142 #define EXTENDED_ARG 144 #define LIST_APPEND 145 @@ -114,6 +117,24 @@ extern "C" { #define KW_NAMES 172 #define CALL_INTRINSIC_1 173 #define CALL_INTRINSIC_2 174 +#define MIN_INSTRUMENTED_OPCODE 238 +#define INSTRUMENTED_POP_JUMP_IF_NONE 238 +#define INSTRUMENTED_POP_JUMP_IF_NOT_NONE 239 +#define INSTRUMENTED_RESUME 240 +#define INSTRUMENTED_CALL 241 +#define INSTRUMENTED_RETURN_VALUE 242 +#define INSTRUMENTED_YIELD_VALUE 243 +#define INSTRUMENTED_CALL_FUNCTION_EX 244 +#define INSTRUMENTED_JUMP_FORWARD 245 +#define INSTRUMENTED_JUMP_BACKWARD 246 +#define INSTRUMENTED_RETURN_CONST 247 +#define INSTRUMENTED_FOR_ITER 248 +#define INSTRUMENTED_POP_JUMP_IF_FALSE 249 +#define INSTRUMENTED_POP_JUMP_IF_TRUE 250 +#define INSTRUMENTED_END_FOR 251 +#define INSTRUMENTED_END_SEND 252 +#define INSTRUMENTED_INSTRUCTION 253 +#define INSTRUMENTED_LINE 254 #define MIN_PSEUDO_OPCODE 256 #define SETUP_FINALLY 256 #define SETUP_CLEANUP 257 @@ -122,75 +143,81 @@ extern "C" { #define JUMP 260 #define JUMP_NO_INTERRUPT 261 #define LOAD_METHOD 262 -#define MAX_PSEUDO_OPCODE 262 -#define BINARY_OP_ADD_FLOAT 5 -#define BINARY_OP_ADD_INT 6 -#define BINARY_OP_ADD_UNICODE 7 -#define BINARY_OP_INPLACE_ADD_UNICODE 8 -#define BINARY_OP_MULTIPLY_FLOAT 10 -#define BINARY_OP_MULTIPLY_INT 13 -#define BINARY_OP_SUBTRACT_FLOAT 14 -#define BINARY_OP_SUBTRACT_INT 16 -#define BINARY_SUBSCR_DICT 17 -#define BINARY_SUBSCR_GETITEM 18 -#define BINARY_SUBSCR_LIST_INT 19 -#define BINARY_SUBSCR_TUPLE_INT 20 -#define CALL_PY_EXACT_ARGS 21 -#define CALL_PY_WITH_DEFAULTS 22 -#define CALL_BOUND_METHOD_EXACT_ARGS 23 -#define CALL_BUILTIN_CLASS 24 -#define CALL_BUILTIN_FAST_WITH_KEYWORDS 28 -#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 29 -#define CALL_NO_KW_BUILTIN_FAST 34 -#define CALL_NO_KW_BUILTIN_O 38 -#define CALL_NO_KW_ISINSTANCE 39 -#define CALL_NO_KW_LEN 40 -#define CALL_NO_KW_LIST_APPEND 41 -#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 42 -#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 43 -#define CALL_NO_KW_METHOD_DESCRIPTOR_O 44 -#define CALL_NO_KW_STR_1 45 -#define CALL_NO_KW_TUPLE_1 46 -#define CALL_NO_KW_TYPE_1 47 -#define COMPARE_OP_FLOAT 48 -#define COMPARE_OP_INT 56 -#define COMPARE_OP_STR 57 -#define FOR_ITER_LIST 58 -#define FOR_ITER_TUPLE 59 -#define FOR_ITER_RANGE 62 -#define FOR_ITER_GEN 63 -#define LOAD_ATTR_CLASS 64 -#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 65 -#define LOAD_ATTR_INSTANCE_VALUE 66 -#define LOAD_ATTR_MODULE 67 -#define LOAD_ATTR_PROPERTY 70 -#define LOAD_ATTR_SLOT 72 -#define LOAD_ATTR_WITH_HINT 73 -#define LOAD_ATTR_METHOD_LAZY_DICT 76 -#define LOAD_ATTR_METHOD_NO_DICT 77 -#define LOAD_ATTR_METHOD_WITH_VALUES 78 -#define LOAD_CONST__LOAD_FAST 79 -#define LOAD_FAST__LOAD_CONST 80 -#define LOAD_FAST__LOAD_FAST 81 -#define LOAD_GLOBAL_BUILTIN 82 -#define LOAD_GLOBAL_MODULE 84 -#define STORE_ATTR_INSTANCE_VALUE 86 -#define STORE_ATTR_SLOT 87 -#define STORE_ATTR_WITH_HINT 88 -#define STORE_FAST__LOAD_FAST 111 -#define STORE_FAST__STORE_FAST 112 -#define STORE_SUBSCR_DICT 113 -#define STORE_SUBSCR_LIST_INT 141 -#define UNPACK_SEQUENCE_LIST 143 -#define UNPACK_SEQUENCE_TUPLE 153 -#define UNPACK_SEQUENCE_TWO_TUPLE 154 -#define SEND_GEN 158 -#define DO_TRACING 255 +#define LOAD_SUPER_METHOD 263 +#define LOAD_ZERO_SUPER_METHOD 264 +#define LOAD_ZERO_SUPER_ATTR 265 +#define MAX_PSEUDO_OPCODE 265 +#define BINARY_OP_ADD_FLOAT 6 +#define BINARY_OP_ADD_INT 7 +#define BINARY_OP_ADD_UNICODE 8 +#define BINARY_OP_INPLACE_ADD_UNICODE 10 +#define BINARY_OP_MULTIPLY_FLOAT 13 +#define BINARY_OP_MULTIPLY_INT 14 +#define BINARY_OP_SUBTRACT_FLOAT 16 +#define BINARY_OP_SUBTRACT_INT 18 +#define BINARY_SUBSCR_DICT 19 +#define BINARY_SUBSCR_GETITEM 20 +#define BINARY_SUBSCR_LIST_INT 21 +#define BINARY_SUBSCR_TUPLE_INT 22 +#define CALL_PY_EXACT_ARGS 23 +#define CALL_PY_WITH_DEFAULTS 24 +#define CALL_BOUND_METHOD_EXACT_ARGS 28 +#define CALL_BUILTIN_CLASS 29 +#define CALL_BUILTIN_FAST_WITH_KEYWORDS 34 +#define CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS 38 +#define CALL_NO_KW_BUILTIN_FAST 39 +#define CALL_NO_KW_BUILTIN_O 40 +#define CALL_NO_KW_ISINSTANCE 41 +#define CALL_NO_KW_LEN 42 +#define CALL_NO_KW_LIST_APPEND 43 +#define CALL_NO_KW_METHOD_DESCRIPTOR_FAST 44 +#define CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS 45 +#define CALL_NO_KW_METHOD_DESCRIPTOR_O 46 +#define CALL_NO_KW_STR_1 47 +#define CALL_NO_KW_TUPLE_1 48 +#define CALL_NO_KW_TYPE_1 56 +#define COMPARE_OP_FLOAT 57 +#define COMPARE_OP_INT 58 +#define COMPARE_OP_STR 59 +#define FOR_ITER_LIST 62 +#define FOR_ITER_TUPLE 63 +#define FOR_ITER_RANGE 64 +#define FOR_ITER_GEN 65 +#define LOAD_SUPER_ATTR_METHOD 66 +#define LOAD_ATTR_CLASS 67 +#define LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN 70 +#define LOAD_ATTR_INSTANCE_VALUE 72 +#define LOAD_ATTR_MODULE 73 +#define LOAD_ATTR_PROPERTY 76 +#define LOAD_ATTR_SLOT 77 +#define LOAD_ATTR_WITH_HINT 78 +#define LOAD_ATTR_METHOD_LAZY_DICT 79 +#define LOAD_ATTR_METHOD_NO_DICT 80 +#define LOAD_ATTR_METHOD_WITH_VALUES 81 +#define LOAD_CONST__LOAD_FAST 82 +#define LOAD_FAST__LOAD_CONST 84 +#define LOAD_FAST__LOAD_FAST 86 +#define LOAD_GLOBAL_BUILTIN 87 +#define LOAD_GLOBAL_MODULE 88 +#define STORE_ATTR_INSTANCE_VALUE 111 +#define STORE_ATTR_SLOT 112 +#define STORE_ATTR_WITH_HINT 113 +#define STORE_FAST__LOAD_FAST 143 +#define STORE_FAST__STORE_FAST 153 +#define STORE_SUBSCR_DICT 154 +#define STORE_SUBSCR_LIST_INT 158 +#define UNPACK_SEQUENCE_LIST 159 +#define UNPACK_SEQUENCE_TUPLE 160 +#define UNPACK_SEQUENCE_TWO_TUPLE 161 +#define SEND_GEN 166 #define HAS_ARG(op) ((((op) >= HAVE_ARGUMENT) && (!IS_PSEUDO_OPCODE(op)))\ || ((op) == JUMP) \ || ((op) == JUMP_NO_INTERRUPT) \ || ((op) == LOAD_METHOD) \ + || ((op) == LOAD_SUPER_METHOD) \ + || ((op) == LOAD_ZERO_SUPER_METHOD) \ + || ((op) == LOAD_ZERO_SUPER_ATTR) \ ) #define HAS_CONST(op) (false\ diff --git a/Include/pyport.h b/Include/pyport.h index eef0fe1bfd71d8..bd0ba6d0681b21 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -184,7 +184,6 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif -// bpo-28126: Py_MEMCPY is kept for backwards compatibility, #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_MEMCPY memcpy #endif @@ -663,6 +662,27 @@ extern char * _getpty(int *, int, mode_t, int); # define WITH_THREAD #endif +#ifdef WITH_THREAD +# ifdef Py_BUILD_CORE +# ifdef HAVE_THREAD_LOCAL +# error "HAVE_THREAD_LOCAL is already defined" +# endif +# define HAVE_THREAD_LOCAL 1 +# ifdef thread_local +# define _Py_thread_local thread_local +# elif __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_THREADS__) +# define _Py_thread_local _Thread_local +# elif defined(_MSC_VER) /* AKA NT_THREADS */ +# define _Py_thread_local __declspec(thread) +# elif defined(__GNUC__) /* includes clang */ +# define _Py_thread_local __thread +# else + // fall back to the PyThread_tss_*() API, or ignore. +# undef HAVE_THREAD_LOCAL +# endif +# endif +#endif + /* Check that ALT_SOABI is consistent with Py_TRACE_REFS: ./configure --with-trace-refs should must be used to define Py_TRACE_REFS */ #if defined(ALT_SOABI) && defined(Py_TRACE_REFS) diff --git a/Include/pystats.h b/Include/pystats.h index 25ed4bddc7240c..4b961bad2a43e4 100644 --- a/Include/pystats.h +++ b/Include/pystats.h @@ -72,8 +72,6 @@ typedef struct _object_stats { uint64_t type_cache_collisions; } ObjectStats; -# - typedef struct _stats { OpcodeStats opcode_stats[256]; CallStats call_stats; diff --git a/Lib/abc.py b/Lib/abc.py index 42048ddb855381..f8a4e11ce9c3b1 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -18,7 +18,7 @@ class that has a metaclass derived from ABCMeta cannot be class C(metaclass=ABCMeta): @abstractmethod - def my_abstract_method(self, ...): + def my_abstract_method(self, arg1, arg2, argN): ... """ funcobj.__isabstractmethod__ = True diff --git a/Lib/ast.py b/Lib/ast.py index 93afa7d8035de8..06df9a77255d7a 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -25,6 +25,7 @@ :license: Python License. """ import sys +import re from _ast import * from contextlib import contextmanager, nullcontext from enum import IntEnum, auto, _simple_enum @@ -305,28 +306,17 @@ def get_docstring(node, clean=True): return text -def _splitlines_no_ff(source): +_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))") +def _splitlines_no_ff(source, maxlines=None): """Split a string into lines ignoring form feed and other chars. This mimics how the Python parser splits source code. """ - idx = 0 lines = [] - next_line = '' - while idx < len(source): - c = source[idx] - next_line += c - idx += 1 - # Keep \r\n together - if c == '\r' and idx < len(source) and source[idx] == '\n': - next_line += '\n' - idx += 1 - if c in '\r\n': - lines.append(next_line) - next_line = '' - - if next_line: - lines.append(next_line) + for lineno, match in enumerate(_line_pattern.finditer(source), 1): + if maxlines is not None and lineno > maxlines: + break + lines.append(match[0]) return lines @@ -360,7 +350,7 @@ def get_source_segment(source, node, *, padded=False): except AttributeError: return None - lines = _splitlines_no_ff(source) + lines = _splitlines_no_ff(source, maxlines=end_lineno+1) if end_lineno == lineno: return lines[lineno].encode()[col_offset:end_col_offset].decode() diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index de5076a96218e0..3a697129e4c914 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1176,6 +1176,9 @@ def writelines(self, list_of_data): return self._buffer.extend([memoryview(data) for data in list_of_data]) self._write_ready() + # If the entire buffer couldn't be written, register a write handler + if self._buffer: + self._loop._add_writer(self._sock_fd, self._write_ready) def can_write_eof(self): return True diff --git a/Lib/bdb.py b/Lib/bdb.py index 7f9b09514ffd00..0f3eec653baaad 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -574,6 +574,8 @@ def format_stack_entry(self, frame_lineno, lprefix=': '): line = linecache.getline(filename, lineno, frame.f_globals) if line: s += lprefix + line.strip() + else: + s += f'{lprefix}Warning: lineno is None' return s # The following methods can be called by clients to use diff --git a/Lib/calendar.py b/Lib/calendar.py index 657396439c91fc..c219f0c218d9fa 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -7,6 +7,7 @@ import sys import datetime +from enum import IntEnum, global_enum import locale as _locale from itertools import repeat @@ -16,6 +17,9 @@ "timegm", "month_name", "month_abbr", "day_name", "day_abbr", "Calendar", "TextCalendar", "HTMLCalendar", "LocaleTextCalendar", "LocaleHTMLCalendar", "weekheader", + "Day", "Month", "JANUARY", "FEBRUARY", "MARCH", + "APRIL", "MAY", "JUNE", "JULY", + "AUGUST", "SEPTEMBER", "OCTOBER", "NOVEMBER", "DECEMBER", "MONDAY", "TUESDAY", "WEDNESDAY", "THURSDAY", "FRIDAY", "SATURDAY", "SUNDAY"] @@ -37,9 +41,35 @@ def __str__(self): return "bad weekday number %r; must be 0 (Monday) to 6 (Sunday)" % self.weekday -# Constants for months referenced later -January = 1 -February = 2 +# Constants for months +@global_enum +class Month(IntEnum): + JANUARY = 1 + FEBRUARY = 2 + MARCH = 3 + APRIL = 4 + MAY = 5 + JUNE = 6 + JULY = 7 + AUGUST = 8 + SEPTEMBER = 9 + OCTOBER = 10 + NOVEMBER = 11 + DECEMBER = 12 + + +# Constants for days +@global_enum +class Day(IntEnum): + MONDAY = 0 + TUESDAY = 1 + WEDNESDAY = 2 + THURSDAY = 3 + FRIDAY = 4 + SATURDAY = 5 + SUNDAY = 6 + + # Number of days per month (except for February in leap years) mdays = [0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] @@ -95,9 +125,6 @@ def __len__(self): month_name = _localized_month('%B') month_abbr = _localized_month('%b') -# Constants for weekdays -(MONDAY, TUESDAY, WEDNESDAY, THURSDAY, FRIDAY, SATURDAY, SUNDAY) = range(7) - def isleap(year): """Return True for leap years, False for non-leap years.""" @@ -125,12 +152,12 @@ def monthrange(year, month): if not 1 <= month <= 12: raise IllegalMonthError(month) day1 = weekday(year, month, 1) - ndays = mdays[month] + (month == February and isleap(year)) + ndays = mdays[month] + (month == FEBRUARY and isleap(year)) return day1, ndays def _monthlen(year, month): - return mdays[month] + (month == February and isleap(year)) + return mdays[month] + (month == FEBRUARY and isleap(year)) def _prevmonth(year, month): @@ -260,10 +287,7 @@ def yeardatescalendar(self, year, width=3): Each month contains between 4 and 6 weeks and each week contains 1-7 days. Days are datetime.date objects. """ - months = [ - self.monthdatescalendar(year, i) - for i in range(January, January+12) - ] + months = [self.monthdatescalendar(year, m) for m in Month] return [months[i:i+width] for i in range(0, len(months), width) ] def yeardays2calendar(self, year, width=3): @@ -273,10 +297,7 @@ def yeardays2calendar(self, year, width=3): (day number, weekday number) tuples. Day numbers outside this month are zero. """ - months = [ - self.monthdays2calendar(year, i) - for i in range(January, January+12) - ] + months = [self.monthdays2calendar(year, m) for m in Month] return [months[i:i+width] for i in range(0, len(months), width) ] def yeardayscalendar(self, year, width=3): @@ -285,10 +306,7 @@ def yeardayscalendar(self, year, width=3): yeardatescalendar()). Entries in the week lists are day numbers. Day numbers outside this month are zero. """ - months = [ - self.monthdayscalendar(year, i) - for i in range(January, January+12) - ] + months = [self.monthdayscalendar(year, m) for m in Month] return [months[i:i+width] for i in range(0, len(months), width) ] @@ -509,7 +527,7 @@ def formatyear(self, theyear, width=3): a('\n') a('%s' % ( width, self.cssclass_year_head, theyear)) - for i in range(January, January+12, width): + for i in range(JANUARY, JANUARY+12, width): # months in this row months = range(i, min(i+width, 13)) a('') diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index a5393aad4249c0..03ca2d7e18f6f0 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -45,6 +45,11 @@ else: _collections_abc.MutableSequence.register(deque) +try: + from _collections import _deque_iterator +except ImportError: + pass + try: from _collections import defaultdict except ImportError: diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 30d9ac25b2bbec..b5acbcb9e6d77c 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -441,7 +441,16 @@ def __exit__(self, exctype, excinst, exctb): # exactly reproduce the limitations of the CPython interpreter. # # See http://bugs.python.org/issue12029 for more details - return exctype is not None and issubclass(exctype, self._exceptions) + if exctype is None: + return + if issubclass(exctype, self._exceptions): + return True + if issubclass(exctype, ExceptionGroup): + match, rest = excinst.split(self._exceptions) + if rest is None: + return True + raise rest + return False class _BaseExitStack: diff --git a/Lib/csv.py b/Lib/csv.py index 4ef8be45ca9e0a..77f30c8d2b1f61 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -9,12 +9,14 @@ unregister_dialect, get_dialect, list_dialects, \ field_size_limit, \ QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ + QUOTE_STRINGS, QUOTE_NOTNULL, \ __doc__ from _csv import Dialect as _Dialect from io import StringIO __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", + "QUOTE_STRINGS", "QUOTE_NOTNULL", "Error", "Dialect", "__doc__", "excel", "excel_tab", "field_size_limit", "reader", "writer", "register_dialect", "get_dialect", "list_dialects", "Sniffer", diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 7558287bad449e..a73cdc22a5f4b3 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -222,6 +222,29 @@ def __repr__(self): # https://bugs.python.org/issue33453 for details. _MODULE_IDENTIFIER_RE = re.compile(r'^(?:\s*(\w+)\s*\.)?\s*(\w+)') +# Atomic immutable types which don't require any recursive handling and for which deepcopy +# returns the same object. We can provide a fast-path for these types in asdict and astuple. +_ATOMIC_TYPES = frozenset({ + # Common JSON Serializable types + types.NoneType, + bool, + int, + float, + str, + # Other common types + complex, + bytes, + # Other types that are also unaffected by deepcopy + types.EllipsisType, + types.NotImplementedType, + types.CodeType, + types.BuiltinFunctionType, + types.FunctionType, + type, + range, + property, +}) + # This function's logic is copied from "recursive_repr" function in # reprlib module to avoid dependency. def _recursive_repr(user_function): @@ -1105,8 +1128,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if not getattr(cls, '__doc__'): # Create a class doc-string. - cls.__doc__ = (cls.__name__ + - str(inspect.signature(cls)).replace(' -> None', '')) + try: + # In some cases fetching a signature is not possible. + # But, we surely should not fail in this case. + text_sig = str(inspect.signature(cls)).replace(' -> None', '') + except (TypeError, ValueError): + text_sig = '' + cls.__doc__ = (cls.__name__ + text_sig) if match_args: # I could probably compute this once @@ -1291,7 +1319,9 @@ class C: def _asdict_inner(obj, dict_factory): - if _is_dataclass_instance(obj): + if type(obj) in _ATOMIC_TYPES: + return obj + elif _is_dataclass_instance(obj): result = [] for f in fields(obj): value = _asdict_inner(getattr(obj, f.name), dict_factory) @@ -1363,7 +1393,9 @@ class C: def _astuple_inner(obj, tuple_factory): - if _is_dataclass_instance(obj): + if type(obj) in _ATOMIC_TYPES: + return obj + elif _is_dataclass_instance(obj): result = [] for f in fields(obj): value = _astuple_inner(getattr(obj, f.name), tuple_factory) diff --git a/Lib/datetime.py b/Lib/datetime.py index 637144637485bc..09a2d2d5381c34 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1965,6 +1965,11 @@ def replace(self, year=None, month=None, day=None, hour=None, def _local_timezone(self): if self.tzinfo is None: ts = self._mktime() + # Detect gap + ts2 = self.replace(fold=1-self.fold)._mktime() + if ts2 != ts: # This happens in a gap or a fold + if (ts2 > ts) == self.fold: + ts = ts2 else: ts = (self - _EPOCH) // timedelta(seconds=1) localtm = _time.localtime(ts) diff --git a/Lib/dis.py b/Lib/dis.py index b39b2835330135..8af84c00d0cf64 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -41,6 +41,7 @@ FOR_ITER = opmap['FOR_ITER'] SEND = opmap['SEND'] LOAD_ATTR = opmap['LOAD_ATTR'] +LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] CACHE = opmap["CACHE"] @@ -64,10 +65,10 @@ def _try_compile(source, name): expect code objects """ try: - c = compile(source, name, 'eval') + return compile(source, name, 'eval') except SyntaxError: - c = compile(source, name, 'exec') - return c + pass + return compile(source, name, 'exec') def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False): """Disassemble classes, methods, functions, and other compiled objects. @@ -475,6 +476,10 @@ def _get_instructions_bytes(code, varname_from_oparg=None, argval, argrepr = _get_name_info(arg//2, get_name) if (arg & 1) and argrepr: argrepr = "NULL|self + " + argrepr + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = "NULL|self + " + argrepr else: argval, argrepr = _get_name_info(arg, get_name) elif deop in hasjabs: diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index e637e6df06612d..0d6bd812475eea 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1987,7 +1987,7 @@ def get_address_list(value): try: token, value = get_address(value) address_list.append(token) - except errors.HeaderParseError as err: + except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) @@ -2096,7 +2096,7 @@ def get_msg_id(value): except errors.HeaderParseError: try: token, value = get_no_fold_literal(value) - except errors.HeaderParseError as e: + except errors.HeaderParseError: try: token, value = get_domain(value) msg_id.defects.append(errors.ObsoleteHeaderDefect( @@ -2443,7 +2443,6 @@ def get_parameter(value): raise errors.HeaderParseError("Parameter not followed by '='") param.append(ValueTerminal('=', 'parameter-separator')) value = value[1:] - leader = None if value and value[0] in CFWS_LEADER: token, value = get_cfws(value) param.append(token) @@ -2568,7 +2567,7 @@ def parse_mime_parameters(value): try: token, value = get_parameter(value) mime_parameters.append(token) - except errors.HeaderParseError as err: + except errors.HeaderParseError: leader = None if value[0] in CFWS_LEADER: leader, value = get_cfws(value) @@ -2626,7 +2625,6 @@ def parse_content_type_header(value): don't do that. """ ctype = ContentType() - recover = False if not value: ctype.defects.append(errors.HeaderMissingRequiredValue( "Missing content type specification")) diff --git a/Lib/email/charset.py b/Lib/email/charset.py index 9af269442fb8af..043801107b60e5 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -341,7 +341,6 @@ def header_encode_lines(self, string, maxlengths): if not lines and not current_line: lines.append(None) else: - separator = (' ' if lines else '') joined_line = EMPTYSTRING.join(current_line) header_bytes = _encode(joined_line, codec) lines.append(encoder(header_bytes)) diff --git a/Lib/email/feedparser.py b/Lib/email/feedparser.py index 6bc4e0c4e59895..885097c7dda067 100644 --- a/Lib/email/feedparser.py +++ b/Lib/email/feedparser.py @@ -264,7 +264,7 @@ def _parsegen(self): yield NeedMoreData continue break - msg = self._pop_message() + self._pop_message() # We need to pop the EOF matcher in order to tell if we're at # the end of the current file, not the end of the last block # of message headers. diff --git a/Lib/email/message.py b/Lib/email/message.py index b540c33984a753..411118c74dabb4 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -14,7 +14,7 @@ # Intrapackage imports from email import utils from email import errors -from email._policybase import Policy, compat32 +from email._policybase import compat32 from email import charset as _charset from email._encoded_words import decode_b Charset = _charset.Charset diff --git a/Lib/email/mime/text.py b/Lib/email/mime/text.py index dfe53c426b2ac4..7672b789138600 100644 --- a/Lib/email/mime/text.py +++ b/Lib/email/mime/text.py @@ -6,7 +6,6 @@ __all__ = ['MIMEText'] -from email.charset import Charset from email.mime.nonmultipart import MIMENonMultipart @@ -36,6 +35,6 @@ def __init__(self, _text, _subtype='plain', _charset=None, *, policy=None): _charset = 'utf-8' MIMENonMultipart.__init__(self, 'text', _subtype, policy=policy, - **{'charset': str(_charset)}) + charset=str(_charset)) self.set_payload(_text, _charset) diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 00e77749e25e77..4278422dfacc9f 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -9,11 +9,9 @@ __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('setuptools', 'pip') -_SETUPTOOLS_VERSION = "65.5.0" -_PIP_VERSION = "23.0.1" +_PACKAGE_NAMES = ('pip',) +_PIP_VERSION = "23.1.1" _PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION, "py3"), ("pip", _PIP_VERSION, "py3"), ] @@ -153,17 +151,17 @@ def _bootstrap(*, root=None, upgrade=False, user=False, _disable_pip_configuration_settings() - # By default, installing pip and setuptools installs all of the + # By default, installing pip installs all of the # following scripts (X.Y == running Python version): # - # pip, pipX, pipX.Y, easy_install, easy_install-X.Y + # pip, pipX, pipX.Y # # pip 1.5+ allows ensurepip to request that some of those be left out if altinstall: - # omit pip, pipX and easy_install + # omit pip, pipX os.environ["ENSUREPIP_OPTIONS"] = "altinstall" elif not default_pip: - # omit pip and easy_install + # omit pip os.environ["ENSUREPIP_OPTIONS"] = "install" with tempfile.TemporaryDirectory() as tmpdir: @@ -271,14 +269,14 @@ def _main(argv=None): action="store_true", default=False, help=("Make an alternate install, installing only the X.Y versioned " - "scripts (Default: pipX, pipX.Y, easy_install-X.Y)."), + "scripts (Default: pipX, pipX.Y)."), ) parser.add_argument( "--default-pip", action="store_true", default=False, help=("Make a default pip install, installing the unqualified pip " - "and easy_install in addition to the versioned scripts."), + "in addition to the versioned scripts."), ) args = parser.parse_args(argv) diff --git a/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.1.1-py3-none-any.whl similarity index 76% rename from Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-23.1.1-py3-none-any.whl index a855dc40e8630d..dee4c0304b2c36 100644 Binary files a/Lib/ensurepip/_bundled/pip-23.0.1-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-23.1.1-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl deleted file mode 100644 index 123a13e2c6b254..00000000000000 Binary files a/Lib/ensurepip/_bundled/setuptools-65.5.0-py3-none-any.whl and /dev/null differ diff --git a/Lib/enum.py b/Lib/enum.py index 10902c4b202a2d..6e497f7ef6a7de 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -190,6 +190,8 @@ class property(DynamicClassAttribute): """ member = None + _attr_type = None + _cls_type = None def __get__(self, instance, ownerclass=None): if instance is None: @@ -199,33 +201,36 @@ def __get__(self, instance, ownerclass=None): raise AttributeError( '%r has no attribute %r' % (ownerclass, self.name) ) - else: - if self.fget is None: - # look for a member by this name. - try: - return ownerclass._member_map_[self.name] - except KeyError: - raise AttributeError( - '%r has no attribute %r' % (ownerclass, self.name) - ) from None - else: - return self.fget(instance) + if self.fget is not None: + # use previous enum.property + return self.fget(instance) + elif self._attr_type == 'attr': + # look up previous attibute + return getattr(self._cls_type, self.name) + elif self._attr_type == 'desc': + # use previous descriptor + return getattr(instance._value_, self.name) + # look for a member by this name. + try: + return ownerclass._member_map_[self.name] + except KeyError: + raise AttributeError( + '%r has no attribute %r' % (ownerclass, self.name) + ) from None def __set__(self, instance, value): - if self.fset is None: - raise AttributeError( - " cannot set attribute %r" % (self.clsname, self.name) - ) - else: + if self.fset is not None: return self.fset(instance, value) + raise AttributeError( + " cannot set attribute %r" % (self.clsname, self.name) + ) def __delete__(self, instance): - if self.fdel is None: - raise AttributeError( - " cannot delete attribute %r" % (self.clsname, self.name) - ) - else: + if self.fdel is not None: return self.fdel(instance) + raise AttributeError( + " cannot delete attribute %r" % (self.clsname, self.name) + ) def __set_name__(self, ownerclass, name): self.name = name @@ -275,6 +280,13 @@ def __set_name__(self, enum_class, member_name): enum_member.__objclass__ = enum_class enum_member.__init__(*args) enum_member._sort_order_ = len(enum_class._member_names_) + + if Flag is not None and issubclass(enum_class, Flag): + enum_class._flag_mask_ |= value + if _is_single_bit(value): + enum_class._singles_mask_ |= value + enum_class._all_bits_ = 2 ** ((enum_class._flag_mask_).bit_length()) - 1 + # If another member with the same value was already defined, the # new member becomes an alias to the existing one. try: @@ -306,27 +318,38 @@ def __set_name__(self, enum_class, member_name): enum_class._member_names_.append(member_name) # if necessary, get redirect in place and then add it to _member_map_ found_descriptor = None + descriptor_type = None + class_type = None for base in enum_class.__mro__[1:]: - descriptor = base.__dict__.get(member_name) - if descriptor is not None: - if isinstance(descriptor, (property, DynamicClassAttribute)): - found_descriptor = descriptor + attr = base.__dict__.get(member_name) + if attr is not None: + if isinstance(attr, (property, DynamicClassAttribute)): + found_descriptor = attr + class_type = base + descriptor_type = 'enum' break - elif ( - hasattr(descriptor, 'fget') and - hasattr(descriptor, 'fset') and - hasattr(descriptor, 'fdel') - ): - found_descriptor = descriptor + elif _is_descriptor(attr): + found_descriptor = attr + descriptor_type = descriptor_type or 'desc' + class_type = class_type or base continue + else: + descriptor_type = 'attr' + class_type = base if found_descriptor: redirect = property() redirect.member = enum_member redirect.__set_name__(enum_class, member_name) - # earlier descriptor found; copy fget, fset, fdel to this one. - redirect.fget = found_descriptor.fget - redirect.fset = found_descriptor.fset - redirect.fdel = found_descriptor.fdel + if descriptor_type in ('enum','desc'): + # earlier descriptor found; copy fget, fset, fdel to this one. + redirect.fget = getattr(found_descriptor, 'fget', None) + redirect._get = getattr(found_descriptor, '__get__', None) + redirect.fset = getattr(found_descriptor, 'fset', None) + redirect._set = getattr(found_descriptor, '__set__', None) + redirect.fdel = getattr(found_descriptor, 'fdel', None) + redirect._del = getattr(found_descriptor, '__delete__', None) + redirect._attr_type = descriptor_type + redirect._cls_type = class_type setattr(enum_class, member_name, redirect) else: setattr(enum_class, member_name, enum_member) @@ -532,12 +555,8 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k classdict['_use_args_'] = use_args # # convert future enum members into temporary _proto_members - # and record integer values in case this will be a Flag - flag_mask = 0 for name in member_names: value = classdict[name] - if isinstance(value, int): - flag_mask |= value classdict[name] = _proto_member(value) # # house-keeping structures @@ -554,8 +573,9 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k boundary or getattr(first_enum, '_boundary_', None) ) - classdict['_flag_mask_'] = flag_mask - classdict['_all_bits_'] = 2 ** ((flag_mask).bit_length()) - 1 + classdict['_flag_mask_'] = 0 + classdict['_singles_mask_'] = 0 + classdict['_all_bits_'] = 0 classdict['_inverted_'] = None try: exc = None @@ -644,21 +664,10 @@ def __new__(metacls, cls, bases, classdict, *, boundary=None, _simple=False, **k ): delattr(enum_class, '_boundary_') delattr(enum_class, '_flag_mask_') + delattr(enum_class, '_singles_mask_') delattr(enum_class, '_all_bits_') delattr(enum_class, '_inverted_') elif Flag is not None and issubclass(enum_class, Flag): - # ensure _all_bits_ is correct and there are no missing flags - single_bit_total = 0 - multi_bit_total = 0 - for flag in enum_class._member_map_.values(): - flag_value = flag._value_ - if _is_single_bit(flag_value): - single_bit_total |= flag_value - else: - # multi-bit flags are considered aliases - multi_bit_total |= flag_value - enum_class._flag_mask_ = single_bit_total - # # set correct __iter__ member_list = [m._value_ for m in enum_class] if member_list != sorted(member_list): @@ -974,6 +983,7 @@ def _find_data_repr_(mcls, class_name, bases): @classmethod def _find_data_type_(mcls, class_name, bases): + # a datatype has a __new__ method, or a __dataclass_fields__ attribute data_types = set() base_chain = set() for chain in bases: @@ -986,7 +996,7 @@ def _find_data_type_(mcls, class_name, bases): if base._member_type_ is not object: data_types.add(base._member_type_) break - elif '__new__' in base.__dict__ or '__init__' in base.__dict__: + elif '__new__' in base.__dict__ or '__dataclass_fields__' in base.__dict__: data_types.add(candidate or base) break else: @@ -1303,8 +1313,8 @@ def _reduce_ex_by_global_name(self, proto): class FlagBoundary(StrEnum): """ control how out of range values are handled - "strict" -> error is raised - "conform" -> extra bits are discarded [default for Flag] + "strict" -> error is raised [default for Flag] + "conform" -> extra bits are discarded "eject" -> lose flag status "keep" -> keep flag status and all bits [default for IntFlag] """ @@ -1315,7 +1325,7 @@ class FlagBoundary(StrEnum): STRICT, CONFORM, EJECT, KEEP = FlagBoundary -class Flag(Enum, boundary=CONFORM): +class Flag(Enum, boundary=STRICT): """ Support for flags """ @@ -1394,6 +1404,7 @@ def _missing_(cls, value): # - value must not include any skipped flags (e.g. if bit 2 is not # defined, then 0d10 is invalid) flag_mask = cls._flag_mask_ + singles_mask = cls._singles_mask_ all_bits = cls._all_bits_ neg_value = None if ( @@ -1425,7 +1436,8 @@ def _missing_(cls, value): value = all_bits + 1 + value # get members and unknown unknown = value & ~flag_mask - member_value = value & flag_mask + aliases = value & ~singles_mask + member_value = value & singles_mask if unknown and cls._boundary_ is not KEEP: raise ValueError( '%s(%r) --> unknown values %r [%s]' @@ -1439,11 +1451,25 @@ def _missing_(cls, value): pseudo_member = cls._member_type_.__new__(cls, value) if not hasattr(pseudo_member, '_value_'): pseudo_member._value_ = value - if member_value: - pseudo_member._name_ = '|'.join([ - m._name_ for m in cls._iter_member_(member_value) - ]) - if unknown: + if member_value or aliases: + members = [] + combined_value = 0 + for m in cls._iter_member_(member_value): + members.append(m) + combined_value |= m._value_ + if aliases: + value = member_value | aliases + for n, pm in cls._member_map_.items(): + if pm not in members and pm._value_ and pm._value_ & value == pm._value_: + members.append(pm) + combined_value |= pm._value_ + unknown = value ^ combined_value + pseudo_member._name_ = '|'.join([m._name_ for m in members]) + if not combined_value: + pseudo_member._name_ = None + elif unknown and cls._boundary_ is STRICT: + raise ValueError('%r: no members with value %r' % (cls, unknown)) + elif unknown: pseudo_member._name_ += '|%s' % cls._numeric_repr_(unknown) else: pseudo_member._name_ = None @@ -1675,6 +1701,7 @@ def convert_class(cls): body['_boundary_'] = boundary or etype._boundary_ body['_flag_mask_'] = None body['_all_bits_'] = None + body['_singles_mask_'] = None body['_inverted_'] = None body['__or__'] = Flag.__or__ body['__xor__'] = Flag.__xor__ @@ -1750,7 +1777,8 @@ def convert_class(cls): else: multi_bits |= value gnv_last_values.append(value) - enum_class._flag_mask_ = single_bits + enum_class._flag_mask_ = single_bits | multi_bits + enum_class._singles_mask_ = single_bits enum_class._all_bits_ = 2 ** ((single_bits|multi_bits).bit_length()) - 1 # set correct __iter__ member_list = [m._value_ for m in enum_class] diff --git a/Lib/idlelib/calltip_w.py b/Lib/idlelib/calltip_w.py index 1e0404aa49f562..278546064adde2 100644 --- a/Lib/idlelib/calltip_w.py +++ b/Lib/idlelib/calltip_w.py @@ -25,7 +25,7 @@ def __init__(self, text_widget): text_widget: a Text widget with code for which call-tips are desired """ # Note: The Text widget will be accessible as self.anchor_widget - super(CalltipWindow, self).__init__(text_widget) + super().__init__(text_widget) self.label = self.text = None self.parenline = self.parencol = self.lastline = None @@ -54,7 +54,7 @@ def position_window(self): return self.lastline = curline self.anchor_widget.see("insert") - super(CalltipWindow, self).position_window() + super().position_window() def showtip(self, text, parenleft, parenright): """Show the call-tip, bind events which will close it and reposition it. @@ -73,7 +73,7 @@ def showtip(self, text, parenleft, parenright): self.parenline, self.parencol = map( int, self.anchor_widget.index(parenleft).split(".")) - super(CalltipWindow, self).showtip() + super().showtip() self._bind_events() @@ -143,7 +143,7 @@ def hidetip(self): # ValueError may be raised by MultiCall pass - super(CalltipWindow, self).hidetip() + super().hidetip() def _bind_events(self): """Bind event handlers.""" diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index ccd03e46e16147..452c62b42655b3 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -49,9 +49,9 @@ def __frame2message(self, frame): filename = code.co_filename lineno = frame.f_lineno basename = os.path.basename(filename) - message = "%s:%s" % (basename, lineno) + message = f"{basename}:{lineno}" if code.co_name != "?": - message = "%s: %s()" % (message, code.co_name) + message = f"{message}: {code.co_name}()" return message @@ -213,7 +213,8 @@ def interaction(self, message, frame, info=None): m1 = "%s" % str(type) if value is not None: try: - m1 = "%s: %s" % (m1, str(value)) + # TODO redo entire section, tries not needed. + m1 = f"{m1}: {value}" except: pass bg = "yellow" diff --git a/Lib/idlelib/debugobj.py b/Lib/idlelib/debugobj.py index 5a4c9978842035..71d01c7070df54 100644 --- a/Lib/idlelib/debugobj.py +++ b/Lib/idlelib/debugobj.py @@ -87,7 +87,7 @@ def GetSubList(self): continue def setfunction(value, key=key, object=self.object): object[key] = value - item = make_objecttreeitem("%r:" % (key,), value, setfunction) + item = make_objecttreeitem(f"{key!r}:", value, setfunction) sublist.append(item) return sublist diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 08d6aa2efde22a..505815502600b1 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -38,12 +38,13 @@ def _sphinx_version(): "Format sys.version_info to produce the Sphinx version string used to install the chm docs" major, minor, micro, level, serial = sys.version_info - release = '%s%s' % (major, minor) - release += '%s' % (micro,) + # TODO remove unneeded function since .chm no longer installed + release = f'{major}{minor}' + release += f'{micro}' if level == 'candidate': - release += 'rc%s' % (serial,) + release += f'rc{serial}' elif level != 'final': - release += '%s%s' % (level[0], serial) + release += f'{level[0]}{serial}' return release @@ -950,7 +951,7 @@ def update_recent_files_list(self, new_file=None): rf_list = [] file_path = self.recent_files_path if file_path and os.path.exists(file_path): - with open(file_path, 'r', + with open(file_path, encoding='utf_8', errors='replace') as rf_list_file: rf_list = rf_list_file.readlines() if new_file: @@ -1458,7 +1459,7 @@ def newline_and_indent_event(self, event): else: self.reindent_to(y.compute_backslash_indent()) else: - assert 0, "bogus continuation type %r" % (c,) + assert 0, f"bogus continuation type {c!r}" return "break" # This line starts a brand new statement; indent relative to diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index 254f5caf6b81b0..f87781d2570fe0 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -22,7 +22,7 @@ def open(self, filename, action=None): # This can happen when bad filename is passed on command line: messagebox.showerror( "File Error", - "%r is a directory." % (filename,), + f"{filename!r} is a directory.", master=self.root) return None key = os.path.normcase(filename) @@ -90,7 +90,7 @@ def filename_changed_edit(self, edit): self.inversedict[conflict] = None messagebox.showerror( "Name Conflict", - "You now have multiple edit windows open for %r" % (filename,), + f"You now have multiple edit windows open for {filename!r}", master=self.root) self.dict[newkey] = edit self.inversedict[edit] = newkey diff --git a/Lib/idlelib/idle_test/test_config.py b/Lib/idlelib/idle_test/test_config.py index 697fda527968de..08ed76fe288294 100644 --- a/Lib/idlelib/idle_test/test_config.py +++ b/Lib/idlelib/idle_test/test_config.py @@ -191,7 +191,7 @@ def setUpClass(cls): idle_dir = os.path.abspath(sys.path[0]) for ctype in conf.config_types: config_path = os.path.join(idle_dir, '../config-%s.def' % ctype) - with open(config_path, 'r') as f: + with open(config_path) as f: cls.config_string[ctype] = f.read() cls.orig_warn = config._warn diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index e347bfca7f191a..d6e85ad674417c 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -159,7 +159,7 @@ def test_file_line_helper(self, mock_open): for line, expected_output in test_lines: self.assertEqual(flh(line), expected_output) if expected_output: - mock_open.assert_called_with(expected_output[0], 'r') + mock_open.assert_called_with(expected_output[0]) if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 049531e66a414e..5506fd2b0e22a5 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -328,7 +328,7 @@ def test_scroll(self): self.assertEqual(self.linenumber.sidebar_text.index('@0,0'), '11.0') # Generate a mouse-wheel event and make sure it scrolled up or down. - # The meaning of the "delta" is OS-dependant, so this just checks for + # The meaning of the "delta" is OS-dependent, so this just checks for # any change. self.linenumber.sidebar_text.event_generate('', x=0, y=0, @@ -691,7 +691,7 @@ def test_mousewheel(self): self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll up using the event. - # The meaning delta is platform-dependant. + # The meaning of delta is platform-dependent. delta = -1 if sys.platform == 'darwin' else 120 sidebar.canvas.event_generate('', x=0, y=0, delta=delta) yield diff --git a/Lib/idlelib/multicall.py b/Lib/idlelib/multicall.py index dc02001292fc14..0200f445cc9340 100644 --- a/Lib/idlelib/multicall.py +++ b/Lib/idlelib/multicall.py @@ -52,9 +52,9 @@ _modifier_masks = (MC_CONTROL, MC_ALT, MC_SHIFT, MC_META) # a dictionary to map a modifier name into its number -_modifier_names = dict([(name, number) +_modifier_names = {name: number for number in range(len(_modifiers)) - for name in _modifiers[number]]) + for name in _modifiers[number]} # In 3.4, if no shell window is ever open, the underlying Tk widget is # destroyed before .__del__ methods here are called. The following @@ -134,7 +134,7 @@ def nbits(n): return nb statelist = [] for state in states: - substates = list(set(state & x for x in states)) + substates = list({state & x for x in states}) substates.sort(key=nbits, reverse=True) statelist.append(substates) return statelist @@ -258,9 +258,9 @@ def __del__(self): _binder_classes = (_ComplexBinder,) * 4 + (_SimpleBinder,) * (len(_types)-4) # A dictionary to map a type name into its number -_type_names = dict([(name, number) +_type_names = {name: number for number in range(len(_types)) - for name in _types[number]]) + for name in _types[number]} _keysym_re = re.compile(r"^\w+$") _button_re = re.compile(r"^[1-5]$") diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 5ab08bbaf4bc95..ac67c904ab9797 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -42,7 +42,7 @@ def file_line_helper(line): if match: filename, lineno = match.group(1, 2) try: - f = open(filename, "r") + f = open(filename) f.close() break except OSError: diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index e68233a5a4131e..bdde156166171b 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -249,7 +249,7 @@ def store_file_breaks(self): breaks = self.breakpoints filename = self.io.filename try: - with open(self.breakpointPath, "r") as fp: + with open(self.breakpointPath) as fp: lines = fp.readlines() except OSError: lines = [] @@ -279,7 +279,7 @@ def restore_file_breaks(self): if filename is None: return if os.path.isfile(self.breakpointPath): - with open(self.breakpointPath, "r") as fp: + with open(self.breakpointPath) as fp: lines = fp.readlines() for line in lines: if line.startswith(filename + '='): @@ -441,7 +441,7 @@ def build_subprocess_arglist(self): # run from the IDLE source directory. del_exitf = idleConf.GetOption('main', 'General', 'delete-exitfunc', default=False, type='bool') - command = "__import__('idlelib.run').run.main(%r)" % (del_exitf,) + command = f"__import__('idlelib.run').run.main({del_exitf!r})" return [sys.executable] + w + ["-c", command, str(self.port)] def start_subprocess(self): @@ -574,9 +574,9 @@ def transfer_path(self, with_cwd=False): self.runcommand("""if 1: import sys as _sys - _sys.path = %r + _sys.path = {!r} del _sys - \n""" % (path,)) + \n""".format(path)) active_seq = None @@ -703,14 +703,14 @@ def stuffsource(self, source): def prepend_syspath(self, filename): "Prepend sys.path with file's directory if not already included" self.runcommand("""if 1: - _filename = %r + _filename = {!r} import sys as _sys from os.path import dirname as _dirname _dir = _dirname(_filename) if not _dir in _sys.path: _sys.path.insert(0, _dir) del _filename, _sys, _dirname, _dir - \n""" % (filename,)) + \n""".format(filename)) def showsyntaxerror(self, filename=None): """Override Interactive Interpreter method: Use Colorizing @@ -1536,7 +1536,7 @@ def main(): try: opts, args = getopt.getopt(sys.argv[1:], "c:deihnr:st:") except getopt.error as msg: - print("Error: %s\n%s" % (msg, usage_msg), file=sys.stderr) + print(f"Error: {msg}\n{usage_msg}", file=sys.stderr) sys.exit(2) for o, a in opts: if o == '-c': @@ -1668,9 +1668,9 @@ def main(): if cmd or script: shell.interp.runcommand("""if 1: import sys as _sys - _sys.argv = %r + _sys.argv = {!r} del _sys - \n""" % (sys.argv,)) + \n""".format(sys.argv)) if cmd: shell.interp.execsource(cmd) elif script: diff --git a/Lib/idlelib/redirector.py b/Lib/idlelib/redirector.py index 9ab34c5acfb22c..4928340e98df68 100644 --- a/Lib/idlelib/redirector.py +++ b/Lib/idlelib/redirector.py @@ -47,9 +47,8 @@ def __init__(self, widget): tk.createcommand(w, self.dispatch) def __repr__(self): - return "%s(%s<%s>)" % (self.__class__.__name__, - self.widget.__class__.__name__, - self.widget._w) + w = self.widget + return f"{self.__class__.__name__,}({w.__class__.__name__}<{w._w}>)" def close(self): "Unregister operations and revert redirection created by .__init__." @@ -143,8 +142,7 @@ def __init__(self, redir, operation): self.orig_and_operation = (redir.orig, operation) def __repr__(self): - return "%s(%r, %r)" % (self.__class__.__name__, - self.redir, self.operation) + return f"{self.__class__.__name__,}({self.redir!r}, {self.operation!r})" def __call__(self, *args): return self.tk_call(self.orig_and_operation + args) diff --git a/Lib/idlelib/rpc.py b/Lib/idlelib/rpc.py index 62eec84c9c8d09..b08b80c9004551 100644 --- a/Lib/idlelib/rpc.py +++ b/Lib/idlelib/rpc.py @@ -174,7 +174,7 @@ def localcall(self, seq, request): except TypeError: return ("ERROR", "Bad request format") if oid not in self.objtable: - return ("ERROR", "Unknown object id: %r" % (oid,)) + return ("ERROR", f"Unknown object id: {oid!r}") obj = self.objtable[oid] if methodname == "__methods__": methods = {} @@ -185,7 +185,7 @@ def localcall(self, seq, request): _getattributes(obj, attributes) return ("OK", attributes) if not hasattr(obj, methodname): - return ("ERROR", "Unsupported method name: %r" % (methodname,)) + return ("ERROR", f"Unsupported method name: {methodname!r}") method = getattr(obj, methodname) try: if how == 'CALL': diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index 577c49eb67b20d..84792a82b0022c 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -52,13 +52,13 @@ def idle_formatwarning(message, category, filename, lineno, line=None): """Format warnings the IDLE way.""" s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) + s += f' File \"{filename}\", line {lineno}\n' if line is None: line = linecache.getline(filename, lineno) line = line.strip() if line: s += " %s\n" % line - s += "%s: %s\n" % (category.__name__, message) + s += f"{category.__name__}: {message}\n" return s def idle_showwarning_subproc( @@ -239,6 +239,7 @@ def print_exception(): efile = sys.stderr typ, val, tb = excinfo = sys.exc_info() sys.last_type, sys.last_value, sys.last_traceback = excinfo + sys.last_exc = val seen = set() def print_exc(typ, exc, tb): diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index a66c1a4309a617..23f0f4cb5027ec 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -169,7 +169,7 @@ def view_file(parent, title, filename, encoding, modal=True, wrap='word', with contents of the file. """ try: - with open(filename, 'r', encoding=encoding) as file: + with open(filename, encoding=encoding) as file: contents = file.read() except OSError: showerror(title='File Load Error', diff --git a/Lib/idlelib/tooltip.py b/Lib/idlelib/tooltip.py index d714318dae8ef1..3983690dd41177 100644 --- a/Lib/idlelib/tooltip.py +++ b/Lib/idlelib/tooltip.py @@ -92,7 +92,7 @@ def __init__(self, anchor_widget, hover_delay=1000): e.g. after hovering over the anchor widget with the mouse for enough time. """ - super(OnHoverTooltipBase, self).__init__(anchor_widget) + super().__init__(anchor_widget) self.hover_delay = hover_delay self._after_id = None @@ -107,7 +107,7 @@ def __del__(self): self.anchor_widget.unbind("