From 536bc8a806008fff4880ad11ddc189e4847d0255 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Tue, 24 Sep 2024 14:19:25 +0800 Subject: [PATCH 01/39] Fix typos (#124409) --- .../2024-08-23-11-26-54.gh-issue-122298.ZMyln4.rst | 2 +- .../next/Library/2024-08-22-09-37-48.gh-issue-123213.owmXnP.rst | 2 +- .../next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-11-26-54.gh-issue-122298.ZMyln4.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-11-26-54.gh-issue-122298.ZMyln4.rst index e7645bfc5f323d..722f69616b6b1b 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-11-26-54.gh-issue-122298.ZMyln4.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-08-23-11-26-54.gh-issue-122298.ZMyln4.rst @@ -1,3 +1,3 @@ Restore printout of GC stats when ``gc.set_debug(gc.DEBUG_STATS)`` is -called. This featue was accidentally removed when implementing incremental +called. This feature was accidentally removed when implementing incremental GC. diff --git a/Misc/NEWS.d/next/Library/2024-08-22-09-37-48.gh-issue-123213.owmXnP.rst b/Misc/NEWS.d/next/Library/2024-08-22-09-37-48.gh-issue-123213.owmXnP.rst index 6bbd194b916ec4..5a31a00f2758f4 100644 --- a/Misc/NEWS.d/next/Library/2024-08-22-09-37-48.gh-issue-123213.owmXnP.rst +++ b/Misc/NEWS.d/next/Library/2024-08-22-09-37-48.gh-issue-123213.owmXnP.rst @@ -1,3 +1,3 @@ :meth:`xml.etree.ElementTree.Element.extend` and :class:`~xml.etree.ElementTree.Element` assignment no longer hide the internal -exception if an erronous generator is passed. Patch by Bar Harel. +exception if an erroneous generator is passed. Patch by Bar Harel. diff --git a/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst b/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst index cec7741bcabbda..641c21331e3e54 100644 --- a/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst +++ b/Misc/NEWS.d/next/Library/2024-09-13-10-34-19.gh-issue-123934.yMe7mL.rst @@ -1,2 +1,2 @@ -Fix :class:`unittest.mock.MagicMock` reseting magic methods return values +Fix :class:`unittest.mock.MagicMock` resetting magic methods return values after ``.reset_mock(return_value=True)`` was called. From b82f07653e1e15a48ebaf8de324f52559e470254 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 09:31:10 +0300 Subject: [PATCH 02/39] gh-124130: Increase test coverage for \b and \B in regular expressions (GH-124330) --- Lib/test/test_re.py | 120 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 113 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 7e454cb060d2ae..ff95f54026e172 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -883,31 +883,137 @@ def test_named_unicode_escapes(self): self.checkPatternError(br'\N{LESS-THAN SIGN}', r'bad escape \N', 0) self.checkPatternError(br'[\N{LESS-THAN SIGN}]', r'bad escape \N', 1) - def test_string_boundaries(self): + def test_word_boundaries(self): # See http://bugs.python.org/issue10713 - self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), - "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc").group(1), "abc") + self.assertEqual(re.search(r"\b(abc)\b", "abc", re.ASCII).group(1), "abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc").group(1), b"abc") + self.assertEqual(re.search(br"\b(abc)\b", b"abc", re.LOCALE).group(1), b"abc") + self.assertEqual(re.search(r"\b(ьюя)\b", "ьюя").group(1), "ьюя") + self.assertIsNone(re.search(r"\b(ьюя)\b", "ьюя", re.ASCII)) + # There's a word boundary between a word and a non-word. + self.assertTrue(re.match(r".\b", "a=")) + self.assertTrue(re.match(r".\b", "a=", re.ASCII)) + self.assertTrue(re.match(br".\b", b"a=")) + self.assertTrue(re.match(br".\b", b"a=", re.LOCALE)) + self.assertTrue(re.match(r".\b", "я=")) + self.assertIsNone(re.match(r".\b", "я=", re.ASCII)) + # There's a word boundary between a non-word and a word. + self.assertTrue(re.match(r".\b", "=a")) + self.assertTrue(re.match(r".\b", "=a", re.ASCII)) + self.assertTrue(re.match(br".\b", b"=a")) + self.assertTrue(re.match(br".\b", b"=a", re.LOCALE)) + self.assertTrue(re.match(r".\b", "=я")) + self.assertIsNone(re.match(r".\b", "=я", re.ASCII)) + # There is no word boundary inside a word. + self.assertIsNone(re.match(r".\b", "ab")) + self.assertIsNone(re.match(r".\b", "ab", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"ab")) + self.assertIsNone(re.match(br".\b", b"ab", re.LOCALE)) + self.assertIsNone(re.match(r".\b", "юя")) + self.assertIsNone(re.match(r".\b", "юя", re.ASCII)) + # There is no word boundary between a non-word characters. + self.assertIsNone(re.match(r".\b", "=-")) + self.assertIsNone(re.match(r".\b", "=-", re.ASCII)) + self.assertIsNone(re.match(br".\b", b"=-")) + self.assertIsNone(re.match(br".\b", b"=-", re.LOCALE)) + # There is no non-boundary match between a word and a non-word. + self.assertIsNone(re.match(r".\B", "a=")) + self.assertIsNone(re.match(r".\B", "a=", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"a=")) + self.assertIsNone(re.match(br".\B", b"a=", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "я=")) + self.assertTrue(re.match(r".\B", "я=", re.ASCII)) + # There is no non-boundary match between a non-word and a word. + self.assertIsNone(re.match(r".\B", "=a")) + self.assertIsNone(re.match(r".\B", "=a", re.ASCII)) + self.assertIsNone(re.match(br".\B", b"=a")) + self.assertIsNone(re.match(br".\B", b"=a", re.LOCALE)) + self.assertIsNone(re.match(r".\B", "=я")) + self.assertTrue(re.match(r".\B", "=я", re.ASCII)) + # There's a non-boundary match inside a word. + self.assertTrue(re.match(r".\B", "ab")) + self.assertTrue(re.match(r".\B", "ab", re.ASCII)) + self.assertTrue(re.match(br".\B", b"ab")) + self.assertTrue(re.match(br".\B", b"ab", re.LOCALE)) + self.assertTrue(re.match(r".\B", "юя")) + self.assertTrue(re.match(r".\B", "юя", re.ASCII)) + # There's a non-boundary match between a non-word characters. + self.assertTrue(re.match(r".\B", "=-")) + self.assertTrue(re.match(r".\B", "=-", re.ASCII)) + self.assertTrue(re.match(br".\B", b"=-")) + self.assertTrue(re.match(br".\B", b"=-", re.LOCALE)) # There's a word boundary at the start of a string. self.assertTrue(re.match(r"\b", "abc")) + self.assertTrue(re.match(r"\b", "abc", re.ASCII)) + self.assertTrue(re.match(br"\b", b"abc")) + self.assertTrue(re.match(br"\b", b"abc", re.LOCALE)) + self.assertTrue(re.match(r"\b", "ьюя")) + self.assertIsNone(re.match(r"\b", "ьюя", re.ASCII)) + # There's a word boundary at the end of a string. + self.assertTrue(re.fullmatch(r".+\b", "abc")) + self.assertTrue(re.fullmatch(r".+\b", "abc", re.ASCII)) + self.assertTrue(re.fullmatch(br".+\b", b"abc")) + self.assertTrue(re.fullmatch(br".+\b", b"abc", re.LOCALE)) + self.assertTrue(re.fullmatch(r".+\b", "ьюя")) + self.assertIsNone(re.search(r"\b", "ьюя", re.ASCII)) # A non-empty string includes a non-boundary zero-length match. - self.assertTrue(re.search(r"\B", "abc")) + self.assertEqual(re.search(r"\B", "abc").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "abc", re.ASCII).span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc").span(), (1, 1)) + self.assertEqual(re.search(br"\B", b"abc", re.LOCALE).span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя").span(), (1, 1)) + self.assertEqual(re.search(r"\B", "ьюя", re.ASCII).span(), (0, 0)) # There is no non-boundary match at the start of a string. - self.assertFalse(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc")) + self.assertIsNone(re.match(r"\B", "abc", re.ASCII)) + self.assertIsNone(re.match(br"\B", b"abc")) + self.assertIsNone(re.match(br"\B", b"abc", re.LOCALE)) + self.assertIsNone(re.match(r"\B", "ьюя")) + self.assertTrue(re.match(r"\B", "ьюя", re.ASCII)) + # There is no non-boundary match at the end of a string. + self.assertIsNone(re.fullmatch(r".+\B", "abc")) + self.assertIsNone(re.fullmatch(r".+\B", "abc", re.ASCII)) + self.assertIsNone(re.fullmatch(br".+\B", b"abc")) + self.assertIsNone(re.fullmatch(br".+\B", b"abc", re.LOCALE)) + self.assertIsNone(re.fullmatch(r".+\B", "ьюя")) + self.assertTrue(re.fullmatch(r".+\B", "ьюя", re.ASCII)) # However, an empty string contains no word boundaries, and also no # non-boundaries. - self.assertIsNone(re.search(r"\B", "")) + self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\b", "", re.ASCII)) + self.assertIsNone(re.search(br"\b", b"")) + self.assertIsNone(re.search(br"\b", b"", re.LOCALE)) # This one is questionable and different from the perlre behaviour, # but describes current behavior. - self.assertIsNone(re.search(r"\b", "")) + self.assertIsNone(re.search(r"\B", "")) + self.assertIsNone(re.search(r"\B", "", re.ASCII)) + self.assertIsNone(re.search(br"\B", b"")) + self.assertIsNone(re.search(br"\B", b"", re.LOCALE)) # A single word-character string has two boundaries, but no # non-boundary gaps. self.assertEqual(len(re.findall(r"\b", "a")), 2) + self.assertEqual(len(re.findall(r"\b", "a", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\b", b"a")), 2) + self.assertEqual(len(re.findall(br"\b", b"a", re.LOCALE)), 2) self.assertEqual(len(re.findall(r"\B", "a")), 0) + self.assertEqual(len(re.findall(r"\B", "a", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\B", b"a")), 0) + self.assertEqual(len(re.findall(br"\B", b"a", re.LOCALE)), 0) # If there are no words, there are no boundaries self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) self.assertEqual(len(re.findall(r"\b", " ")), 0) + self.assertEqual(len(re.findall(r"\b", " ", re.ASCII)), 0) + self.assertEqual(len(re.findall(br"\b", b" ")), 0) + self.assertEqual(len(re.findall(br"\b", b" ", re.LOCALE)), 0) # Can match around the whitespace. self.assertEqual(len(re.findall(r"\B", " ")), 2) + self.assertEqual(len(re.findall(r"\B", " ", re.ASCII)), 2) + self.assertEqual(len(re.findall(br"\B", b" ")), 2) + self.assertEqual(len(re.findall(br"\B", b" ", re.LOCALE)), 2) def test_bigcharset(self): self.assertEqual(re.match("([\u2222\u2223])", From 9d344fafc4385cb2e17425b77b54660ca83c61ac Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 Sep 2024 08:42:58 +0200 Subject: [PATCH 03/39] gh-124127: Make Py_REFCNT() opaque in limited C API 3.14 (#124128) --- Doc/data/stable_abi.dat | 1 + Doc/whatsnew/3.14.rst | 7 ++-- Include/refcount.h | 34 ++++++++++++------- Lib/test/test_stable_abi_ctypes.py | 1 + ...-09-16-16-21-39.gh-issue-124127.LB8DBU.rst | 3 ++ Misc/stable_abi.toml | 2 ++ Objects/object.c | 12 ++++++- PC/python3dll.c | 1 + 8 files changed, 44 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-09-16-16-21-39.gh-issue-124127.LB8DBU.rst diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 7c4b56d1dd652f..19dc71a345b474 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -881,6 +881,7 @@ func,Py_Main,3.2,, func,Py_MakePendingCalls,3.2,, func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, +func,Py_REFCNT,3.14,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, func,Py_SetProgramName,3.2,, diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 04f1a195bca561..5acb9bfe18b2d0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -650,9 +650,10 @@ New Features Porting to Python 3.14 ---------------------- -* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` is now implemented as - an opaque function call to hide implementation details. - (Contributed by Victor Stinner in :gh:`120600`.) +* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide + implementation details. + (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) Deprecated diff --git a/Include/refcount.h b/Include/refcount.h index 1d736b194dcc77..9a4e15065ecab8 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -77,21 +77,29 @@ check by comparing the reference count field to the immortality reference count. #endif // Py_GIL_DISABLED -static inline Py_ssize_t Py_REFCNT(PyObject *ob) { -#if !defined(Py_GIL_DISABLED) - return ob->ob_refcnt; +// Py_REFCNT() implementation for the stable ABI +PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); + +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030e0000 + // Stable ABI implements Py_REFCNT() as a function call + // on limited C API version 3.14 and newer. #else - uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); - if (local == _Py_IMMORTAL_REFCNT_LOCAL) { - return _Py_IMMORTAL_REFCNT; + static inline Py_ssize_t _Py_REFCNT(PyObject *ob) { + #if !defined(Py_GIL_DISABLED) + return ob->ob_refcnt; + #else + uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local); + if (local == _Py_IMMORTAL_REFCNT_LOCAL) { + return _Py_IMMORTAL_REFCNT; + } + Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); + return _Py_STATIC_CAST(Py_ssize_t, local) + + Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); + #endif } - Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&ob->ob_ref_shared); - return _Py_STATIC_CAST(Py_ssize_t, local) + - Py_ARITHMETIC_RIGHT_SHIFT(Py_ssize_t, shared, _Py_REF_SHARED_SHIFT); -#endif -} -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 -# define Py_REFCNT(ob) Py_REFCNT(_PyObject_CAST(ob)) + #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 + # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #endif #endif diff --git a/Lib/test/test_stable_abi_ctypes.py b/Lib/test/test_stable_abi_ctypes.py index 4e509e30bee122..d16ad7ef5d4328 100644 --- a/Lib/test/test_stable_abi_ctypes.py +++ b/Lib/test/test_stable_abi_ctypes.py @@ -899,6 +899,7 @@ def test_windows_feature_macros(self): "Py_MakePendingCalls", "Py_NewInterpreter", "Py_NewRef", + "Py_REFCNT", "Py_ReprEnter", "Py_ReprLeave", "Py_SetPath", diff --git a/Misc/NEWS.d/next/C_API/2024-09-16-16-21-39.gh-issue-124127.LB8DBU.rst b/Misc/NEWS.d/next/C_API/2024-09-16-16-21-39.gh-issue-124127.LB8DBU.rst new file mode 100644 index 00000000000000..883f173f8fbbc4 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-09-16-16-21-39.gh-issue-124127.LB8DBU.rst @@ -0,0 +1,3 @@ +In the limited C API 3.14 and newer, :c:func:`Py_REFCNT` is now implemented +as an opaque function call to hide implementation details. Patch by Victor +Stinner. diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d8a9d1f3335583..fe0a5e44f8fb15 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -2508,6 +2508,8 @@ [function.Py_TYPE] added = '3.14' +[function.Py_REFCNT] + added = '3.14' [function.PyIter_NextItem] added = '3.14' [function.PyLong_FromInt32] diff --git a/Objects/object.c b/Objects/object.c index bc63b8120a5160..8a819dd336e421 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -3039,7 +3039,17 @@ Py_GetConstantBorrowed(unsigned int constant_id) // Py_TYPE() implementation for the stable ABI #undef Py_TYPE -PyTypeObject* Py_TYPE(PyObject *ob) +PyTypeObject* +Py_TYPE(PyObject *ob) { return _Py_TYPE(ob); } + + +// Py_REFCNT() implementation for the stable ABI +#undef Py_REFCNT +Py_ssize_t +Py_REFCNT(PyObject *ob) +{ + return _Py_REFCNT(ob); +} diff --git a/PC/python3dll.c b/PC/python3dll.c index 7bd04cb483b547..6b8208ab90bd95 100755 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -81,6 +81,7 @@ EXPORT_FUNC(Py_Main) EXPORT_FUNC(Py_MakePendingCalls) EXPORT_FUNC(Py_NewInterpreter) EXPORT_FUNC(Py_NewRef) +EXPORT_FUNC(Py_REFCNT) EXPORT_FUNC(Py_ReprEnter) EXPORT_FUNC(Py_ReprLeave) EXPORT_FUNC(Py_SetPath) From faef3fa653f2901cc905f98eae0ddcd8dc334d33 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 24 Sep 2024 09:53:04 +0300 Subject: [PATCH 04/39] gh-124120: Document `Annotated.__origin__` (#124125) Co-authored-by: Brian Schubert Co-authored-by: Alex Waygood --- Doc/library/typing.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 075c58d65ce2b4..f52c593a086c0a 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1458,6 +1458,23 @@ These can be used as types in annotations. They all support subscription using >>> X.__metadata__ ('very', 'important', 'metadata') + * At runtime, if you want to retrieve the original + type wrapped by ``Annotated``, use the :attr:`!__origin__` attribute: + + .. doctest:: + + >>> from typing import Annotated, get_origin + >>> Password = Annotated[str, "secret"] + >>> Password.__origin__ + + + Note that using :func:`get_origin` will return ``Annotated`` itself: + + .. doctest:: + + >>> get_origin(Password) + typing.Annotated + .. seealso:: :pep:`593` - Flexible function and variable annotations @@ -3298,6 +3315,7 @@ Introspection helpers assert get_origin(str) is None assert get_origin(Dict[str, int]) is dict assert get_origin(Union[int, str]) is Union + assert get_origin(Annotated[str, "metadata"]) is Annotated P = ParamSpec('P') assert get_origin(P.args) is P assert get_origin(P.kwargs) is P From 3094cd17b0e5ba69309c54964744c797a70aa11b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 10:23:07 +0300 Subject: [PATCH 05/39] gh-63143: Fix parsing mutually exclusive arguments in argparse (GH-124307) Arguments with the value identical to the default value (e.g. booleans, small integers, empty or 1-character strings) are no longer considered "not present". --- Lib/argparse.py | 5 +- Lib/test/test_argparse.py | 121 +++++++++++++++++- ...4-09-21-23-56-41.gh-issue-63143.YKu-LQ.rst | 3 + 3 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-21-23-56-41.gh-issue-63143.YKu-LQ.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 98d65312417ab8..7f6b31bf028cfb 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1949,9 +1949,8 @@ def take_action(action, argument_strings, option_string=None): argument_values = self._get_values(action, argument_strings) # error if this argument is not allowed with other previously - # seen arguments, assuming that actions that use the default - # value don't really count as "present" - if argument_values is not action.default: + # seen arguments + if action.option_strings or argument_strings: seen_non_default_actions.add(action) for conflict_action in action_conflicts.get(action, []): if conflict_action in seen_non_default_actions: diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index f51a690f7ddf1a..2d8905e6599954 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2879,26 +2879,30 @@ def test_failures_when_not_required(self): parse_args = self.get_parser(required=False).parse_args error = ArgumentParserError for args_string in self.failures: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_failures_when_required(self): parse_args = self.get_parser(required=True).parse_args error = ArgumentParserError for args_string in self.failures + ['']: - self.assertRaises(error, parse_args, args_string.split()) + with self.subTest(args=args_string): + self.assertRaises(error, parse_args, args_string.split()) def test_successes_when_not_required(self): parse_args = self.get_parser(required=False).parse_args successes = self.successes + self.successes_when_not_required for args_string, expected_ns in successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_successes_when_required(self): parse_args = self.get_parser(required=True).parse_args for args_string, expected_ns in self.successes: - actual_ns = parse_args(args_string.split()) - self.assertEqual(actual_ns, expected_ns) + with self.subTest(args=args_string): + actual_ns = parse_args(args_string.split()) + self.assertEqual(actual_ns, expected_ns) def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage @@ -3285,6 +3289,111 @@ def get_parser(self, required): test_successes_when_not_required = None test_successes_when_required = None + +class TestMutuallyExclusiveOptionalOptional(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('--bar', nargs='?') + return parser + + failures = [ + '--foo X --bar Y', + '--foo X --bar', + ] + successes = [ + ('--foo X', NS(foo='X', bar=None)), + ('--bar X', NS(foo=None, bar='X')), + ('--bar', NS(foo=None, bar=None)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=None)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar [BAR]) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar [BAR]] + ''' + help = '''\ + + options: + -h, --help show this help message and exit + --foo FOO + --bar [BAR] + ''' + + +class TestMutuallyExclusiveOptionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('--bar', type=bool, default=True) + return parser + + failures = [ + '--foo X --bar Y', + '--foo X --bar=', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('--bar X', NS(foo=None, bar=True)), + ('--bar=', NS(foo=None, bar=False)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | --bar BAR) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | --bar BAR] + ''' + help = '''\ + + options: + -h, --help show this help message and exit + --foo FOO + --bar BAR + ''' + + +class TestMutuallyExclusivePositionalWithDefault(MEMixin, TestCase): + def get_parser(self, required=None): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=required) + group.add_argument('--foo') + group.add_argument('bar', nargs='?', type=bool, default=True) + return parser + + failures = [ + '--foo X Y', + ] + successes = [ + ('--foo X', NS(foo='X', bar=True)), + ('X', NS(foo=None, bar=True)), + ] + successes_when_not_required = [ + ('', NS(foo=None, bar=True)), + ] + usage_when_required = '''\ + usage: PROG [-h] (--foo FOO | bar) + ''' + usage_when_not_required = '''\ + usage: PROG [-h] [--foo FOO | bar] + ''' + help = '''\ + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + # ================================================= # Mutually exclusive group in parent parser tests # ================================================= diff --git a/Misc/NEWS.d/next/Library/2024-09-21-23-56-41.gh-issue-63143.YKu-LQ.rst b/Misc/NEWS.d/next/Library/2024-09-21-23-56-41.gh-issue-63143.YKu-LQ.rst new file mode 100644 index 00000000000000..cb031fd601a9bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-21-23-56-41.gh-issue-63143.YKu-LQ.rst @@ -0,0 +1,3 @@ +Fix parsing mutually exclusive arguments in :mod:`argparse`. Arguments with +the value identical to the default value (e.g. booleans, small integers, +empty or 1-character strings) are no longer considered "not present". From c578271366176a1d1b0941897efefb6e4d6508b4 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 10:35:28 +0300 Subject: [PATCH 06/39] gh-53780: Ignore the first "--" (double dash) between an option and command in argparse (GH-124275) --- Lib/argparse.py | 12 ++++++++---- Lib/test/test_argparse.py | 14 ++++++++++++++ .../2024-09-20-12-23-11.gh-issue-53780.mrV1zi.rst | 1 + 3 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-20-12-23-11.gh-issue-53780.mrV1zi.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 7f6b31bf028cfb..89496cbe454e09 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2068,11 +2068,15 @@ def consume_positionals(start_index): # and add the Positional and its args to the list for action, arg_count in zip(positionals, arg_counts): args = arg_strings[start_index: start_index + arg_count] - # Strip out the first '--' if it is not in PARSER or REMAINDER arg. - if (action.nargs not in [PARSER, REMAINDER] - and arg_strings_pattern.find('-', start_index, + # Strip out the first '--' if it is not in REMAINDER arg. + if action.nargs == PARSER: + if arg_strings_pattern[start_index] == '-': + assert args[0] == '--' + args.remove('--') + elif action.nargs != REMAINDER: + if (arg_strings_pattern.find('-', start_index, start_index + arg_count) >= 0): - args.remove('--') + args.remove('--') start_index += arg_count if args and action.deprecated and action.dest not in warned: self._warning(_("argument '%(argument_name)s' is deprecated") % diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 2d8905e6599954..3d43fc82d20614 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -5984,6 +5984,20 @@ def test_subparser(self): "invalid choice: '--'", parser.parse_args, ['--', 'x', '--', 'run', 'a', 'b']) + def test_subparser_after_multiple_argument_option(self): + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', nargs='*') + subparsers = parser.add_subparsers() + parser1 = subparsers.add_parser('run') + parser1.add_argument('-f') + parser1.add_argument('bar', nargs='*') + + args = parser.parse_args(['--foo', 'x', 'y', '--', 'run', 'a', 'b', '-f', 'c']) + self.assertEqual(NS(foo=['x', 'y'], f='c', bar=['a', 'b']), args) + self.assertRaisesRegex(argparse.ArgumentError, + "invalid choice: '--'", + parser.parse_args, ['--foo', 'x', '--', '--', 'run', 'a', 'b']) + # =========================== # parse_intermixed_args tests diff --git a/Misc/NEWS.d/next/Library/2024-09-20-12-23-11.gh-issue-53780.mrV1zi.rst b/Misc/NEWS.d/next/Library/2024-09-20-12-23-11.gh-issue-53780.mrV1zi.rst new file mode 100644 index 00000000000000..fb700c722c8a8b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-20-12-23-11.gh-issue-53780.mrV1zi.rst @@ -0,0 +1 @@ +:mod:`argparse` now ignores the first ``"--"`` (double dash) between an option and command. From 3c83f9958c14cd62ad8951c53536f7788745b0ba Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 10:55:33 +0300 Subject: [PATCH 07/39] gh-72795: Make positional arguments with nargs='*' or REMAINDER non-required (GH-124306) This allows to use positional argument with nargs='*' and without default in mutually exclusive group and improves error message about required arguments. --- Lib/argparse.py | 5 ++- Lib/test/test_argparse.py | 31 ++++++++++++++++--- ...4-09-21-22-32-21.gh-issue-72795.naLmkX.rst | 4 +++ 3 files changed, 32 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-21-22-32-21.gh-issue-72795.naLmkX.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 89496cbe454e09..3974ebedabe6f4 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1532,9 +1532,8 @@ def _get_positional_kwargs(self, dest, **kwargs): # mark positional arguments as required if at least one is # always required - if kwargs.get('nargs') not in [OPTIONAL, ZERO_OR_MORE]: - kwargs['required'] = True - if kwargs.get('nargs') == ZERO_OR_MORE and 'default' not in kwargs: + nargs = kwargs.get('nargs') + if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS, 0]: kwargs['required'] = True # return the keyword arguments with no option strings diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 3d43fc82d20614..b04f044f5e2580 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -3079,7 +3079,7 @@ def get_parser(self, required): group = parser.add_mutually_exclusive_group(required=required) group.add_argument('--foo', action='store_true', help='FOO') group.add_argument('--spam', help='SPAM') - group.add_argument('badger', nargs='*', default='X', help='BADGER') + group.add_argument('badger', nargs='*', help='BADGER') return parser failures = [ @@ -3090,13 +3090,13 @@ def get_parser(self, required): '--foo X Y', ] successes = [ - ('--foo', NS(foo=True, spam=None, badger='X')), - ('--spam S', NS(foo=False, spam='S', badger='X')), + ('--foo', NS(foo=True, spam=None, badger=[])), + ('--spam S', NS(foo=False, spam='S', badger=[])), ('X', NS(foo=False, spam=None, badger=['X'])), ('X Y Z', NS(foo=False, spam=None, badger=['X', 'Y', 'Z'])), ] successes_when_not_required = [ - ('', NS(foo=False, spam=None, badger='X')), + ('', NS(foo=False, spam=None, badger=[])), ] usage_when_not_required = '''\ @@ -6369,7 +6369,28 @@ def test_required_args(self): self.parser.add_argument('bar') self.parser.add_argument('baz') self.assertRaisesRegex(argparse.ArgumentError, - 'the following arguments are required: bar, baz', + 'the following arguments are required: bar, baz$', + self.parser.parse_args, []) + + def test_required_args_optional(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='?') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_zero_or_more(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='*') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', + self.parser.parse_args, []) + + def test_required_args_remainder(self): + self.parser.add_argument('bar') + self.parser.add_argument('baz', nargs='...') + self.assertRaisesRegex(argparse.ArgumentError, + 'the following arguments are required: bar$', self.parser.parse_args, []) def test_required_mutually_exclusive_args(self): diff --git a/Misc/NEWS.d/next/Library/2024-09-21-22-32-21.gh-issue-72795.naLmkX.rst b/Misc/NEWS.d/next/Library/2024-09-21-22-32-21.gh-issue-72795.naLmkX.rst new file mode 100644 index 00000000000000..15c0918097367f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-21-22-32-21.gh-issue-72795.naLmkX.rst @@ -0,0 +1,4 @@ +Positional arguments with :ref:`nargs` equal to ``'*'`` or +:data:`!argparse.REMAINDER` are no longer required. This allows to use +positional argument with ``nargs='*'`` and without ``default`` in mutually +exclusive group and improves error message about required arguments. From e2f710792b0418b8ca1ca3b8cdf39588c7268495 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 11:01:37 +0300 Subject: [PATCH 08/39] gh-124188: Fix PyErr_ProgramTextObject() (GH-124189) * Detect source file encoding. * Use the "replace" error handler even for UTF-8 (default) encoding. * Remove the BOM. * Fix detection of too long lines if they contain NUL. * Return the head rather than the tail for truncated long lines. --- Lib/test/support/script_helper.py | 10 +- Lib/test/test_compiler_codegen.py | 5 +- Lib/test/test_eof.py | 166 +++++++++++++---- Lib/test/test_exceptions.py | 171 +++++++++++++----- ...-09-17-22-06-01.gh-issue-124188.aFqNAB.rst | 2 + Python/errors.c | 91 ++++++---- 6 files changed, 328 insertions(+), 117 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-17-22-06-01.gh-issue-124188.aFqNAB.rst diff --git a/Lib/test/support/script_helper.py b/Lib/test/support/script_helper.py index d0be3179b0efa3..46ce950433ddf6 100644 --- a/Lib/test/support/script_helper.py +++ b/Lib/test/support/script_helper.py @@ -234,9 +234,13 @@ def make_script(script_dir, script_basename, source, omit_suffix=False): if not omit_suffix: script_filename += os.extsep + 'py' script_name = os.path.join(script_dir, script_filename) - # The script should be encoded to UTF-8, the default string encoding - with open(script_name, 'w', encoding='utf-8') as script_file: - script_file.write(source) + if isinstance(source, str): + # The script should be encoded to UTF-8, the default string encoding + with open(script_name, 'w', encoding='utf-8') as script_file: + script_file.write(source) + else: + with open(script_name, 'wb') as script_file: + script_file.write(source) importlib.invalidate_caches() return script_name diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index d82fb85ed259ab..8a15c400a449e1 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -152,5 +152,8 @@ def g(): def test_syntax_error__return_not_in_function(self): snippet = "return 42" - with self.assertRaisesRegex(SyntaxError, "'return' outside function"): + with self.assertRaisesRegex(SyntaxError, "'return' outside function") as cm: self.codegen_test(snippet, None) + self.assertIsNone(cm.exception.text) + self.assertEqual(cm.exception.offset, 1) + self.assertEqual(cm.exception.end_offset, 10) diff --git a/Lib/test/test_eof.py b/Lib/test/test_eof.py index be4fd73bfdc36b..e377383450e19d 100644 --- a/Lib/test/test_eof.py +++ b/Lib/test/test_eof.py @@ -1,6 +1,7 @@ """test script for a few new invalid token catches""" import sys +from codecs import BOM_UTF8 from test import support from test.support import os_helper from test.support import script_helper @@ -11,67 +12,158 @@ class EOFTestCase(unittest.TestCase): def test_EOF_single_quote(self): expect = "unterminated string literal (detected at line 1) (, line 1)" for quote in ("'", "\""): - try: + with self.assertRaises(SyntaxError) as cm: eval(f"""{quote}this is a test\ """) - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - self.assertEqual(msg.offset, 1) - else: - raise support.TestFailed + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.offset, 1) def test_EOFS(self): - expect = ("unterminated triple-quoted string literal (detected at line 1) (, line 1)") - try: - eval("""'''this is a test""") - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - self.assertEqual(msg.offset, 1) - else: - raise support.TestFailed + expect = ("unterminated triple-quoted string literal (detected at line 3) (, line 1)") + with self.assertRaises(SyntaxError) as cm: + eval("""ä = '''thîs is \na \ntest""") + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval("""ä = '''thîs is \na \ntest""".encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval(BOM_UTF8 + """ä = '''thîs is \na \ntest""".encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) + + with self.assertRaises(SyntaxError) as cm: + eval("""# coding: latin1\nä = '''thîs is \na \ntest""".encode('latin1')) + self.assertEqual(str(cm.exception), "unterminated triple-quoted string literal (detected at line 4) (, line 2)") + self.assertEqual(cm.exception.text, "ä = '''thîs is ") + self.assertEqual(cm.exception.offset, 5) def test_EOFS_with_file(self): expect = ("(, line 1)") with os_helper.temp_dir() as temp_dir: - file_name = script_helper.make_script(temp_dir, 'foo', """'''this is \na \ntest""") - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unterminated triple-quoted string literal (detected at line 3)', err) + file_name = script_helper.make_script(temp_dir, 'foo', + """ä = '''thîs is \na \ntest""") + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + """ä = '''thîs is \na \ntest""".encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + BOM_UTF8 + """ä = '''thîs is \na \ntest""".encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 3)']) + + file_name = script_helper.make_script(temp_dir, 'foo', + """# coding: latin1\nä = '''thîs is \na \ntest""".encode('latin1')) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + " ä = '''thîs is ", + ' ^', + 'SyntaxError: unterminated triple-quoted string literal (detected at line 4)']) @warnings_helper.ignore_warnings(category=SyntaxWarning) def test_eof_with_line_continuation(self): expect = "unexpected EOF while parsing (, line 1)" - try: + with self.assertRaises(SyntaxError) as cm: compile('"\\Xhh" \\', '', 'exec') - except SyntaxError as msg: - self.assertEqual(str(msg), expect) - else: - raise support.TestFailed + self.assertEqual(str(cm.exception), expect) def test_line_continuation_EOF(self): """A continuation at the end of input must be an error; bpo2180.""" expect = 'unexpected EOF while parsing (, line 1)' - with self.assertRaises(SyntaxError) as excinfo: - exec('x = 5\\') - self.assertEqual(str(excinfo.exception), expect) - with self.assertRaises(SyntaxError) as excinfo: + with self.assertRaises(SyntaxError) as cm: + exec('ä = 5\\') + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec('ä = 5\\'.encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec('# coding:latin1\nä = 5\\'.encode('latin1')) + self.assertEqual(str(cm.exception), + 'unexpected EOF while parsing (, line 2)') + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: + exec(BOM_UTF8 + 'ä = 5\\'.encode()) + self.assertEqual(str(cm.exception), expect) + self.assertEqual(cm.exception.text, 'ä = 5\\\n') + self.assertEqual(cm.exception.offset, 7) + + with self.assertRaises(SyntaxError) as cm: exec('\\') - self.assertEqual(str(excinfo.exception), expect) + self.assertEqual(str(cm.exception), expect) @unittest.skipIf(not sys.executable, "sys.executable required") def test_line_continuation_EOF_from_file_bpo2180(self): """Ensure tok_nextc() does not add too many ending newlines.""" with os_helper.temp_dir() as temp_dir: file_name = script_helper.make_script(temp_dir, 'foo', '\\') - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 1', err) - self.assertIn(b'\\', err) - - file_name = script_helper.make_script(temp_dir, 'foo', 'y = 6\\') - rc, out, err = script_helper.assert_python_failure(file_name) - self.assertIn(b'unexpected EOF while parsing', err) - self.assertIn(b'line 1', err) - self.assertIn(b'y = 6\\', err) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-2:], [ + ' \\', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-3][-8:], ', line 1', err) + + file_name = script_helper.make_script(temp_dir, 'foo', 'ä = 6\\') + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 6\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 1', err) + + file_name = script_helper.make_script(temp_dir, 'foo', + '# coding:latin1\n' + 'ä = 7\\'.encode('latin1')) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 7\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 2', err) + + file_name = script_helper.make_script(temp_dir, 'foo', + BOM_UTF8 + 'ä = 8\\'.encode()) + rc, out, err = script_helper.assert_python_failure('-X', 'utf8', file_name) + err = err.decode().splitlines() + self.assertEqual(err[-3:], [ + ' ä = 8\\', + ' ^', + 'SyntaxError: unexpected EOF while parsing']) + self.assertEqual(err[-4][-8:], ', line 1', err) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index e4f2e3a97b8bb8..ba858c49400911 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -7,6 +7,7 @@ import pickle import weakref import errno +from codecs import BOM_UTF8 from textwrap import dedent from test.support import (captured_stderr, check_impl_detail, @@ -2011,16 +2012,20 @@ def test_copy_pickle(self): self.assertEqual(exc.path, orig.path) +def run_script(source): + if isinstance(source, str): + with open(TESTFN, 'w', encoding='utf-8') as testfile: + testfile.write(dedent(source)) + else: + with open(TESTFN, 'wb') as testfile: + testfile.write(source) + _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) + return err.decode('utf-8').splitlines() + class AssertionErrorTests(unittest.TestCase): def tearDown(self): unlink(TESTFN) - def write_source(self, source): - with open(TESTFN, 'w') as testfile: - testfile.write(dedent(source)) - _rc, _out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - return err.decode('utf-8').splitlines() - @force_not_colorized def test_assertion_error_location(self): cases = [ @@ -2052,11 +2057,32 @@ def test_assertion_error_location(self): 'AssertionError', ], ), - ('assert 1 > 2, "message"', + ('assert 1 > 2, "messäge"', + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', + ], + ), + ('assert 1 > 2, "messäge"'.encode(), [ - ' assert 1 > 2, "message"', + ' assert 1 > 2, "messäge"', ' ^^^^^', - 'AssertionError: message', + 'AssertionError: messäge', + ], + ), + ('# coding: latin1\nassert 1 > 2, "messäge"'.encode('latin1'), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', + ], + ), + (BOM_UTF8 + 'assert 1 > 2, "messäge"'.encode(), + [ + ' assert 1 > 2, "messäge"', + ' ^^^^^', + 'AssertionError: messäge', ], ), @@ -2094,8 +2120,8 @@ def test_assertion_error_location(self): ), ] for source, expected in cases: - with self.subTest(source): - result = self.write_source(source) + with self.subTest(source=source): + result = run_script(source) self.assertEqual(result[-3:], expected) @force_not_colorized @@ -2125,12 +2151,14 @@ def test_multiline_not_highlighted(self): ), ] for source, expected in cases: - with self.subTest(source): - result = self.write_source(source) + with self.subTest(source=source): + result = run_script(source) self.assertEqual(result[-len(expected):], expected) class SyntaxErrorTests(unittest.TestCase): + maxDiff = None + @force_not_colorized def test_range_of_offsets(self): cases = [ @@ -2223,45 +2251,106 @@ def test_range_of_offsets(self): the_exception = exc def test_encodings(self): + self.addCleanup(unlink, TESTFN) source = ( '# -*- coding: cp437 -*-\n' '"┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))\n' ) - try: - with open(TESTFN, 'w', encoding='cp437') as testfile: - testfile.write(source) - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() - - self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') - self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') - finally: - unlink(TESTFN) + err = run_script(source.encode('cp437')) + self.assertEqual(err[-3], ' "┬ó┬ó┬ó┬ó┬ó┬ó" + f(4, x for x in range(1))') + self.assertEqual(err[-2], ' ^^^^^^^^^^^^^^^^^^^') # Check backwards tokenizer errors source = '# -*- coding: ascii -*-\n\n(\n' - try: - with open(TESTFN, 'w', encoding='ascii') as testfile: - testfile.write(source) - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() - - self.assertEqual(err[-3], ' (') - self.assertEqual(err[-2], ' ^') - finally: - unlink(TESTFN) + err = run_script(source) + self.assertEqual(err[-3], ' (') + self.assertEqual(err[-2], ' ^') def test_non_utf8(self): # Check non utf-8 characters - try: - with open(TESTFN, 'bw') as testfile: - testfile.write(b"\x89") - rc, out, err = script_helper.assert_python_failure('-Wd', '-X', 'utf8', TESTFN) - err = err.decode('utf-8').splitlines() + self.addCleanup(unlink, TESTFN) + err = run_script(b"\x89") + self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - self.assertIn("SyntaxError: Non-UTF-8 code starting with '\\x89' in file", err[-1]) - finally: - unlink(TESTFN) + def test_string_source(self): + def try_compile(source): + with self.assertRaises(SyntaxError) as cm: + compile(source, '', 'exec') + return cm.exception + + exc = try_compile('return "ä"') + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä"'.encode()) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile(BOM_UTF8 + 'return "ä"'.encode()) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('# coding: latin1\nreturn "ä"'.encode('latin1')) + self.assertEqual(str(exc), "'return' outside function (, line 2)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä" #' + 'ä'*1000) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + exc = try_compile('return "ä" # ' + 'ä'*1000) + self.assertEqual(str(exc), "'return' outside function (, line 1)") + self.assertIsNone(exc.text) + self.assertEqual(exc.offset, 1) + self.assertEqual(exc.end_offset, 12) + + def test_file_source(self): + self.addCleanup(unlink, TESTFN) + err = run_script('return "ä"') + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('return "ä"'.encode()) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script(BOM_UTF8 + 'return "ä"'.encode()) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('# coding: latin1\nreturn "ä"'.encode('latin1')) + self.assertEqual(err[-3:], [ + ' return "ä"', + ' ^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + + err = run_script('return "ä" #' + 'ä'*1000) + self.assertEqual(err[-2:], [ + ' ^^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + self.assertEqual(err[-3][:100], ' return "ä" #' + 'ä'*84) + + err = run_script('return "ä" # ' + 'ä'*1000) + self.assertEqual(err[-2:], [ + ' ^^^^^^^^^^^', + "SyntaxError: 'return' outside function"]) + self.assertEqual(err[-3][:100], ' return "ä" # ' + 'ä'*83) def test_attributes_new_constructor(self): args = ("bad.py", 1, 2, "abcdefg", 1, 100) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-17-22-06-01.gh-issue-124188.aFqNAB.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-17-22-06-01.gh-issue-124188.aFqNAB.rst new file mode 100644 index 00000000000000..0c2935fbe000bc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-17-22-06-01.gh-issue-124188.aFqNAB.rst @@ -0,0 +1,2 @@ +Fix reading and decoding a line from the source file witn non-UTF-8 encoding +for syntax errors raised in the compiler. diff --git a/Python/errors.c b/Python/errors.c index 29249ac41c6346..9e2a3ce062a6fe 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1903,44 +1903,44 @@ _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_ functionality in tb_displayline() in traceback.c. */ static PyObject * -err_programtext(PyThreadState *tstate, FILE *fp, int lineno, const char* encoding) +err_programtext(FILE *fp, int lineno, const char* encoding) { - int i; char linebuf[1000]; - if (fp == NULL) { - return NULL; + size_t line_size = 0; + + for (int i = 0; i < lineno; ) { + line_size = 0; + if (_Py_UniversalNewlineFgetsWithSize(linebuf, sizeof(linebuf), + fp, NULL, &line_size) == NULL) + { + /* Error or EOF. */ + return NULL; + } + /* fgets read *something*; if it didn't fill the + whole buffer, it must have found a newline + or hit the end of the file; if the last character is \n, + it obviously found a newline; else we haven't + yet seen a newline, so must continue */ + if (i + 1 < lineno + && line_size == sizeof(linebuf) - 1 + && linebuf[sizeof(linebuf) - 2] != '\n') + { + continue; + } + i++; } - for (i = 0; i < lineno; i++) { - char *pLastChar = &linebuf[sizeof(linebuf) - 2]; - do { - *pLastChar = '\0'; - if (Py_UniversalNewlineFgets(linebuf, sizeof linebuf, - fp, NULL) == NULL) { - goto after_loop; - } - /* fgets read *something*; if it didn't get as - far as pLastChar, it must have found a newline - or hit the end of the file; if pLastChar is \n, - it obviously found a newline; else we haven't - yet seen a newline, so must continue */ - } while (*pLastChar != '\0' && *pLastChar != '\n'); + const char *line = linebuf; + /* Skip BOM. */ + if (lineno == 1 && line_size >= 3 && memcmp(line, "\xef\xbb\xbf", 3) == 0) { + line += 3; + line_size -= 3; } - -after_loop: - fclose(fp); - if (i == lineno) { - PyObject *res; - if (encoding != NULL) { - res = PyUnicode_Decode(linebuf, strlen(linebuf), encoding, "replace"); - } else { - res = PyUnicode_FromString(linebuf); - } - if (res == NULL) - _PyErr_Clear(tstate); - return res; + PyObject *res = PyUnicode_Decode(line, line_size, encoding, "replace"); + if (res == NULL) { + PyErr_Clear(); } - return NULL; + return res; } PyObject * @@ -1960,20 +1960,41 @@ PyErr_ProgramText(const char *filename, int lineno) return res; } +/* Function from Parser/tokenizer/file_tokenizer.c */ +extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *); + PyObject * _PyErr_ProgramDecodedTextObject(PyObject *filename, int lineno, const char* encoding) { + char *found_encoding = NULL; if (filename == NULL || lineno <= 0) { return NULL; } - PyThreadState *tstate = _PyThreadState_GET(); FILE *fp = _Py_fopen_obj(filename, "r" PY_STDIOTEXTMODE); if (fp == NULL) { - _PyErr_Clear(tstate); + PyErr_Clear(); return NULL; } - return err_programtext(tstate, fp, lineno, encoding); + if (encoding == NULL) { + int fd = fileno(fp); + found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename); + encoding = found_encoding; + if (encoding == NULL) { + PyErr_Clear(); + encoding = "utf-8"; + } + /* Reset position */ + if (lseek(fd, 0, SEEK_SET) == (off_t)-1) { + fclose(fp); + PyMem_Free(found_encoding); + return NULL; + } + } + PyObject *res = err_programtext(fp, lineno, encoding); + fclose(fp); + PyMem_Free(found_encoding); + return res; } PyObject * From e670a113b5e1fcc3ce3bb9b5b4b1f126264ae21b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 11:26:25 +0300 Subject: [PATCH 09/39] gh-123978: Remove broken time.thread_time() on NetBSD (GH-124116) --- ...4-09-16-12-31-48.gh-issue-123978.z3smEu.rst | 1 + Modules/timemodule.c | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst diff --git a/Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst b/Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst new file mode 100644 index 00000000000000..e5b3229122b509 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-16-12-31-48.gh-issue-123978.z3smEu.rst @@ -0,0 +1 @@ +Remove broken :func:`time.thread_time` and :func:`time.thread_time_ns` on NetBSD. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 46f85bc9c30f9c..ee59fb73ac1e31 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1288,9 +1288,14 @@ py_process_time(time_module_state *state, PyTime_t *tp, /* clock_gettime */ // gh-115714: Don't use CLOCK_PROCESS_CPUTIME_ID on WASI. +/* CLOCK_PROF is defined on NetBSD, but not supported. + * CLOCK_PROCESS_CPUTIME_ID is broken on NetBSD for the same reason as + * CLOCK_THREAD_CPUTIME_ID (see comment below). + */ #if defined(HAVE_CLOCK_GETTIME) \ && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) \ - && !defined(__wasi__) + && !defined(__wasi__) \ + && !defined(__NetBSD__) struct timespec ts; if (HAVE_CLOCK_GETTIME_RUNTIME) { @@ -1483,9 +1488,16 @@ _PyTime_GetThreadTimeWithInfo(PyTime_t *tp, _Py_clock_info_t *info) return 0; } +/* CLOCK_THREAD_CPUTIME_ID is broken on NetBSD: the result of clock_gettime() + * includes the sleeping time, that defeats the purpose of the clock. + * Also, clock_getres() does not support it. + * https://github.com/python/cpython/issues/123978 + * https://gnats.netbsd.org/57512 + */ #elif defined(HAVE_CLOCK_GETTIME) && \ - defined(CLOCK_PROCESS_CPUTIME_ID) && \ - !defined(__EMSCRIPTEN__) && !defined(__wasi__) + defined(CLOCK_THREAD_CPUTIME_ID) && \ + !defined(__EMSCRIPTEN__) && !defined(__wasi__) && \ + !defined(__NetBSD__) #define HAVE_THREAD_TIME #if defined(__APPLE__) && _Py__has_attribute(availability) From e69ff34e81eceb69de6623205c87e0145f1831e4 Mon Sep 17 00:00:00 2001 From: Stanley <46876382+slateny@users.noreply.github.com> Date: Tue, 24 Sep 2024 03:57:37 -0700 Subject: [PATCH 10/39] gh-65169: Clarify prog default in argparse (GH-31602) --- Doc/library/argparse.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index c4e2dbcac2156c..f22aeb8c13c605 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -249,11 +249,12 @@ The following sections describe how each of these are used. prog ^^^^ -By default, :class:`ArgumentParser` objects use ``sys.argv[0]`` to determine +By default, :class:`ArgumentParser` objects use the base name +(see :func:`os.path.basename`) of ``sys.argv[0]`` to determine how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: +always desirable because it will make the help messages match the name that was +used to invoke the program on the command line. For example, consider a file +named ``myprogram.py`` with the following code:: import argparse parser = argparse.ArgumentParser() From 4a5e4aade420c594c5b3fe0589e9e6b444bd6ee5 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 24 Sep 2024 15:46:41 +0300 Subject: [PATCH 11/39] gh-59317: Improve parsing optional positional arguments in argparse (GH-124303) Fix parsing positional argument with nargs equal to '?' or '*' if it is preceded by an option and another positional argument. --- Lib/argparse.py | 13 +- Lib/test/test_argparse.py | 129 +++++++++++++----- ...4-09-21-19-02-37.gh-issue-59317.OAhNZZ.rst | 2 + 3 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-21-19-02-37.gh-issue-59317.OAhNZZ.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 3974ebedabe6f4..66192fb92832a2 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -2227,18 +2227,19 @@ def _match_argument(self, action, arg_strings_pattern): def _match_arguments_partial(self, actions, arg_strings_pattern): # progressively shorten the actions list by slicing off the # final actions until we find a match - result = [] for i in range(len(actions), 0, -1): actions_slice = actions[:i] pattern = ''.join([self._get_nargs_pattern(action) for action in actions_slice]) match = _re.match(pattern, arg_strings_pattern) if match is not None: - result.extend([len(string) for string in match.groups()]) - break - - # return the list of arg string counts - return result + result = [len(string) for string in match.groups()] + if (match.end() < len(arg_strings_pattern) + and arg_strings_pattern[match.end()] == 'O'): + while result and not result[-1]: + del result[-1] + return result + return [] def _parse_optional(self, arg_string): # if it's an empty string, it was meant to be a positional diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index b04f044f5e2580..c6b76986940977 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -280,16 +280,18 @@ def test_failures(self, tester): parser = self._get_parser(tester) for args_str in tester.failures: args = args_str.split() - with tester.assertRaises(ArgumentParserError, msg=args): - parser.parse_args(args) + with tester.subTest(args=args): + with tester.assertRaises(ArgumentParserError, msg=args): + parser.parse_args(args) def test_successes(self, tester): parser = self._get_parser(tester) for args, expected_ns in tester.successes: if isinstance(args, str): args = args.split() - result_ns = self._parse_args(parser, args) - tester.assertEqual(expected_ns, result_ns) + with tester.subTest(args=args): + result_ns = self._parse_args(parser, args) + tester.assertEqual(expected_ns, result_ns) # add tests for each combination of an optionals adding method # and an arg parsing method @@ -1089,57 +1091,87 @@ class TestPositionalsNargs2None(ParserTestCase): class TestPositionalsNargsNoneZeroOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with unlimited""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='*')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='*')] + failures = ['', '--foo', 'a b -x X c'] successes = [ - ('a', NS(foo='a', bar=[])), - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a', NS(x=None, foo='a', bar=[])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a', NS(x='X', foo='a', bar=[])), + ('a -x X', NS(x='X', foo='a', bar=[])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOneOrMore(ParserTestCase): """Test a Positional with no nargs followed by one with one or more""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='+')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='+')] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=['b'])), - ('a b c', NS(foo='a', bar=['b', 'c'])), + ('a b', NS(x=None, foo='a', bar=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b', 'c'])), + ('-x X a b', NS(x='X', foo='a', bar=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b', 'c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b', 'c'])), ] class TestPositionalsNargsNoneOptional(ParserTestCase): """Test a Positional with no nargs followed by one with an Optional""" - argument_signatures = [Sig('foo'), Sig('bar', nargs='?')] + argument_signatures = [Sig('-x'), Sig('foo'), Sig('bar', nargs='?')] failures = ['', '--foo', 'a b c'] successes = [ - ('a', NS(foo='a', bar=None)), - ('a b', NS(foo='a', bar='b')), + ('a', NS(x=None, foo='a', bar=None)), + ('a b', NS(x=None, foo='a', bar='b')), + ('-x X a', NS(x='X', foo='a', bar=None)), + ('a -x X', NS(x='X', foo='a', bar=None)), + ('-x X a b', NS(x='X', foo='a', bar='b')), + ('a -x X b', NS(x='X', foo='a', bar='b')), + ('a b -x X', NS(x='X', foo='a', bar='b')), ] class TestPositionalsNargsZeroOrMoreNone(ParserTestCase): """Test a Positional with unlimited nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='*'), Sig('bar')] - failures = ['', '--foo'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='*'), Sig('bar')] + failures = ['', '--foo', 'a -x X b', 'a -x X b c', 'a b -x X c'] successes = [ - ('a', NS(foo=[], bar='a')), - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a', NS(x=None, foo=[], bar='a')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a', NS(x='X', foo=[], bar='a')), + ('a -x X', NS(x='X', foo=[], bar='a')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] class TestPositionalsNargsOneOrMoreNone(ParserTestCase): """Test a Positional with one or more nargs followed by one with none""" - argument_signatures = [Sig('foo', nargs='+'), Sig('bar')] - failures = ['', '--foo', 'a'] + argument_signatures = [Sig('-x'), Sig('foo', nargs='+'), Sig('bar')] + failures = ['', '--foo', 'a', 'a -x X b c', 'a b -x X c'] successes = [ - ('a b', NS(foo=['a'], bar='b')), - ('a b c', NS(foo=['a', 'b'], bar='c')), + ('a b', NS(x=None, foo=['a'], bar='b')), + ('a b c', NS(x=None, foo=['a', 'b'], bar='c')), + ('-x X a b', NS(x='X', foo=['a'], bar='b')), + ('a -x X b', NS(x='X', foo=['a'], bar='b')), + ('a b -x X', NS(x='X', foo=['a'], bar='b')), + ('-x X a b c', NS(x='X', foo=['a', 'b'], bar='c')), + ('a b c -x X', NS(x='X', foo=['a', 'b'], bar='c')), ] @@ -1224,14 +1256,21 @@ class TestPositionalsNargsNoneZeroOrMore1(ParserTestCase): """Test three Positionals: no nargs, unlimited nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='*'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=[], baz=['b'])), - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), + ('a b', NS(x=None, foo='a', bar=[], baz=['b'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=[], baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=[], baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), ] @@ -1239,14 +1278,22 @@ class TestPositionalsNargsNoneOneOrMore1(ParserTestCase): """Test three Positionals: no nargs, one or more nargs and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='+'), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a', 'b'] + failures = ['', '--foo', 'a', 'b', 'a b -x X c d', 'a b c -x X d'] successes = [ - ('a b c', NS(foo='a', bar=['b'], baz=['c'])), - ('a b c d', NS(foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c', NS(x=None, foo='a', bar=['b'], baz=['c'])), + ('a b c d', NS(x=None, foo='a', bar=['b', 'c'], baz=['d'])), + ('-x X a b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b -x X c', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar=['b'], baz=['c'])), + ('-x X a b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a -x X b c d', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), + ('a b c d -x X', NS(x='X', foo='a', bar=['b', 'c'], baz=['d'])), ] @@ -1254,14 +1301,21 @@ class TestPositionalsNargsNoneOptional1(ParserTestCase): """Test three Positionals: no nargs, optional narg and 1 nargs""" argument_signatures = [ + Sig('-x'), Sig('foo'), Sig('bar', nargs='?', default=0.625), Sig('baz', nargs=1), ] - failures = ['', '--foo', 'a'] + failures = ['', '--foo', 'a', 'a b -x X c'] successes = [ - ('a b', NS(foo='a', bar=0.625, baz=['b'])), - ('a b c', NS(foo='a', bar='b', baz=['c'])), + ('a b', NS(x=None, foo='a', bar=0.625, baz=['b'])), + ('a b c', NS(x=None, foo='a', bar='b', baz=['c'])), + ('-x X a b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a -x X b', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('a b -x X', NS(x='X', foo='a', bar=0.625, baz=['b'])), + ('-x X a b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a -x X b c', NS(x='X', foo='a', bar='b', baz=['c'])), + ('a b c -x X', NS(x='X', foo='a', bar='b', baz=['c'])), ] @@ -1477,6 +1531,9 @@ class TestNargsRemainder(ParserTestCase): successes = [ ('X', NS(x='X', y=[], z=None)), ('-z Z X', NS(x='X', y=[], z='Z')), + ('-z Z X A B', NS(x='X', y=['A', 'B'], z='Z')), + ('X -z Z A B', NS(x='X', y=['-z', 'Z', 'A', 'B'], z=None)), + ('X A -z Z B', NS(x='X', y=['A', '-z', 'Z', 'B'], z=None)), ('X A B -z Z', NS(x='X', y=['A', 'B', '-z', 'Z'], z=None)), ('X Y --foo', NS(x='X', y=['Y', '--foo'], z=None)), ] @@ -6018,8 +6075,8 @@ def test_basic(self): args, extras = parser.parse_known_args(argv) # cannot parse the '1,2,3' - self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[]), args) - self.assertEqual(["1", "2", "3"], extras) + self.assertEqual(NS(bar='y', cmd='cmd', foo='x', rest=[1]), args) + self.assertEqual(["2", "3"], extras) argv = 'cmd --foo x 1 --error 2 --bar y 3'.split() args, extras = parser.parse_known_intermixed_args(argv) diff --git a/Misc/NEWS.d/next/Library/2024-09-21-19-02-37.gh-issue-59317.OAhNZZ.rst b/Misc/NEWS.d/next/Library/2024-09-21-19-02-37.gh-issue-59317.OAhNZZ.rst new file mode 100644 index 00000000000000..0b1df9e3b7dea8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-21-19-02-37.gh-issue-59317.OAhNZZ.rst @@ -0,0 +1,2 @@ +Fix parsing positional argument with :ref:`nargs` equal to ``'?'`` or ``'*'`` +if it is preceded by an option and another positional argument. From d21b0b5d36834d4d35aec3a01661597019594936 Mon Sep 17 00:00:00 2001 From: Payton <72841140+paytonward6@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:14:35 -0500 Subject: [PATCH 12/39] gh-113008: Correct argparse usage output for required, mutually exclusive groups (GH-113085) --- Lib/argparse.py | 4 ++-- Lib/test/test_argparse.py | 23 +++++++++++++++++++ ...-12-14-13-43-27.gh-issue-113008.jWYn8T.rst | 1 + 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2023-12-14-13-43-27.gh-issue-113008.jWYn8T.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 66192fb92832a2..694c46db61d177 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -395,12 +395,12 @@ def _get_actions_usage_parts(self, actions, groups): continue try: - start = actions.index(group._group_actions[0]) + start = min(actions.index(item) for item in group._group_actions) except ValueError: continue else: end = start + len(group._group_actions) - if actions[start:end] == group._group_actions: + if set(actions[start:end]) == set(group._group_actions): group_actions.update(group._group_actions) inserts[start, end] = group diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index c6b76986940977..ef05a6fefcffcc 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2902,6 +2902,29 @@ def test_help(self): ''' self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_optional_order(self): + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--foo') + group.add_argument('bar', nargs='?') + expected = '''\ + usage: PROG [-h] (--foo FOO | bar) + + positional arguments: + bar + + options: + -h, --help show this help message and exit + --foo FOO + ''' + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + + parser = ErrorRaisingArgumentParser(prog='PROG') + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('bar', nargs='?') + group.add_argument('--foo') + self.assertEqual(parser.format_help(), textwrap.dedent(expected)) + def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): self.maxDiff = None parser = ErrorRaisingArgumentParser(prog='PROG') diff --git a/Misc/NEWS.d/next/Library/2023-12-14-13-43-27.gh-issue-113008.jWYn8T.rst b/Misc/NEWS.d/next/Library/2023-12-14-13-43-27.gh-issue-113008.jWYn8T.rst new file mode 100644 index 00000000000000..0f2a44299717c0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-14-13-43-27.gh-issue-113008.jWYn8T.rst @@ -0,0 +1 @@ +Correct argparse usage output for required, mutually exclusive groups containing a positional argument From 38a5beb12a64461a06b84fab285944ecb66de04c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 Sep 2024 16:33:27 +0200 Subject: [PATCH 13/39] gh-124402: Require cpu resource in test_free_threading (#124438) Require the 'cpu' test resource on slow test_free_threading tests. --- Lib/test/test_free_threading/test_list.py | 3 +++ Lib/test/test_free_threading/test_monitoring.py | 3 +++ Lib/test/test_free_threading/test_type.py | 2 ++ 3 files changed, 8 insertions(+) diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py index 6ad806d67a80ed..c6b58fcd86f449 100644 --- a/Lib/test/test_free_threading/test_list.py +++ b/Lib/test/test_free_threading/test_list.py @@ -3,6 +3,7 @@ from threading import Thread from unittest import TestCase +from test import support from test.support import threading_helper @@ -13,6 +14,7 @@ def __init__(self, v): @threading_helper.requires_working_threading() class TestList(TestCase): + @support.requires_resource('cpu') def test_racing_iter_append(self): l = [] @@ -42,6 +44,7 @@ def reader_func(): for reader in readers: reader.join() + @support.requires_resource('cpu') def test_racing_iter_extend(self): iters = [ lambda x: [x], diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index 78303f4206decc..be582455d118ac 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -7,6 +7,7 @@ import weakref from sys import monitoring +from test import support from test.support import threading_helper from threading import Thread, _PyRLock from unittest import TestCase @@ -43,6 +44,7 @@ def after_test(self): """Runs once after the test is done""" pass + @support.requires_resource('cpu') def test_instrumentation(self): # Setup a bunch of functions which will need instrumentation... funcs = [] @@ -218,6 +220,7 @@ def test_register_callback(self): for ref in self.refs: self.assertEqual(ref(), None) + @support.requires_resource('cpu') def test_set_local_trace_opcodes(self): def trace(frame, event, arg): frame.f_trace_opcodes = True diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 649676db9c08a5..977bfd2c7fd2f7 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -5,6 +5,7 @@ from threading import Thread from unittest import TestCase +from test import support from test.support import threading_helper @@ -96,6 +97,7 @@ def reader_func(): self.run_one(writer_func, reader_func) + @support.requires_resource('cpu') def test___class___modification(self): class Foo: pass From 20ccda000b5f8365d5f864fd07876804157c2378 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 24 Sep 2024 09:30:01 -0700 Subject: [PATCH 14/39] GH-87358: Add clarification about nargs and default argparse behaviour (#124094) --- Doc/library/argparse.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index f22aeb8c13c605..53ecc97d5659f4 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -1123,6 +1123,9 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +For required_ arguments, the ``default`` value is ignored. For example, this +applies to positional arguments with nargs_ values other than ``?`` or ``*``, +or optional arguments marked as ``required=True``. Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the command-line argument was not present:: From b4d0d7de0f6d938128bf525e119c18af5632b804 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 24 Sep 2024 10:03:16 -0700 Subject: [PATCH 15/39] GH-124398: Pin LLVM to 18.1.0 for Windows JIT CI (GH-124399) --- .github/workflows/jit.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 5e3ac9e9e0fada..8f56220feea90b 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -110,8 +110,7 @@ jobs: - name: Native Windows if: runner.os == 'Windows' && matrix.architecture != 'ARM64' run: | - choco upgrade llvm -y - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }} ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 @@ -119,8 +118,7 @@ jobs: - name: Emulated Windows if: runner.os == 'Windows' && matrix.architecture == 'ARM64' run: | - choco upgrade llvm -y - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }} + choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} - name: Native macOS From 17b3bc9cc7608920077f5f08f970c7157a2d2e72 Mon Sep 17 00:00:00 2001 From: sobolevn Date: Tue, 24 Sep 2024 21:06:39 +0300 Subject: [PATCH 16/39] gh-124405: Fix `NameError` in `openpty` (#124406) --- Lib/pty.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/pty.py b/Lib/pty.py index eb3d5f1ff657bb..4b25ac32c8da14 100644 --- a/Lib/pty.py +++ b/Lib/pty.py @@ -39,8 +39,8 @@ def openpty(): except ImportError: return master_fd, slave_fd try: - ioctl(result, I_PUSH, "ptem") - ioctl(result, I_PUSH, "ldterm") + ioctl(slave_fd, I_PUSH, "ptem") + ioctl(slave_fd, I_PUSH, "ldterm") except OSError: pass return master_fd, slave_fd From fc9e6bf53d1c9ce2b5f802864e0da265a77c111f Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Tue, 24 Sep 2024 12:49:58 -0700 Subject: [PATCH 17/39] gh-124448: Update bundled Tcl/Tk in macOS installer to 8.6.15. (#124453) --- Mac/BuildScript/build-installer.py | 6 +++--- .../macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b97738836d92cc..f5f0ed44884142 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.14' - tcl_checksum='5880225babf7954c58d4fb0f5cf6279104ce1cd6aa9b71e9a6322540e1c4de66' + tcl_tk_ver='8.6.15' + tcl_checksum='861e159753f2e2fbd6ec1484103715b0be56be3357522b858d3cbb5f893ffef1' - tk_checksum='8ffdb720f47a6ca6107eac2dd877e30b0ef7fac14f3a84ebbd0b3612cee41a94' + tk_checksum='550969f35379f952b3020f3ab7b9dd5bfd11c1ef7c9b7c6a75f5c49aca793fec' tk_patches = [] diff --git a/Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst b/Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst new file mode 100644 index 00000000000000..6d57aa1ee190d6 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2024-09-24-10-48-46.gh-issue-124448.bFMrS6.rst @@ -0,0 +1 @@ +Update bundled Tcl/Tk in macOS installer to 8.6.15. From af8403a58dbe45130400a133f756cbf53c5f1d7e Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Tue, 24 Sep 2024 12:52:15 -0700 Subject: [PATCH 18/39] gh-120254: Add a `commands` argument to `pdb.set_trace` (#120255) --- Doc/library/pdb.rst | 8 +++++++- Lib/doctest.py | 4 ++-- Lib/pdb.py | 13 +++++++++---- Lib/test/test_pdb.py | 11 +++++++++++ .../2024-06-08-03-29-01.gh-issue-120254.h682ke.rst | 1 + 5 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index d696161876e99d..ce7516a52b1d74 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -159,12 +159,15 @@ slightly different way: is entered. -.. function:: set_trace(*, header=None) +.. function:: set_trace(*, header=None, commands=None) Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console just before debugging begins. + The *commands* argument, if given, is a list of commands to execute + when the debugger starts. + .. versionchanged:: 3.7 The keyword-only argument *header*. @@ -173,6 +176,9 @@ slightly different way: :func:`set_trace` will enter the debugger immediately, rather than on the next line of code to be executed. + .. versionadded:: 3.14 + The *commands* argument. + .. function:: post_mortem(traceback=None) Enter post-mortem debugging of the given *traceback* object. If no diff --git a/Lib/doctest.py b/Lib/doctest.py index ea7d275c91db04..bb281fc483c41c 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -389,11 +389,11 @@ def __init__(self, out): # still use input() to get user input self.use_rawinput = 1 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): self.__debugger_used = True if frame is None: frame = sys._getframe().f_back - pdb.Pdb.set_trace(self, frame) + pdb.Pdb.set_trace(self, frame, commands=commands) def set_continue(self): # Calling set_continue unconditionally would break unit test diff --git a/Lib/pdb.py b/Lib/pdb.py index 228de489a9cef1..dd21207a627bee 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -361,10 +361,14 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self._chained_exceptions = tuple() self._chained_exception_index = 0 - def set_trace(self, frame=None): + def set_trace(self, frame=None, *, commands=None): Pdb._last_pdb_instance = self if frame is None: frame = sys._getframe().f_back + + if commands is not None: + self.rcLines.extend(commands) + super().set_trace(frame) def sigint_handler(self, signum, frame): @@ -2350,13 +2354,14 @@ def runcall(*args, **kwds): """ return Pdb().runcall(*args, **kwds) -def set_trace(*, header=None): +def set_trace(*, header=None, commands=None): """Enter the debugger at the calling stack frame. This is useful to hard-code a breakpoint at a given point in a program, even if the code is not otherwise being debugged (e.g. when an assertion fails). If given, *header* is printed to the console - just before debugging begins. + just before debugging begins. *commands* is an optional list of + pdb commands to run when the debugger starts. """ if Pdb._last_pdb_instance is not None: pdb = Pdb._last_pdb_instance @@ -2364,7 +2369,7 @@ def set_trace(*, header=None): pdb = Pdb() if header is not None: pdb.message(header) - pdb.set_trace(sys._getframe().f_back) + pdb.set_trace(sys._getframe().f_back, commands=commands) # Post-Mortem interface diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index db7d1b1e9cd935..3173b0553c232f 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -901,6 +901,17 @@ def test_pdb_where_command(): (Pdb) continue """ +def test_pdb_commands_with_set_trace(): + """Test that commands can be passed to Pdb.set_trace() + + >>> def test_function(): + ... x = 1 + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace(commands=['p x', 'c']) + + >>> test_function() + 1 + """ + # skip this test if sys.flags.no_site = True; # exit() isn't defined unless there's a site module. diff --git a/Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst b/Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst new file mode 100644 index 00000000000000..33ef1c91591c54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-03-29-01.gh-issue-120254.h682ke.rst @@ -0,0 +1 @@ +Added ``commands`` argument to :func:`pdb.set_trace` which allows users to send debugger commands from the source file. From d3c76dff444046504754a437dceebc9a9c87ef18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Detrey?= Date: Tue, 24 Sep 2024 21:54:50 +0200 Subject: [PATCH 19/39] bpo-44864: Do not translate user-provided strings in ArgumentParser.add_subparsers() (GH-27667) Call _() on literal strings only. --- Lib/argparse.py | 4 ++-- .../next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst diff --git a/Lib/argparse.py b/Lib/argparse.py index 694c46db61d177..690b2a9db9481b 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -1804,8 +1804,8 @@ def add_subparsers(self, **kwargs): kwargs.setdefault('parser_class', type(self)) if 'title' in kwargs or 'description' in kwargs: - title = _(kwargs.pop('title', 'subcommands')) - description = _(kwargs.pop('description', None)) + title = kwargs.pop('title', _('subcommands')) + description = kwargs.pop('description', None) self._subparsers = self.add_argument_group(title, description) else: self._subparsers = self._positionals diff --git a/Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst b/Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst new file mode 100644 index 00000000000000..9610fa90ef0a98 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-08-24-19-37-46.bpo-44864.KzxaDh.rst @@ -0,0 +1 @@ +Do not translate user-provided strings in :class:`argparse.ArgumentParser`. From f4997bb3ac961d6aaf07ce650cd074e28ce6ccd0 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 24 Sep 2024 13:08:18 -0700 Subject: [PATCH 20/39] gh-123923: Defer refcounting for `f_funcobj` in `_PyInterpreterFrame` (#124026) Use a `_PyStackRef` and defer the reference to `f_funcobj` when possible. This avoids some reference count contention in the common case of executing the same code object from multiple threads concurrently in the free-threaded build. --- Include/internal/pycore_frame.h | 26 +++++++---- Include/internal/pycore_gc.h | 11 +++++ Modules/_testinternalcapi.c | 6 +-- Objects/frameobject.c | 9 ++-- Objects/genobject.c | 5 +-- Objects/typevarobject.c | 4 +- Python/bytecodes.c | 45 +++++++++---------- Python/ceval.c | 27 +++++------ Python/executor_cases.c.h | 51 ++++++++------------- Python/frame.c | 9 ++-- Python/gc_free_threading.c | 5 +-- Python/generated_cases.c.h | 52 ++++++++++------------ Python/optimizer.c | 2 +- Python/optimizer_analysis.c | 2 +- Python/sysmodule.c | 5 ++- Tools/cases_generator/analyzer.py | 12 +++++ Tools/cases_generator/generators_common.py | 9 ++-- 17 files changed, 143 insertions(+), 137 deletions(-) diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index a77658134fae8c..c9ac3819d0390b 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -62,7 +62,7 @@ enum _frameowner { typedef struct _PyInterpreterFrame { _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */ struct _PyInterpreterFrame *previous; - PyObject *f_funcobj; /* Strong reference. Only valid if not on C stack */ + _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */ PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */ PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */ @@ -84,6 +84,12 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + assert(PyFunction_Check(func)); + return (PyFunctionObject *)func; +} + static inline _PyStackRef *_PyFrame_Stackbase(_PyInterpreterFrame *f) { return (f->localsplus + _PyFrame_GetCode(f)->co_nlocalsplus); } @@ -144,14 +150,15 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * */ static inline void _PyFrame_Initialize( - _PyInterpreterFrame *frame, PyFunctionObject *func, + _PyInterpreterFrame *frame, _PyStackRef func, PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous) { frame->previous = previous; - frame->f_funcobj = (PyObject *)func; + frame->f_funcobj = func; frame->f_executable = PyStackRef_FromPyObjectNew(code); - frame->f_builtins = func->func_builtins; - frame->f_globals = func->func_globals; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + frame->f_builtins = func_obj->func_builtins; + frame->f_globals = func_obj->func_globals; frame->f_locals = locals; frame->stackpointer = frame->localsplus + code->co_nlocalsplus; frame->frame_obj = NULL; @@ -300,10 +307,11 @@ PyAPI_FUNC(void) _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFr * Must be guarded by _PyThreadState_HasStackSpace() * Consumes reference to func. */ static inline _PyInterpreterFrame * -_PyFrame_PushUnchecked(PyThreadState *tstate, PyFunctionObject *func, int null_locals_from, _PyInterpreterFrame * previous) +_PyFrame_PushUnchecked(PyThreadState *tstate, _PyStackRef func, int null_locals_from, _PyInterpreterFrame * previous) { CALL_STAT_INC(frames_pushed); - PyCodeObject *code = (PyCodeObject *)func->func_code; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + PyCodeObject *code = (PyCodeObject *)func_obj->func_code; _PyInterpreterFrame *new_frame = (_PyInterpreterFrame *)tstate->datastack_top; tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); @@ -321,7 +329,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int tstate->datastack_top += code->co_framesize; assert(tstate->datastack_top < tstate->datastack_limit); frame->previous = previous; - frame->f_funcobj = Py_None; + frame->f_funcobj = PyStackRef_None; frame->f_executable = PyStackRef_FromPyObjectNew(code); #ifdef Py_DEBUG frame->f_builtins = NULL; @@ -345,7 +353,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int } PyAPI_FUNC(_PyInterpreterFrame *) -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index cb67a7ee2b3402..cf96f661e6cd7e 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -387,6 +387,17 @@ union _PyStackRef; extern int _PyGC_VisitFrameStack(struct _PyInterpreterFrame *frame, visitproc visit, void *arg); extern int _PyGC_VisitStackRef(union _PyStackRef *ref, visitproc visit, void *arg); +// Like Py_VISIT but for _PyStackRef fields +#define _Py_VISIT_STACKREF(ref) \ + do { \ + if (!PyStackRef_IsNull(ref)) { \ + int vret = _PyGC_VisitStackRef(&(ref), visit, arg); \ + if (vret) \ + return vret; \ + } \ + } while (0) + + #ifdef __cplusplus } #endif diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 0451688a46c75f..c403075fbb2501 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -681,13 +681,13 @@ set_eval_frame_default(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * record_eval(PyThreadState *tstate, struct _PyInterpreterFrame *f, int exc) { - if (PyFunction_Check(f->f_funcobj)) { + if (PyStackRef_FunctionCheck(f->f_funcobj)) { + PyFunctionObject *func = _PyFrame_GetFunction(f); PyObject *module = _get_current_module(); assert(module != NULL); module_state *state = get_module_state(module); Py_DECREF(module); - int res = PyList_Append(state->record_list, - ((PyFunctionObject *)f->f_funcobj)->func_name); + int res = PyList_Append(state->record_list, func->func_name); if (res < 0) { return NULL; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index b567327f970836..9f1c031dcb9a9d 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1634,7 +1634,7 @@ frame_dealloc(PyFrameObject *f) /* Kill all local variables including specials, if we own them */ if (f->f_frame == frame && frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { PyStackRef_CLEAR(frame->f_executable); - Py_CLEAR(frame->f_funcobj); + PyStackRef_CLEAR(frame->f_funcobj); Py_CLEAR(frame->f_locals); _PyStackRef *locals = _PyFrame_GetLocalsArray(frame); _PyStackRef *sp = frame->stackpointer; @@ -1790,7 +1790,7 @@ static void init_frame(_PyInterpreterFrame *frame, PyFunctionObject *func, PyObject *locals) { PyCodeObject *code = (PyCodeObject *)func->func_code; - _PyFrame_Initialize(frame, (PyFunctionObject*)Py_NewRef(func), + _PyFrame_Initialize(frame, PyStackRef_FromPyObjectNew(func), Py_XNewRef(locals), code, 0, NULL); } @@ -1861,14 +1861,15 @@ frame_init_get_vars(_PyInterpreterFrame *frame) PyCodeObject *co = _PyFrame_GetCode(frame); int lasti = _PyInterpreterFrame_LASTI(frame); if (!(lasti < 0 && _PyCode_CODE(co)->op.code == COPY_FREE_VARS - && PyFunction_Check(frame->f_funcobj))) + && PyStackRef_FunctionCheck(frame->f_funcobj))) { /* Free vars are initialized */ return; } /* Free vars have not been initialized -- Do that */ - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + PyFunctionObject *func = _PyFrame_GetFunction(frame); + PyObject *closure = func->func_closure; int offset = PyUnstable_Code_GetFirstFree(co); for (int i = 0; i < co->co_nfreevars; ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); diff --git a/Objects/genobject.c b/Objects/genobject.c index 5dc8f926557b52..41cf8fdcc9dee8 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -58,10 +58,7 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) else { // We still need to visit the code object when the frame is cleared to // ensure that it's kept alive if the reference is deferred. - int err = _PyGC_VisitStackRef(&gen->gi_iframe.f_executable, visit, arg); - if (err) { - return err; - } + _Py_VISIT_STACKREF(gen->gi_iframe.f_executable); } /* No need to visit cr_origin, because it's just tuples/str/int, so can't participate in a reference cycle. */ diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 3c96850589d378..d3656155fae330 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -372,10 +372,10 @@ caller(void) if (f == NULL) { Py_RETURN_NONE; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *r = PyFunction_GetModule(PyStackRef_AsPyObjectBorrow(f->f_funcobj)); if (!r) { PyErr_Clear(); Py_RETURN_NONE; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 846404e28bb18f..5f194aec0073c8 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -808,14 +808,13 @@ dummy_func( assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); } op(_BINARY_SUBSCR_INIT_CALL, (container, sub -- new_frame: _PyInterpreterFrame* )) { PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); SYNC_SP(); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; @@ -1666,8 +1665,9 @@ dummy_func( inst(COPY_FREE_VARS, (--)) { /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { @@ -2170,8 +2170,7 @@ dummy_func( DEOPT_IF(code->co_argcount != 1); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize)); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; } @@ -2202,8 +2201,8 @@ dummy_func( STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -3251,7 +3250,7 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -3340,7 +3339,7 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, @@ -3475,11 +3474,9 @@ dummy_func( } replicate(5) pure op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null[1], args[oparg] -- new_frame: _PyInterpreterFrame*)) { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3601,10 +3598,9 @@ dummy_func( assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); - PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); args[-1] = self; init_frame = _PyEvalFramePushAndInit( - tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, shim); SYNC_SP(); if (init_frame == NULL) { _PyEval_FrameClearAndPop(tstate, shim); @@ -4080,7 +4076,7 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -4148,7 +4144,7 @@ dummy_func( int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -4332,9 +4328,9 @@ dummy_func( int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, - nargs, callargs, kwargs, frame); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, + nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); if (new_frame == NULL) { @@ -4408,8 +4404,8 @@ dummy_func( } inst(RETURN_GENERATOR, (-- res)) { - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { ERROR_NO_POP(); @@ -4771,8 +4767,9 @@ dummy_func( } tier2 op(_CHECK_FUNCTION, (func_version/2 -- )) { - assert(PyFunction_Check(frame->f_funcobj)); - DEOPT_IF(((PyFunctionObject *)frame->f_funcobj)->func_version != func_version); + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + DEOPT_IF(func->func_version != func_version); } /* Internal -- for testing executors */ diff --git a/Python/ceval.c b/Python/ceval.c index 44b39f5d36c93c..6e62939adb3745 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -196,7 +196,7 @@ lltrace_instruction(_PyInterpreterFrame *frame, static void lltrace_resume_frame(_PyInterpreterFrame *frame) { - PyObject *fobj = frame->f_funcobj; + PyObject *fobj = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); if (!PyStackRef_CodeCheck(frame->f_executable) || fobj == NULL || !PyFunction_Check(fobj) @@ -277,7 +277,7 @@ static void monitor_throw(PyThreadState *tstate, static int check_args_iterable(PyThreadState *, PyObject *func, PyObject *vararg); static int get_exception_handler(PyCodeObject *, int, int*, int*, int*); static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous); #ifdef HAVE_ERRNO_H @@ -781,7 +781,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int #ifdef Py_DEBUG /* Set these to invalid but identifiable values for debugging. */ - entry_frame.f_funcobj = (PyObject*)0xaaa0; + entry_frame.f_funcobj = (_PyStackRef){.bits = 0xaaa0}; entry_frame.f_locals = (PyObject*)0xaaa1; entry_frame.frame_obj = (PyFrameObject*)0xaaa2; entry_frame.f_globals = (PyObject*)0xaaa3; @@ -1719,18 +1719,19 @@ _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) /* Consumes references to func, locals and all the args */ _PyInterpreterFrame * -_PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit(PyThreadState *tstate, _PyStackRef func, PyObject *locals, _PyStackRef const* args, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { - PyCodeObject * code = (PyCodeObject *)func->func_code; + PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func); + PyCodeObject * code = (PyCodeObject *)func_obj->func_code; CALL_STAT_INC(frames_pushed); _PyInterpreterFrame *frame = _PyThreadState_PushFrame(tstate, code->co_framesize); if (frame == NULL) { goto fail; } _PyFrame_Initialize(frame, func, locals, code, 0, previous); - if (initialize_locals(tstate, func, frame->localsplus, args, argcount, kwnames)) { + if (initialize_locals(tstate, func_obj, frame->localsplus, args, argcount, kwnames)) { assert(frame->owner == FRAME_OWNED_BY_THREAD); clear_thread_frame(tstate, frame); return NULL; @@ -1738,7 +1739,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, return frame; fail: /* Consume the references */ - Py_DECREF(func); + PyStackRef_CLOSE(func); Py_XDECREF(locals); for (size_t i = 0; i < argcount; i++) { PyStackRef_CLOSE(args[i]); @@ -1754,7 +1755,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, } static _PyInterpreterFrame * -_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, _PyStackRef func, PyObject *locals, PyObject *const* args, size_t argcount, PyObject *kwnames, _PyInterpreterFrame *previous) { @@ -1784,7 +1785,7 @@ _PyEvalFramePushAndInit_UnTagged(PyThreadState *tstate, PyFunctionObject *func, Steals references to func, callargs and kwargs. */ static _PyInterpreterFrame * -_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, +_PyEvalFramePushAndInit_Ex(PyThreadState *tstate, _PyStackRef func, PyObject *locals, Py_ssize_t nargs, PyObject *callargs, PyObject *kwargs, _PyInterpreterFrame *previous) { bool has_dict = (kwargs != NULL && PyDict_GET_SIZE(kwargs) > 0); @@ -1793,7 +1794,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, if (has_dict) { newargs = _PyStack_UnpackDict(tstate, _PyTuple_ITEMS(callargs), nargs, kwargs, &kwnames); if (newargs == NULL) { - Py_DECREF(func); + PyStackRef_CLOSE(func); goto error; } } @@ -1805,7 +1806,7 @@ _PyEvalFramePushAndInit_Ex(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_UnTagged( - tstate, (PyFunctionObject *)func, locals, + tstate, func, locals, newargs, nargs, kwnames, previous ); if (has_dict) { @@ -1831,7 +1832,6 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, { /* _PyEvalFramePushAndInit consumes the references * to func, locals and all its arguments */ - Py_INCREF(func); Py_XINCREF(locals); for (size_t i = 0; i < argcount; i++) { Py_INCREF(args[i]); @@ -1843,7 +1843,8 @@ _PyEval_Vector(PyThreadState *tstate, PyFunctionObject *func, } } _PyInterpreterFrame *frame = _PyEvalFramePushAndInit_UnTagged( - tstate, func, locals, args, argcount, kwnames, NULL); + tstate, PyStackRef_FromPyObjectNew(func), locals, + args, argcount, kwnames, NULL); if (frame == NULL) { return NULL; } diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 93ab068f9de949..7285acec0bacaf 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1018,7 +1018,6 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); break; } @@ -1031,7 +1030,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -1852,8 +1851,9 @@ oparg = CURRENT_OPARG(); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { @@ -2553,8 +2553,7 @@ JUMP_TO_JUMP_TARGET(); } STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; stack_pointer[-1].bits = (uintptr_t)new_frame; break; @@ -3603,7 +3602,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, @@ -3833,11 +3832,9 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3859,11 +3856,9 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3885,11 +3880,9 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3911,11 +3904,9 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3937,11 +3928,9 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -3962,11 +3951,9 @@ args = &stack_pointer[-oparg]; self_or_null = &stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -4145,10 +4132,9 @@ assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); - PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); args[-1] = self; init_frame = _PyEvalFramePushAndInit( - tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (init_frame == NULL) { @@ -4780,7 +4766,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -5001,8 +4987,8 @@ case _RETURN_GENERATOR: { _PyStackRef res; - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { JUMP_TO_ERROR(); @@ -5376,8 +5362,9 @@ case _CHECK_FUNCTION: { uint32_t func_version = (uint32_t)CURRENT_OPERAND(); - assert(PyFunction_Check(frame->f_funcobj)); - if (((PyFunctionObject *)frame->f_funcobj)->func_version != func_version) { + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + if (func->func_version != func_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } diff --git a/Python/frame.c b/Python/frame.c index d7bb29811bfa50..35e6c2d0a93333 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -13,11 +13,8 @@ _PyFrame_Traverse(_PyInterpreterFrame *frame, visitproc visit, void *arg) { Py_VISIT(frame->frame_obj); Py_VISIT(frame->f_locals); - Py_VISIT(frame->f_funcobj); - int err = _PyGC_VisitStackRef(&frame->f_executable, visit, arg); - if (err) { - return err; - } + _Py_VISIT_STACKREF(frame->f_funcobj); + _Py_VISIT_STACKREF(frame->f_executable); return _PyGC_VisitFrameStack(frame, visit, arg); } @@ -126,7 +123,7 @@ _PyFrame_ClearExceptCode(_PyInterpreterFrame *frame) Py_DECREF(f); } _PyFrame_ClearLocals(frame); - Py_DECREF(frame->f_funcobj); + PyStackRef_CLEAR(frame->f_funcobj); } /* Unstable API functions */ diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index c645f1b9a63806..a5bc9b9b5782b2 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -200,6 +200,7 @@ frame_disable_deferred_refcounting(_PyInterpreterFrame *frame) } } + frame->f_funcobj = PyStackRef_AsStrongReference(frame->f_funcobj); for (_PyStackRef *ref = frame->localsplus; ref < frame->stackpointer; ref++) { if (!PyStackRef_IsNull(*ref) && PyStackRef_IsDeferred(*ref)) { *ref = PyStackRef_AsStrongReference(*ref); @@ -994,9 +995,7 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) _PyStackRef *ref = _PyFrame_GetLocalsArray(frame); /* locals and stack */ for (; ref < frame->stackpointer; ref++) { - if (_PyGC_VisitStackRef(ref, visit, arg) < 0) { - return -1; - } + _Py_VISIT_STACKREF(*ref); } return 0; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 6d902e2c1d9ba8..58792a2101ab28 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -506,7 +506,6 @@ assert(code->co_argcount == 2); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); - Py_INCREF(getitem); } // _BINARY_SUBSCR_INIT_CALL sub = stack_pointer[-1]; @@ -514,7 +513,7 @@ PyTypeObject *tp = Py_TYPE(PyStackRef_AsPyObjectBorrow(container)); PyHeapTypeObject *ht = (PyHeapTypeObject *)tp; PyObject *getitem = ht->_spec_cache.getitem; - new_frame = _PyFrame_PushUnchecked(tstate, (PyFunctionObject *)getitem, 2, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(getitem), 2, frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[0] = container; @@ -892,7 +891,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -1021,10 +1020,9 @@ assert(_PyCode_CODE(_PyFrame_GetCode(shim))[0].op.code == EXIT_INIT_CHECK); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self); - PyFunctionObject *init_func = (PyFunctionObject *)PyStackRef_AsPyObjectSteal(init); args[-1] = self; init_frame = _PyEvalFramePushAndInit( - tstate, init_func, NULL, args-1, oparg+1, NULL, shim); + tstate, init, NULL, args-1, oparg+1, NULL, shim); stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); if (init_frame == NULL) { @@ -1119,11 +1117,9 @@ // _INIT_CALL_PY_EXACT_ARGS args = &stack_pointer[-oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -1216,7 +1212,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, @@ -1616,8 +1612,8 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); @@ -1802,7 +1798,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -1936,7 +1932,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -2104,7 +2100,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, positional_args, kwnames_o, frame ); PyStackRef_CLOSE(kwnames); @@ -2649,11 +2645,9 @@ // _INIT_CALL_PY_EXACT_ARGS args = &stack_pointer[-oparg]; { - PyObject *callable_o = PyStackRef_AsPyObjectBorrow(callable); int has_self = !PyStackRef_IsNull(self_or_null[0]); STAT_INC(CALL, hit); - PyFunctionObject *func = (PyFunctionObject *)callable_o; - new_frame = _PyFrame_PushUnchecked(tstate, func, oparg + has_self, frame); + new_frame = _PyFrame_PushUnchecked(tstate, callable, oparg + has_self, frame); _PyStackRef *first_non_self_local = new_frame->localsplus + has_self; new_frame->localsplus[0] = self_or_null[0]; for (int i = 0; i < oparg; i++) { @@ -2726,7 +2720,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // The frame has stolen all the arguments from the stack, @@ -3274,8 +3268,9 @@ INSTRUCTION_STATS(COPY_FREE_VARS); /* Copy closure variables to free variables */ PyCodeObject *co = _PyFrame_GetCode(frame); - assert(PyFunction_Check(frame->f_funcobj)); - PyObject *closure = ((PyFunctionObject *)frame->f_funcobj)->func_closure; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + PyObject *closure = func->func_closure; assert(oparg == co->co_nfreevars); int offset = co->co_nlocalsplus - oparg; for (int i = 0; i < oparg; ++i) { @@ -4102,7 +4097,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)PyStackRef_AsPyObjectSteal(callable), locals, + tstate, callable, locals, args, total_args, NULL, frame ); // Manipulate stack directly since we leave using DISPATCH_INLINED(). @@ -4997,8 +4992,8 @@ DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - Py_INCREF(f); - _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked(tstate, f, 2, frame); + _PyInterpreterFrame *new_frame = _PyFrame_PushUnchecked( + tstate, PyStackRef_FromPyObjectNew(f), 2, frame); // Manipulate stack directly because we exit with DISPATCH_INLINED(). STACK_SHRINK(1); new_frame->localsplus[0] = owner; @@ -5328,8 +5323,7 @@ DEOPT_IF(code->co_argcount != 1, LOAD_ATTR); DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize), LOAD_ATTR); STAT_INC(LOAD_ATTR, hit); - Py_INCREF(fget); - new_frame = _PyFrame_PushUnchecked(tstate, f, 1, frame); + new_frame = _PyFrame_PushUnchecked(tstate, PyStackRef_FromPyObjectNew(fget), 1, frame); new_frame->localsplus[0] = owner; } // _SAVE_RETURN_OFFSET @@ -6504,8 +6498,8 @@ next_instr += 1; INSTRUCTION_STATS(RETURN_GENERATOR); _PyStackRef res; - assert(PyFunction_Check(frame->f_funcobj)); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + assert(PyStackRef_FunctionCheck(frame->f_funcobj)); + PyFunctionObject *func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj); PyGenObject *gen = (PyGenObject *)_Py_MakeCoro(func); if (gen == NULL) { goto error; @@ -7718,8 +7712,8 @@ Py_ssize_t nargs = PyTuple_GET_SIZE(callargs); int code_flags = ((PyCodeObject *)PyFunction_GET_CODE(func))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(func)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex(tstate, - (PyFunctionObject *)PyStackRef_AsPyObjectSteal(func_st), locals, + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit_Ex( + tstate, func_st, locals, nargs, callargs, kwargs, frame); // Need to manually shrink the stack since we exit with DISPATCH_INLINED. STACK_SHRINK(oparg + 3); diff --git a/Python/optimizer.c b/Python/optimizer.c index 9198e410627dd4..bb7a90b3204f40 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -533,7 +533,7 @@ translate_bytecode_to_trace( { bool first = true; PyCodeObject *code = _PyFrame_GetCode(frame); - PyFunctionObject *func = (PyFunctionObject *)frame->f_funcobj; + PyFunctionObject *func = _PyFrame_GetFunction(frame); assert(PyFunction_Check(func)); PyCodeObject *initial_code = code; _Py_BloomFilter_Add(dependencies, initial_code); diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index f7adb44c9e09ef..b202b58a8b7214 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -145,7 +145,7 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, return 1; } PyObject *globals = frame->f_globals; - PyFunctionObject *function = (PyFunctionObject *)frame->f_funcobj; + PyFunctionObject *function = _PyFrame_GetFunction(frame); assert(PyFunction_Check(function)); assert(function->func_builtins == builtins); assert(function->func_globals == globals); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 887a916563a2e1..ac343a8048e008 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2384,10 +2384,11 @@ sys__getframemodulename_impl(PyObject *module, int depth) while (f && (_PyFrame_IsIncomplete(f) || depth-- > 0)) { f = f->previous; } - if (f == NULL || f->f_funcobj == NULL) { + if (f == NULL || PyStackRef_IsNull(f->f_funcobj)) { Py_RETURN_NONE; } - PyObject *r = PyFunction_GetModule(f->f_funcobj); + PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); + PyObject *r = PyFunction_GetModule(func); if (!r) { PyErr_Clear(); r = Py_None; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 3cc36b6b5841bd..0680c21a3c24c5 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -365,12 +365,24 @@ def find_assignment_target(idx: int) -> list[lexer.Token]: offset += 1 return [] + def in_frame_push(idx: int) -> bool: + for tkn in reversed(node.block.tokens[: idx - 1]): + if tkn.kind == "SEMI" or tkn.kind == "LBRACE" or tkn.kind == "RBRACE": + return False + if tkn.kind == "IDENTIFIER" and tkn.text == "_PyFrame_PushUnchecked": + return True + return False + refs: dict[lexer.Token, str | None] = {} for idx, tkn in enumerate(node.block.tokens): if tkn.kind != "IDENTIFIER" or tkn.text != "PyStackRef_FromPyObjectNew": continue if idx == 0 or node.block.tokens[idx - 1].kind != "EQUALS": + if in_frame_push(idx): + # PyStackRef_FromPyObjectNew() is called in _PyFrame_PushUnchecked() + refs[tkn] = None + continue raise analysis_error("Expected '=' before PyStackRef_FromPyObjectNew", tkn) lhs = find_assignment_target(idx) diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 2f8fccec2ea409..4cfd4ad3d05988 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -200,15 +200,16 @@ def py_stack_ref_from_py_object_new( stack: Stack, inst: Instruction | None, ) -> None: - self.out.emit(tkn) - emit_to(self.out, tkn_iter, "SEMI") - self.out.emit(";\n") - target = uop.deferred_refs[tkn] if target is None: # An assignment we don't handle, such as to a pointer or array. + self.out.emit(tkn) return + self.out.emit(tkn) + emit_to(self.out, tkn_iter, "SEMI") + self.out.emit(";\n") + # Flush the assignment to the stack. Note that we don't flush the # stack pointer here, and instead are currently relying on initializing # unused portions of the stack to NULL. From b169cf394fe70dfbc7bbe22ae703be3b21845add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Tue, 24 Sep 2024 22:23:17 +0200 Subject: [PATCH 21/39] gh-86009: Fix solaris detection in `_USE_CP_SENDFILE` check (GH-124289) --- Lib/shutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/shutil.py b/Lib/shutil.py index 89c12b76b61dfc..dab3ca5ee91245 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -48,7 +48,7 @@ # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 _USE_CP_SENDFILE = (hasattr(os, "sendfile") - and sys.platform.startswith(("linux", "android", "solaris"))) + and sys.platform.startswith(("linux", "android", "sunos"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 From 5a605660745d32a9b9f4208666889c702527208c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 24 Sep 2024 22:36:45 +0200 Subject: [PATCH 22/39] gh-124402: Require cpu resource in test_super slow method (#124434) test___class___modification_multithreaded() now requires the 'cpu' test resource on a Free Threaded build. --- Lib/test/test_super.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 3ffbe03f0c2f11..b0d1f12513d404 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -4,6 +4,7 @@ import threading import unittest from unittest.mock import patch +from test import support from test.support import import_helper, threading_helper @@ -513,6 +514,11 @@ def test___class___modification_multithreaded(self): This should be the case anyways as our test suite sets an audit hook. """ + + if support.Py_GIL_DISABLED: + # gh-124402: On a Free Threaded build, the test takes a few minutes + support.requires('cpu') + class Foo: pass From 950fab46ad3a1960aa289d2d1de55447b88e25d7 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 24 Sep 2024 14:02:18 -0700 Subject: [PATCH 23/39] dataclasses: Avoid using private class (#124465) typing.get_origin() does what we need here, without reaching into typing internals. This shouldn't change any behavior (so I am going to skip news), but it sets a good example for other users introspecting typing objects. --- Lib/dataclasses.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index ac7d40cf2cac2e..6255d8980974e0 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -690,11 +690,8 @@ def _frozen_get_del_attr(cls, fields, func_builder): def _is_classvar(a_type, typing): - # This test uses a typing internal class, but it's the best way to - # test if this is a ClassVar. return (a_type is typing.ClassVar - or (type(a_type) is typing._GenericAlias - and a_type.__origin__ is typing.ClassVar)) + or (typing.get_origin(a_type) is typing.ClassVar)) def _is_initvar(a_type, dataclasses): From b6471f4a391e9d3c8e1b244c56a4cc25974632c9 Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Tue, 24 Sep 2024 18:09:50 -0700 Subject: [PATCH 24/39] GH-124408: Temporarily skip test_strftime_y2k for emulated Linux JIT CI (GH-124466) --- .github/workflows/jit.yml | 2 +- Tools/jit/ignore-tests-emulated-linux.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 8f56220feea90b..754f179f105591 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -157,7 +157,7 @@ jobs: CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \ CPP="$CC --preprocess" \ HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes + ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes make all --jobs 4 ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt index dbb364673b5c1a..e379e39def0eaf 100644 --- a/Tools/jit/ignore-tests-emulated-linux.txt +++ b/Tools/jit/ignore-tests-emulated-linux.txt @@ -1,4 +1,5 @@ test_multiprocessing_fork +test_strftime_y2k test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling From 5c6e3b715082bfccd0b4cf2bb1c18e8b1afcad3e Mon Sep 17 00:00:00 2001 From: "Eric V. Smith" Date: Tue, 24 Sep 2024 21:26:26 -0400 Subject: [PATCH 25/39] gh-90562: Support zero argument super with dataclasses when slots=True (gh-124455) Co-authored-by: @wookie184 Co-authored-by: Carl Meyer --- Doc/library/dataclasses.rst | 7 - Lib/dataclasses.py | 57 ++++++-- Lib/test/test_dataclasses/__init__.py | 126 +++++++++++++++++- ...4-09-23-18-26-17.gh-issue-90562.Yj566G.rst | 3 + 4 files changed, 177 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index cfca11afbd2e41..1457392ce6e86c 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -187,13 +187,6 @@ Module contents If :attr:`!__slots__` is already defined in the class, then :exc:`TypeError` is raised. - .. warning:: - Calling no-arg :func:`super` in dataclasses using ``slots=True`` - will result in the following exception being raised: - ``TypeError: super(type, obj): obj must be an instance or subtype of type``. - The two-arg :func:`super` is a valid workaround. - See :gh:`90562` for full details. - .. warning:: Passing parameters to a base class :meth:`~object.__init_subclass__` when using ``slots=True`` will result in a :exc:`TypeError`. diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 6255d8980974e0..f5cb97edaf72cd 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1218,9 +1218,31 @@ def _get_slots(cls): raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") +def _update_func_cell_for__class__(f, oldcls, newcls): + # Returns True if we update a cell, else False. + if f is None: + # f will be None in the case of a property where not all of + # fget, fset, and fdel are used. Nothing to do in that case. + return False + try: + idx = f.__code__.co_freevars.index("__class__") + except ValueError: + # This function doesn't reference __class__, so nothing to do. + return False + # Fix the cell to point to the new class, if it's already pointing + # at the old class. I'm not convinced that the "is oldcls" test + # is needed, but other than performance can't hurt. + closure = f.__closure__[idx] + if closure.cell_contents is oldcls: + closure.cell_contents = newcls + return True + return False + + def _add_slots(cls, is_frozen, weakref_slot): - # Need to create a new class, since we can't set __slots__ - # after a class has been created. + # Need to create a new class, since we can't set __slots__ after a + # class has been created, and the @dataclass decorator is called + # after the class is created. # Make sure __slots__ isn't already set. if '__slots__' in cls.__dict__: @@ -1259,18 +1281,37 @@ def _add_slots(cls, is_frozen, weakref_slot): # And finally create the class. qualname = getattr(cls, '__qualname__', None) - cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) + newcls = type(cls)(cls.__name__, cls.__bases__, cls_dict) if qualname is not None: - cls.__qualname__ = qualname + newcls.__qualname__ = qualname if is_frozen: # Need this for pickling frozen classes with slots. if '__getstate__' not in cls_dict: - cls.__getstate__ = _dataclass_getstate + newcls.__getstate__ = _dataclass_getstate if '__setstate__' not in cls_dict: - cls.__setstate__ = _dataclass_setstate - - return cls + newcls.__setstate__ = _dataclass_setstate + + # Fix up any closures which reference __class__. This is used to + # fix zero argument super so that it points to the correct class + # (the newly created one, which we're returning) and not the + # original class. We can break out of this loop as soon as we + # make an update, since all closures for a class will share a + # given cell. + for member in newcls.__dict__.values(): + # If this is a wrapped function, unwrap it. + member = inspect.unwrap(member) + + if isinstance(member, types.FunctionType): + if _update_func_cell_for__class__(member, cls, newcls): + break + elif isinstance(member, property): + if (_update_func_cell_for__class__(member.fget, cls, newcls) + or _update_func_cell_for__class__(member.fset, cls, newcls) + or _update_func_cell_for__class__(member.fdel, cls, newcls)): + break + + return newcls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 6934e88d9d338c..69e86162e0c11a 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -17,7 +17,7 @@ from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints from collections import deque, OrderedDict, namedtuple, defaultdict -from functools import total_ordering +from functools import total_ordering, wraps import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. @@ -4869,5 +4869,129 @@ class A: self.assertEqual(fs[0].name, 'x') +class TestZeroArgumentSuperWithSlots(unittest.TestCase): + def test_zero_argument_super(self): + @dataclass(slots=True) + class A: + def foo(self): + super() + + A().foo() + + def test_dunder_class_with_old_property(self): + @dataclass(slots=True) + class A: + def _get_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + return __class__ + + def _set_foo(slf, value): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + def _del_foo(slf): + self.assertIs(__class__, type(slf)) + self.assertIs(__class__, slf.__class__) + + foo = property(_get_foo, _set_foo, _del_foo) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + def test_dunder_class_with_new_property(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return slf.__class__ + + @foo.setter + def foo(slf, value): + self.assertIs(__class__, type(slf)) + + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + self.assertIs(a.foo, A) + a.foo = 4 + del a.foo + + # Test the parts of a property individually. + def test_slots_dunder_class_property_getter(self): + @dataclass(slots=True) + class A: + @property + def foo(slf): + return __class__ + + a = A() + self.assertIs(a.foo, A) + + def test_slots_dunder_class_property_setter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.setter + def foo(slf, val): + self.assertIs(__class__, type(slf)) + + a = A() + a.foo = 4 + + def test_slots_dunder_class_property_deleter(self): + @dataclass(slots=True) + class A: + foo = property() + @foo.deleter + def foo(slf): + self.assertIs(__class__, type(slf)) + + a = A() + del a.foo + + def test_wrapped(self): + def mydecorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + return f(*args, **kwargs) + return wrapper + + @dataclass(slots=True) + class A: + @mydecorator + def foo(self): + super() + + A().foo() + + def test_remembered_class(self): + # Apply the dataclass decorator manually (not when the class + # is created), so that we can keep a reference to the + # undecorated class. + class A: + def cls(self): + return __class__ + + self.assertIs(A().cls(), A) + + B = dataclass(slots=True)(A) + self.assertIs(B().cls(), B) + + # This is undesirable behavior, but is a function of how + # modifying __class__ in the closure works. I'm not sure this + # should be tested or not: I don't really want to guarantee + # this behavior, but I don't want to lose the point that this + # is how it works. + + # The underlying class is "broken" by changing its __class__ + # in A.foo() to B. This normally isn't a problem, because no + # one will be keeping a reference to the underlying class A. + self.assertIs(A().cls(), B) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst b/Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst new file mode 100644 index 00000000000000..7a389fefc6c54b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-23-18-26-17.gh-issue-90562.Yj566G.rst @@ -0,0 +1,3 @@ +Modify dataclasses to support zero-argument super() when ``slots=True`` is +specified. This works by modifying all references to ``__class__`` to point +to the newly created class. From 54dd77fb8c880d7655fffab934978e277b4275fe Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Tue, 24 Sep 2024 20:27:23 -0500 Subject: [PATCH 26/39] Adjust build_ubuntu_ssltests job to use cache for the correct OS version (GH-124403) --- .github/workflows/build.yml | 7 ++++--- .github/workflows/reusable-ubuntu.yml | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5f6fd47e7367b..ec7904c2e2cc73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -195,13 +195,14 @@ jobs: build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source if: needs.check_source.outputs.run_tests == 'true' strategy: fail-fast: false matrix: + os: [ubuntu-22.04] openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2] env: OPENSSL_VER: ${{ matrix.openssl_ver }} @@ -231,7 +232,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux @@ -410,7 +411,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 01bd914af79fa0..769f1210de4d3c 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -14,7 +14,11 @@ jobs: build_ubuntu_reusable: name: 'build and test' timeout-minutes: 60 - runs-on: ubuntu-22.04 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04] env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 @@ -36,7 +40,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux From 0d38409f422b7be158a45e59766d8f4605dfa5df Mon Sep 17 00:00:00 2001 From: Tom Most Date: Wed, 25 Sep 2024 00:54:13 -0700 Subject: [PATCH 27/39] Doc: Improve documentation for the ``path`` argument in ``shutil.which()`` (#124494) --- Doc/library/shutil.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index e623c3df7beba6..d25701c087ed07 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -449,9 +449,10 @@ Directory and files operations *mode* is a permission mask passed to :func:`os.access`, by default determining if the file exists and is executable. - *path* is a "``PATH`` string" specifying the lookup directory list. When no - *path* is specified, the results of :func:`os.environ` are used, returning - either the "PATH" value or a fallback of :data:`os.defpath`. + *path* is a "``PATH`` string" specifying the directories to look in, + delimited by :data:`os.pathsep`. When no *path* is specified, the + :envvar:`PATH` environment variable is read from :data:`os.environ`, + falling back to :data:`os.defpath` if it is not set. On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the @@ -460,9 +461,9 @@ Directory and files operations consulting the current working directory for executables: set the environment variable ``NoDefaultCurrentDirectoryInExePath``. - Also on Windows, the ``PATHEXT`` variable is used to resolve commands - that may not already include an extension. For example, if you call - ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` + Also on Windows, the :envvar:`PATHEXT` environment variable is used to + resolve commands that may not already include an extension. For example, + if you call ``shutil.which("python")``, :func:`which` will search ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* directories. For example, on Windows:: From 8447c933da308939b06e33544ca9abc9fc46aa8b Mon Sep 17 00:00:00 2001 From: Mat S Date: Wed, 25 Sep 2024 10:24:47 +0100 Subject: [PATCH 28/39] gh-123445: calendar: Improve descriptions for day and month attributes (#123483) Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> Co-authored-by: Eric V. Smith Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/calendar.rst | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index d5876054da3eee..eafc038d6cb722 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -393,13 +393,22 @@ The :mod:`calendar` module exports the following data attributes: .. data:: day_name - An array that represents the days of the week in the current locale. + A sequence that represents the days of the week in the current locale, + where Monday is day number 0. + + >>> import calendar + >>> list(calendar.day_name) + ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday'] .. data:: day_abbr - An array that represents the abbreviated days of the week in the current locale. + A sequence that represents the abbreviated days of the week in the current locale, + where Mon is day number 0. + >>> import calendar + >>> list(calendar.day_abbr) + ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] .. data:: MONDAY TUESDAY @@ -426,17 +435,24 @@ The :mod:`calendar` module exports the following data attributes: .. data:: month_name - An array that represents the months of the year in the current locale. This + A sequence that represents the months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_name[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_name) + ['', 'January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December'] + .. data:: month_abbr - An array that represents the abbreviated months of the year in the current + A sequence that represents the abbreviated months of the year in the current locale. This follows normal convention of January being month number 1, so it has a length of 13 and ``month_abbr[0]`` is the empty string. + >>> import calendar + >>> list(calendar.month_abbr) + ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] .. data:: JANUARY FEBRUARY From 461c12b43870d51ea29eae7b0969b20565d50eb6 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 25 Sep 2024 15:02:18 +0100 Subject: [PATCH 29/39] gh-116622: Make test_unzip_zipfile recognize Android error message format (#124462) Make test_unzip_zipfile recognize Android error message format --- Lib/test/test_shutil.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 80e1d73b6b2aab..37e54d23b22516 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -1909,7 +1909,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) From c58c572a65eb5b93d054e779df289e975a0b9864 Mon Sep 17 00:00:00 2001 From: Malcolm Smith Date: Wed, 25 Sep 2024 15:23:30 +0100 Subject: [PATCH 30/39] gh-123014: Disable pidfd API on older Android versions (#124458) --- Doc/library/os.rst | 2 +- Doc/library/signal.rst | 2 +- .../Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst | 3 +++ Modules/clinic/posixmodule.c.h | 6 +++--- Modules/clinic/signalmodule.c.h | 6 +++--- Modules/posixmodule.c | 5 ++++- Modules/signalmodule.c | 4 +++- 7 files changed, 18 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index cd7ae7bdd7385a..33dd58febd9a5e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -4602,7 +4602,7 @@ written in Python, such as a mail server's external command delivery program. See the :manpage:`pidfd_open(2)` man page for more details. - .. availability:: Linux >= 5.3 + .. availability:: Linux >= 5.3, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 .. data:: PIDFD_NONBLOCK diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 79c4948e99e967..17fcb2b3707978 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -411,7 +411,7 @@ The :mod:`signal` module defines the following functions: See the :manpage:`pidfd_send_signal(2)` man page for more information. - .. availability:: Linux >= 5.1 + .. availability:: Linux >= 5.1, Android >= :func:`build-time ` API level 31 .. versionadded:: 3.9 diff --git a/Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst b/Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst new file mode 100644 index 00000000000000..53dbabd9480ddb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-24-19-32-14.gh-issue-123014.zVcfkZ.rst @@ -0,0 +1,3 @@ +:func:`os.pidfd_open` and :func:`signal.pidfd_send_signal` are now +unavailable when building against Android API levels older than 31, since +the underlying system calls may cause a crash. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9722f06a5935b9..749fe54598cc39 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -5954,7 +5954,7 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ -#if (defined(__linux__) && defined(__NR_pidfd_open)) +#if (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(os_pidfd_open__doc__, "pidfd_open($module, /, pid, flags=0)\n" @@ -6013,7 +6013,7 @@ os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_open)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_open) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #if defined(HAVE_SETNS) @@ -12837,4 +12837,4 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=a736ad3f7205176e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b93bbaaa8eb5b0ce input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 1d3a143dfd8d39..986c0289f2bfcb 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -670,7 +670,7 @@ signal_pthread_kill(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #endif /* defined(HAVE_PTHREAD_KILL) */ -#if (defined(__linux__) && defined(__NR_pidfd_send_signal)) +#if (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) PyDoc_STRVAR(signal_pidfd_send_signal__doc__, "pidfd_send_signal($module, pidfd, signalnum, siginfo=None, flags=0, /)\n" @@ -723,7 +723,7 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar return return_value; } -#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal)) */ +#endif /* (defined(__linux__) && defined(__NR_pidfd_send_signal) && !(defined(__ANDROID__) && __ANDROID_API__ < 31)) */ #ifndef SIGNAL_ALARM_METHODDEF #define SIGNAL_ALARM_METHODDEF @@ -776,4 +776,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=6d8e17a32cef668f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c57b4b98fad6f4b8 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 86366c66c46552..334350285f3b6f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -10121,7 +10121,10 @@ os_wait_impl(PyObject *module) } #endif /* HAVE_WAIT */ -#if defined(__linux__) && defined(__NR_pidfd_open) + +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_open) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] os.pidfd_open pid: pid_t diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 73bfcb756657b8..0e53a36bca55f0 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -1299,7 +1299,9 @@ signal_pthread_kill_impl(PyObject *module, unsigned long thread_id, #endif /* #if defined(HAVE_PTHREAD_KILL) */ -#if defined(__linux__) && defined(__NR_pidfd_send_signal) +// This system call always crashes on older Android versions. +#if defined(__linux__) && defined(__NR_pidfd_send_signal) && \ + !(defined(__ANDROID__) && __ANDROID_API__ < 31) /*[clinic input] signal.pidfd_send_signal From 78aeb38f7d880a340295214abc4f7e77ffdad509 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:51:25 +0100 Subject: [PATCH 31/39] gh-124285: Fix bug where bool() is called multiple times for the same part of a boolean expression (#124394) --- Doc/library/dis.rst | 6 ++ Include/internal/pycore_magic_number.h | 3 +- Include/internal/pycore_opcode_metadata.h | 51 ++++++++----- Include/opcode_ids.h | 16 ++-- Lib/_opcode_metadata.py | 16 ++-- Lib/test/test_compile.py | 39 ++++++++++ Lib/test/test_generated_cases.py | 30 ++++++++ ...-09-23-23-06-19.gh-issue-124285.mahGTg.rst | 2 + Python/bytecodes.c | 8 ++ Python/codegen.c | 6 +- Python/flowgraph.c | 73 ++++++++++++++++++- Tools/cases_generator/analyzer.py | 2 + .../opcode_metadata_generator.py | 4 +- Tools/cases_generator/parsing.py | 25 +++++-- 14 files changed, 234 insertions(+), 47 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index cad73192f7cd43..662c890d996a24 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1872,6 +1872,12 @@ but are replaced by real opcodes or removed before bytecode is generated. Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. +.. opcode:: JUMP_IF_TRUE +.. opcode:: JUMP_IF_FALSE + + Conditional jumps which do not impact the stack. Replaced by the sequence + ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + .. opcode:: LOAD_CLOSURE (i) Pushes a reference to the cell contained in slot ``i`` of the "fast locals" diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 095eb0f8a89b79..2414d25d41bfbf 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -258,6 +258,7 @@ Known values: Python 3.14a1 3604 (Do not duplicate test at end of while statements) Python 3.14a1 3605 (Move ENTER_EXECUTOR to opcode 255) Python 3.14a1 3606 (Specialize CALL_KW) + Python 3.14a1 3607 (Add pseudo instructions JUMP_IF_TRUE/FALSE) Python 3.15 will start with 3650 @@ -270,7 +271,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3606 +#define PYC_MAGIC_NUMBER 3607 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 51479afae3833d..3344ede5e92c07 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -22,6 +22,8 @@ extern "C" { ((OP) == STORE_FAST_MAYBE_NULL) || \ ((OP) == JUMP) || \ ((OP) == JUMP_NO_INTERRUPT) || \ + ((OP) == JUMP_IF_FALSE) || \ + ((OP) == JUMP_IF_TRUE) || \ ((OP) == SETUP_FINALLY) || \ ((OP) == SETUP_CLEANUP) || \ ((OP) == SETUP_WITH) || \ @@ -269,6 +271,10 @@ int _PyOpcode_num_popped(int opcode, int oparg) { return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; case JUMP_NO_INTERRUPT: return 0; case LIST_APPEND: @@ -726,6 +732,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg) { return 0; case JUMP_FORWARD: return 0; + case JUMP_IF_FALSE: + return 1; + case JUMP_IF_TRUE: + return 1; case JUMP_NO_INTERRUPT: return 0; case LIST_APPEND: @@ -956,7 +966,7 @@ enum InstructionFormat { }; #define IS_VALID_OPCODE(OP) \ - (((OP) >= 0) && ((OP) < 264) && \ + (((OP) >= 0) && ((OP) < 266) && \ (_PyOpcode_opcode_metadata[(OP)].valid_entry)) #define HAS_ARG_FLAG (1) @@ -1005,9 +1015,9 @@ struct opcode_metadata { int16_t flags; }; -extern const struct opcode_metadata _PyOpcode_opcode_metadata[264]; +extern const struct opcode_metadata _PyOpcode_opcode_metadata[266]; #ifdef NEED_OPCODE_METADATA -const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { +const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, @@ -1224,6 +1234,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[264] = { [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [_DO_CALL_FUNCTION_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, + [JUMP_IF_TRUE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_NO_INTERRUPT] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG }, [LOAD_CLOSURE] = { true, -1, HAS_ARG_FLAG | HAS_LOCAL_FLAG | HAS_PURE_FLAG }, [POP_BLOCK] = { true, -1, HAS_PURE_FLAG }, @@ -1422,9 +1434,9 @@ _PyOpcode_macro_expansion[256] = { }; #endif // NEED_OPCODE_METADATA -extern const char *_PyOpcode_OpName[264]; +extern const char *_PyOpcode_OpName[266]; #ifdef NEED_OPCODE_METADATA -const char *_PyOpcode_OpName[264] = { +const char *_PyOpcode_OpName[266] = { [BINARY_OP] = "BINARY_OP", [BINARY_OP_ADD_FLOAT] = "BINARY_OP_ADD_FLOAT", [BINARY_OP_ADD_INT] = "BINARY_OP_ADD_INT", @@ -1543,6 +1555,8 @@ const char *_PyOpcode_OpName[264] = { [JUMP_BACKWARD] = "JUMP_BACKWARD", [JUMP_BACKWARD_NO_INTERRUPT] = "JUMP_BACKWARD_NO_INTERRUPT", [JUMP_FORWARD] = "JUMP_FORWARD", + [JUMP_IF_FALSE] = "JUMP_IF_FALSE", + [JUMP_IF_TRUE] = "JUMP_IF_TRUE", [JUMP_NO_INTERRUPT] = "JUMP_NO_INTERRUPT", [LIST_APPEND] = "LIST_APPEND", [LIST_EXTEND] = "LIST_EXTEND", @@ -1943,25 +1957,28 @@ const uint8_t _PyOpcode_Deopt[256] = { case 235: \ ; struct pseudo_targets { - uint8_t targets[3]; + uint8_t as_sequence; + uint8_t targets[4]; }; -extern const struct pseudo_targets _PyOpcode_PseudoTargets[8]; +extern const struct pseudo_targets _PyOpcode_PseudoTargets[10]; #ifdef NEED_OPCODE_METADATA -const struct pseudo_targets _PyOpcode_PseudoTargets[8] = { - [LOAD_CLOSURE-256] = { { LOAD_FAST, 0, 0 } }, - [STORE_FAST_MAYBE_NULL-256] = { { STORE_FAST, 0, 0 } }, - [JUMP-256] = { { JUMP_FORWARD, JUMP_BACKWARD, 0 } }, - [JUMP_NO_INTERRUPT-256] = { { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0 } }, - [SETUP_FINALLY-256] = { { NOP, 0, 0 } }, - [SETUP_CLEANUP-256] = { { NOP, 0, 0 } }, - [SETUP_WITH-256] = { { NOP, 0, 0 } }, - [POP_BLOCK-256] = { { NOP, 0, 0 } }, +const struct pseudo_targets _PyOpcode_PseudoTargets[10] = { + [LOAD_CLOSURE-256] = { 0, { LOAD_FAST, 0, 0, 0 } }, + [STORE_FAST_MAYBE_NULL-256] = { 0, { STORE_FAST, 0, 0, 0 } }, + [JUMP-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD, 0, 0 } }, + [JUMP_NO_INTERRUPT-256] = { 0, { JUMP_FORWARD, JUMP_BACKWARD_NO_INTERRUPT, 0, 0 } }, + [JUMP_IF_FALSE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_FALSE, 0 } }, + [JUMP_IF_TRUE-256] = { 1, { COPY, TO_BOOL, POP_JUMP_IF_TRUE, 0 } }, + [SETUP_FINALLY-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_CLEANUP-256] = { 0, { NOP, 0, 0, 0 } }, + [SETUP_WITH-256] = { 0, { NOP, 0, 0, 0 } }, + [POP_BLOCK-256] = { 0, { NOP, 0, 0, 0 } }, }; #endif // NEED_OPCODE_METADATA static inline bool is_pseudo_target(int pseudo, int target) { - if (pseudo < 256 || pseudo >= 264) { + if (pseudo < 256 || pseudo >= 266) { return false; } for (int i = 0; _PyOpcode_PseudoTargets[pseudo-256].targets[i]; i++) { diff --git a/Include/opcode_ids.h b/Include/opcode_ids.h index 5ded0b41b4830e..8ba1ab25a77770 100644 --- a/Include/opcode_ids.h +++ b/Include/opcode_ids.h @@ -226,13 +226,15 @@ extern "C" { #define INSTRUMENTED_LINE 254 #define ENTER_EXECUTOR 255 #define JUMP 256 -#define JUMP_NO_INTERRUPT 257 -#define LOAD_CLOSURE 258 -#define POP_BLOCK 259 -#define SETUP_CLEANUP 260 -#define SETUP_FINALLY 261 -#define SETUP_WITH 262 -#define STORE_FAST_MAYBE_NULL 263 +#define JUMP_IF_FALSE 257 +#define JUMP_IF_TRUE 258 +#define JUMP_NO_INTERRUPT 259 +#define LOAD_CLOSURE 260 +#define POP_BLOCK 261 +#define SETUP_CLEANUP 262 +#define SETUP_FINALLY 263 +#define SETUP_WITH 264 +#define STORE_FAST_MAYBE_NULL 265 #define HAVE_ARGUMENT 41 #define MIN_SPECIALIZED_OPCODE 150 diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py index 6e4b33921863cb..dd70c5250c0b1e 100644 --- a/Lib/_opcode_metadata.py +++ b/Lib/_opcode_metadata.py @@ -335,13 +335,15 @@ 'INSTRUMENTED_CALL': 252, 'INSTRUMENTED_JUMP_BACKWARD': 253, 'JUMP': 256, - 'JUMP_NO_INTERRUPT': 257, - 'LOAD_CLOSURE': 258, - 'POP_BLOCK': 259, - 'SETUP_CLEANUP': 260, - 'SETUP_FINALLY': 261, - 'SETUP_WITH': 262, - 'STORE_FAST_MAYBE_NULL': 263, + 'JUMP_IF_FALSE': 257, + 'JUMP_IF_TRUE': 258, + 'JUMP_NO_INTERRUPT': 259, + 'LOAD_CLOSURE': 260, + 'POP_BLOCK': 261, + 'SETUP_CLEANUP': 262, + 'SETUP_FINALLY': 263, + 'SETUP_WITH': 264, + 'STORE_FAST_MAYBE_NULL': 265, } HAVE_ARGUMENT = 41 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 736eff35c1d5f2..b81d847c824273 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1527,6 +1527,45 @@ async def name_4(): pass [[]] +class TestBooleanExpression(unittest.TestCase): + class Value: + def __init__(self): + self.called = 0 + + def __bool__(self): + self.called += 1 + return self.value + + class Yes(Value): + value = True + + class No(Value): + value = False + + def test_short_circuit_and(self): + v = [self.Yes(), self.No(), self.Yes()] + res = v[0] and v[1] and v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_short_circuit_or(self): + v = [self.No(), self.Yes(), self.No()] + res = v[0] or v[1] or v[0] + self.assertIs(res, v[1]) + self.assertEqual([e.called for e in v], [1, 1, 0]) + + def test_compound(self): + # See gh-124285 + v = [self.No(), self.Yes(), self.Yes(), self.Yes()] + res = v[0] and v[1] or v[2] or v[3] + self.assertIs(res, v[2]) + self.assertEqual([e.called for e in v], [1, 0, 1, 0]) + + v = [self.No(), self.No(), self.Yes(), self.Yes(), self.No()] + res = v[0] or v[1] and v[2] or v[3] or v[4] + self.assertIs(res, v[3]) + self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 5d20e3c30bcf10..214e53dde64bbf 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -523,6 +523,36 @@ def test_pseudo_instruction_with_flags(self): """ self.run_cases_test(input, output) + def test_pseudo_instruction_as_sequence(self): + input = """ + pseudo(OP, (in -- out1, out2)) = [ + OP1, OP2 + ]; + + inst(OP1, (--)) { + } + + inst(OP2, (--)) { + } + """ + output = """ + TARGET(OP1) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP1); + DISPATCH(); + } + + TARGET(OP2) { + frame->instr_ptr = next_instr; + next_instr += 1; + INSTRUCTION_STATS(OP2); + DISPATCH(); + } + """ + self.run_cases_test(input, output) + + def test_array_input(self): input = """ inst(OP, (below, values[oparg*2], above --)) { diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst b/Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst new file mode 100644 index 00000000000000..a6dec66a743f92 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-09-23-23-06-19.gh-issue-124285.mahGTg.rst @@ -0,0 +1,2 @@ +Fix bug where ``bool(a)`` can be invoked more than once during the +evaluation of a compound boolean expression. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 5f194aec0073c8..bf8f6af83fc56e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -2570,6 +2570,14 @@ dummy_func( JUMP_BACKWARD_NO_INTERRUPT, }; + pseudo(JUMP_IF_FALSE, (cond -- cond)) = [ + COPY, TO_BOOL, POP_JUMP_IF_FALSE, + ]; + + pseudo(JUMP_IF_TRUE, (cond -- cond)) = [ + COPY, TO_BOOL, POP_JUMP_IF_TRUE, + ]; + tier1 inst(ENTER_EXECUTOR, (--)) { #ifdef _Py_TIER2 PyCodeObject *code = _PyFrame_GetCode(frame); diff --git a/Python/codegen.c b/Python/codegen.c index 0305f4299aec56..896c30cc14952a 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -3140,17 +3140,15 @@ codegen_boolop(compiler *c, expr_ty e) location loc = LOC(e); assert(e->kind == BoolOp_kind); if (e->v.BoolOp.op == And) - jumpi = POP_JUMP_IF_FALSE; + jumpi = JUMP_IF_FALSE; else - jumpi = POP_JUMP_IF_TRUE; + jumpi = JUMP_IF_TRUE; NEW_JUMP_TARGET_LABEL(c, end); s = e->v.BoolOp.values; n = asdl_seq_LEN(s) - 1; assert(n >= 0); for (i = 0; i < n; ++i) { VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i)); - ADDOP_I(c, loc, COPY, 1); - ADDOP(c, loc, TO_BOOL); ADDOP_JUMP(c, loc, jumpi, end); ADDOP(c, loc, POP_TOP); } diff --git a/Python/flowgraph.c b/Python/flowgraph.c index f7d8efb28e21c4..69d7e0a872aa48 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -1589,6 +1589,8 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * switch(nextop) { case POP_JUMP_IF_FALSE: case POP_JUMP_IF_TRUE: + case JUMP_IF_FALSE: + case JUMP_IF_TRUE: { /* Remove LOAD_CONST const; conditional jump */ PyObject* cnt = get_const_value(opcode, oparg, consts); @@ -1600,8 +1602,11 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * if (is_true == -1) { return ERROR; } - INSTR_SET_OP0(inst, NOP); - int jump_if_true = nextop == POP_JUMP_IF_TRUE; + if (PyCompile_OpcodeStackEffect(nextop, 0) == -1) { + /* POP_JUMP_IF_FALSE or POP_JUMP_IF_TRUE */ + INSTR_SET_OP0(inst, NOP); + } + int jump_if_true = (nextop == POP_JUMP_IF_TRUE || nextop == JUMP_IF_TRUE); if (is_true == jump_if_true) { bb->b_instr[i+1].i_opcode = JUMP; } @@ -1761,6 +1766,36 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) i -= jump_thread(bb, inst, target, POP_JUMP_IF_TRUE); } break; + case JUMP_IF_FALSE: + switch (target->i_opcode) { + case JUMP: + case JUMP_IF_FALSE: + i -= jump_thread(bb, inst, target, JUMP_IF_FALSE); + continue; + case JUMP_IF_TRUE: + // No need to check for loops here, a block's b_next + // cannot point to itself. + assert(inst->i_target != inst->i_target->b_next); + inst->i_target = inst->i_target->b_next; + i--; + continue; + } + break; + case JUMP_IF_TRUE: + switch (target->i_opcode) { + case JUMP: + case JUMP_IF_TRUE: + i -= jump_thread(bb, inst, target, JUMP_IF_TRUE); + continue; + case JUMP_IF_FALSE: + // No need to check for loops here, a block's b_next + // cannot point to itself. + assert(inst->i_target != inst->i_target->b_next); + inst->i_target = inst->i_target->b_next; + i--; + continue; + } + break; case JUMP: case JUMP_NO_INTERRUPT: switch (target->i_opcode) { @@ -2367,6 +2402,38 @@ push_cold_blocks_to_end(cfg_builder *g) { return SUCCESS; } +static int +convert_pseudo_conditional_jumps(cfg_builder *g) +{ + basicblock *entryblock = g->g_entryblock; + for (basicblock *b = entryblock; b != NULL; b = b->b_next) { + for (int i = 0; i < b->b_iused; i++) { + cfg_instr *instr = &b->b_instr[i]; + if (instr->i_opcode == JUMP_IF_FALSE || instr->i_opcode == JUMP_IF_TRUE) { + assert(i == b->b_iused - 1); + instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? + POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; + location loc = instr->i_loc; + cfg_instr copy = { + .i_opcode = COPY, + .i_oparg = 1, + .i_loc = loc, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, ©)); + cfg_instr to_bool = { + .i_opcode = TO_BOOL, + .i_oparg = 0, + .i_loc = loc, + .i_target = NULL, + }; + RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); + } + } + } + return SUCCESS; +} + static int convert_pseudo_ops(cfg_builder *g) { @@ -2826,6 +2893,8 @@ _PyCfg_OptimizedCfgToInstructionSequence(cfg_builder *g, int *stackdepth, int *nlocalsplus, _PyInstructionSequence *seq) { + RETURN_IF_ERROR(convert_pseudo_conditional_jumps(g)); + *stackdepth = calculate_stackdepth(g); if (*stackdepth < 0) { return ERROR; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 0680c21a3c24c5..aabe205125856c 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -248,6 +248,7 @@ class PseudoInstruction: name: str stack: StackEffect targets: list[Instruction] + as_sequence: bool flags: list[str] opcode: int = -1 @@ -852,6 +853,7 @@ def add_pseudo( pseudo.name, analyze_stack(pseudo), [instructions[target] for target in pseudo.targets], + pseudo.as_sequence, pseudo.flags, ) diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 9b1bc98b5c08d7..2ad7604af9cc0d 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -305,6 +305,7 @@ def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: table_size = len(analysis.pseudos) max_targets = max(len(pseudo.targets) for pseudo in analysis.pseudos.values()) out.emit("struct pseudo_targets {\n") + out.emit(f"uint8_t as_sequence;\n") out.emit(f"uint8_t targets[{max_targets + 1}];\n") out.emit("};\n") out.emit( @@ -315,10 +316,11 @@ def generate_pseudo_targets(analysis: Analysis, out: CWriter) -> None: f"const struct pseudo_targets _PyOpcode_PseudoTargets[{table_size}] = {{\n" ) for pseudo in analysis.pseudos.values(): + as_sequence = "1" if pseudo.as_sequence else "0" targets = ["0"] * (max_targets + 1) for i, target in enumerate(pseudo.targets): targets[i] = target.name - out.emit(f"[{pseudo.name}-256] = {{ {{ {', '.join(targets)} }} }},\n") + out.emit(f"[{pseudo.name}-256] = {{ {as_sequence}, {{ {', '.join(targets)} }} }},\n") out.emit("};\n\n") out.emit("#endif // NEED_OPCODE_METADATA\n") out.emit("static inline bool\n") diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py index ab5444d41ac6a9..de31d9b232f9df 100644 --- a/Tools/cases_generator/parsing.py +++ b/Tools/cases_generator/parsing.py @@ -148,6 +148,7 @@ class Pseudo(Node): outputs: list[OutputEffect] flags: list[str] # instr flags to set on the pseudo instruction targets: list[str] # opcodes this can be replaced by + as_sequence: bool AstNode = InstDef | Macro | Pseudo | Family @@ -423,16 +424,22 @@ def pseudo_def(self) -> Pseudo | None: flags = [] if self.expect(lx.RPAREN): if self.expect(lx.EQUALS): - if not self.expect(lx.LBRACE): - raise self.make_syntax_error("Expected {") - if members := self.members(): - if self.expect(lx.RBRACE) and self.expect(lx.SEMI): + if self.expect(lx.LBRACE): + as_sequence = False + closing = lx.RBRACE + elif self.expect(lx.LBRACKET): + as_sequence = True + closing = lx.RBRACKET + else: + raise self.make_syntax_error("Expected { or [") + if members := self.members(allow_sequence=True): + if self.expect(closing) and self.expect(lx.SEMI): return Pseudo( - tkn.text, inp, outp, flags, members + tkn.text, inp, outp, flags, members, as_sequence ) return None - def members(self) -> list[str] | None: + def members(self, allow_sequence : bool=False) -> list[str] | None: here = self.getpos() if tkn := self.expect(lx.IDENTIFIER): members = [tkn.text] @@ -442,8 +449,10 @@ def members(self) -> list[str] | None: else: break peek = self.peek() - if not peek or peek.kind != lx.RBRACE: - raise self.make_syntax_error("Expected comma or right paren") + kinds = [lx.RBRACE, lx.RBRACKET] if allow_sequence else [lx.RBRACE] + if not peek or peek.kind not in kinds: + raise self.make_syntax_error( + f"Expected comma or right paren{'/bracket' if allow_sequence else ''}") return members self.setpos(here) return None From f8651a2988274be091ecd6efc2916f38d9fb9577 Mon Sep 17 00:00:00 2001 From: Robert Wolff Date: Wed, 25 Sep 2024 18:47:45 +0200 Subject: [PATCH 32/39] gh-123968: fix -f/--float command line option description (#124517) * fix -f/--float command line option description See gh-123968 gh-124009 * Update Doc/library/random.rst --------- Co-authored-by: Sergey B Kirpichev --- Doc/library/random.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index c7f6b0bdd5b822..ef0cfb0e76cef6 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -741,7 +741,7 @@ The following options are accepted: .. option:: -f --float - Print a random floating-point number between 1 and N inclusive, + Print a random floating-point number between 0 and N inclusive, using :meth:`uniform`. If no options are given, the output depends on the input: From 828583a7855f15edc96e6721c226ace098ba463b Mon Sep 17 00:00:00 2001 From: Damien <81557462+Damien-Chen@users.noreply.github.com> Date: Thu, 26 Sep 2024 01:30:34 +0800 Subject: [PATCH 33/39] gh-123223: Adding hyperlink of argument in warnings.catch_warnings: (#123231) * Adding hyperlink of argument * Modify as reviewer suggested --- Doc/library/warnings.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index 68b9ff5ce2f78c..5ea65cbd8ca94c 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -597,6 +597,9 @@ Available Context Managers passed to :func:`simplefilter` as if it were called immediately on entering the context. + See :ref:`warning-filter` for the meaning of the *category* and *lineno* + parameters. + .. note:: The :class:`catch_warnings` manager works by replacing and From 162d152146a563e266232f395f28bbf152a6761c Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Wed, 25 Sep 2024 18:39:14 +0100 Subject: [PATCH 34/39] import: permit __name__ for use in __name__ = "__main__": (#124381) permit __name__ for use in __name__ = "__main__": --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 19b8aa05072c73..0b9d1c233d182a 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -544,7 +544,7 @@ the module. It is **strongly** recommended that you rely on :attr:`__spec__` and its attributes instead of any of the other individual attributes -listed below. +listed below, except :attr:`__name__`. .. attribute:: __name__ From da5855e99a8c2d6ef2bb20124d2ebb862dbb971f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 25 Sep 2024 19:46:01 +0200 Subject: [PATCH 35/39] gh-112301: Use literal format strings in unicode_fromformat_arg (GH-124203) --- Objects/unicodeobject.c | 78 ++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 43 deletions(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 2494c989544ca0..e9589cfe44f3bf 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -2694,11 +2694,6 @@ unicode_fromformat_write_wcstr(_PyUnicodeWriter *writer, const wchar_t *str, #define F_SIZE 3 #define F_PTRDIFF 4 #define F_INTMAX 5 -static const char * const formats[] = {"%d", "%ld", "%lld", "%zd", "%td", "%jd"}; -static const char * const formats_o[] = {"%o", "%lo", "%llo", "%zo", "%to", "%jo"}; -static const char * const formats_u[] = {"%u", "%lu", "%llu", "%zu", "%tu", "%ju"}; -static const char * const formats_x[] = {"%x", "%lx", "%llx", "%zx", "%tx", "%jx"}; -static const char * const formats_X[] = {"%X", "%lX", "%llX", "%zX", "%tX", "%jX"}; static const char* unicode_fromformat_arg(_PyUnicodeWriter *writer, @@ -2840,47 +2835,44 @@ unicode_fromformat_arg(_PyUnicodeWriter *writer, case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': { - /* used by sprintf */ char buffer[MAX_INTMAX_CHARS]; - const char *fmt = NULL; - switch (*f) { - case 'o': fmt = formats_o[sizemod]; break; - case 'u': fmt = formats_u[sizemod]; break; - case 'x': fmt = formats_x[sizemod]; break; - case 'X': fmt = formats_X[sizemod]; break; - default: fmt = formats[sizemod]; break; - } - int issigned = (*f == 'd' || *f == 'i'); + + // Fill buffer using sprinf, with one of many possible format + // strings, like "%llX" for `long long` in hexadecimal. + // The type/size is in `sizemod`; the format is in `*f`. + + // Use macros with nested switches to keep the sprintf format strings + // as compile-time literals, avoiding warnings and maybe allowing + // optimizations. + + // `SPRINT` macro does one sprintf + // Example usage: SPRINT("l", "X", unsigned long) expands to + // sprintf(buffer, "%" "l" "X", va_arg(*vargs, unsigned long)) + #define SPRINT(SIZE_SPEC, FMT_CHAR, TYPE) \ + sprintf(buffer, "%" SIZE_SPEC FMT_CHAR, va_arg(*vargs, TYPE)) + + // One inner switch to handle all format variants + #define DO_SPRINTS(SIZE_SPEC, SIGNED_TYPE, UNSIGNED_TYPE) \ + switch (*f) { \ + case 'o': len = SPRINT(SIZE_SPEC, "o", UNSIGNED_TYPE); break; \ + case 'u': len = SPRINT(SIZE_SPEC, "u", UNSIGNED_TYPE); break; \ + case 'x': len = SPRINT(SIZE_SPEC, "x", UNSIGNED_TYPE); break; \ + case 'X': len = SPRINT(SIZE_SPEC, "X", UNSIGNED_TYPE); break; \ + default: len = SPRINT(SIZE_SPEC, "d", SIGNED_TYPE); break; \ + } + + // Outer switch to handle all the sizes/types switch (sizemod) { - case F_LONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long)); - break; - case F_LONGLONG: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, long long)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned long long)); - break; - case F_SIZE: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, Py_ssize_t)) : - sprintf(buffer, fmt, va_arg(*vargs, size_t)); - break; - case F_PTRDIFF: - len = sprintf(buffer, fmt, va_arg(*vargs, ptrdiff_t)); - break; - case F_INTMAX: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, intmax_t)) : - sprintf(buffer, fmt, va_arg(*vargs, uintmax_t)); - break; - default: - len = issigned ? - sprintf(buffer, fmt, va_arg(*vargs, int)) : - sprintf(buffer, fmt, va_arg(*vargs, unsigned int)); - break; + case F_LONG: DO_SPRINTS("l", long, unsigned long); break; + case F_LONGLONG: DO_SPRINTS("ll", long long, unsigned long long); break; + case F_SIZE: DO_SPRINTS("z", Py_ssize_t, size_t); break; + case F_PTRDIFF: DO_SPRINTS("t", ptrdiff_t, ptrdiff_t); break; + case F_INTMAX: DO_SPRINTS("j", intmax_t, uintmax_t); break; + default: DO_SPRINTS("", int, unsigned int); break; } + #undef SPRINT + #undef DO_SPRINTS + assert(len >= 0); int sign = (buffer[0] == '-'); From 28efeefab7d577ea4fb6e3f6e82f903f2aee271d Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Wed, 25 Sep 2024 11:18:01 -0700 Subject: [PATCH 36/39] gh-123756: Disable restart command if pdb is in inline mode (#123757) --- Doc/library/pdb.rst | 16 +++++++++++++- Doc/whatsnew/3.14.rst | 3 +++ Lib/pdb.py | 12 ++++++++--- Lib/test/test_pdb.py | 21 +++++++++++++++++++ ...-09-06-01-35-11.gh-issue-123756.Ozbhke.rst | 1 + 5 files changed, 49 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index ce7516a52b1d74..1682eb0fbea42d 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -198,7 +198,7 @@ The ``run*`` functions and :func:`set_trace` are aliases for instantiating the access further features, you have to do this yourself: .. class:: Pdb(completekey='tab', stdin=None, stdout=None, skip=None, \ - nosigint=False, readrc=True) + nosigint=False, readrc=True, mode=None) :class:`Pdb` is the debugger class. @@ -217,6 +217,13 @@ access further features, you have to do this yourself: The *readrc* argument defaults to true and controls whether Pdb will load .pdbrc files from the filesystem. + The *mode* argument specifies how the debugger was invoked. + It impacts the workings of some debugger commands. + Valid values are ``'inline'`` (used by the breakpoint() builtin), + ``'cli'`` (used by the command line invocation) + or ``None`` (for backwards compatible behaviour, as before the *mode* + argument was added). + Example call to enable tracing with *skip*:: import pdb; pdb.Pdb(skip=['django.*']).set_trace() @@ -233,6 +240,9 @@ access further features, you have to do this yourself: .. versionchanged:: 3.6 The *readrc* argument. + .. versionadded:: 3.14 + Added the *mode* argument. + .. method:: run(statement, globals=None, locals=None) runeval(expression, globals=None, locals=None) runcall(function, *args, **kwds) @@ -675,6 +685,10 @@ can be overridden by the local file. History, breakpoints, actions and debugger options are preserved. :pdbcmd:`restart` is an alias for :pdbcmd:`run`. + .. versionchanged:: 3.14 + :pdbcmd:`run` and :pdbcmd:`restart` commands are disabled when the + debugger is invoked in ``'inline'`` mode. + .. pdbcommand:: q(uit) Quit from the debugger. The program being executed is aborted. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 5acb9bfe18b2d0..4d22f7ac58a5b7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -318,6 +318,9 @@ pdb :pdbcmd:`commands` are preserved across hard-coded breakpoints. (Contributed by Tian Gao in :gh:`121450`.) +* Added a new argument ``mode`` to :class:`pdb.Pdb`. Disabled ``restart`` + command when :mod:`pdb` is in ``inline`` mode. + (Contributed by Tian Gao in :gh:`123757`.) pickle ------ diff --git a/Lib/pdb.py b/Lib/pdb.py index dd21207a627bee..443160eaaae887 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -309,7 +309,7 @@ class Pdb(bdb.Bdb, cmd.Cmd): _last_pdb_instance = None def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, - nosigint=False, readrc=True): + nosigint=False, readrc=True, mode=None): bdb.Bdb.__init__(self, skip=skip) cmd.Cmd.__init__(self, completekey, stdin, stdout) sys.audit("pdb.Pdb") @@ -321,6 +321,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, self.mainpyfile = '' self._wait_for_mainpyfile = False self.tb_lineno = {} + self.mode = mode # Try to load readline if it exists try: import readline @@ -1611,6 +1612,11 @@ def do_run(self, arg): sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run". """ + if self.mode == 'inline': + self.error('run/restart command is disabled when pdb is running in inline mode.\n' + 'Use the command line interface to enable restarting your program\n' + 'e.g. "python -m pdb myscript.py"') + return if arg: import shlex argv0 = sys.argv[0:1] @@ -2366,7 +2372,7 @@ def set_trace(*, header=None, commands=None): if Pdb._last_pdb_instance is not None: pdb = Pdb._last_pdb_instance else: - pdb = Pdb() + pdb = Pdb(mode='inline') if header is not None: pdb.message(header) pdb.set_trace(sys._getframe().f_back, commands=commands) @@ -2481,7 +2487,7 @@ def main(): # modified by the script being debugged. It's a bad idea when it was # changed by the user from the command line. There is a "restart" command # which allows explicit specification of command line arguments. - pdb = Pdb() + pdb = Pdb(mode='cli') pdb.rcLines.extend(opts.commands) while True: try: diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 3173b0553c232f..84c0e1073a1054 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -901,6 +901,27 @@ def test_pdb_where_command(): (Pdb) continue """ +def test_pdb_restart_command(): + """Test restart command + + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + ... x = 1 + + >>> with PdbTestInput([ # doctest: +ELLIPSIS + ... 'restart', + ... 'continue', + ... ]): + ... test_function() + > (2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False, mode='inline').set_trace() + (Pdb) restart + *** run/restart command is disabled when pdb is running in inline mode. + Use the command line interface to enable restarting your program + e.g. "python -m pdb myscript.py" + (Pdb) continue + """ + def test_pdb_commands_with_set_trace(): """Test that commands can be passed to Pdb.set_trace() diff --git a/Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst b/Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst new file mode 100644 index 00000000000000..258dd591fce767 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-09-06-01-35-11.gh-issue-123756.Ozbhke.rst @@ -0,0 +1 @@ +Added a new argument ``mode`` to :class:`pdb.Pdb`. Only allow :mod:`pdb` from command line to use ``restart`` command. From c1600c78e4565b6bb558ade451abe2648ba4dd0a Mon Sep 17 00:00:00 2001 From: Emily Morehouse Date: Wed, 25 Sep 2024 11:22:03 -0700 Subject: [PATCH 37/39] gh-123856: Fix PyREPL failure when a keyboard interrupt is triggered after using a history search (#124396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Łukasz Langa --- Lib/_pyrepl/simple_interact.py | 11 +- Lib/test/test_pyrepl/test_pyrepl.py | 192 ++++++++++-------- ...-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst | 2 + 3 files changed, 115 insertions(+), 90 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 3c79cf61d04051..342a4b58bfd0f3 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -28,6 +28,7 @@ import _sitebuiltins import linecache import functools +import os import sys import code @@ -50,7 +51,9 @@ def check() -> str: try: _get_reader() except _error as e: - return str(e) or repr(e) or "unknown error" + if term := os.environ.get("TERM", ""): + term = f"; TERM={term}" + return str(str(e) or repr(e) or "unknown error") + term return "" @@ -159,10 +162,8 @@ def maybe_run_command(statement: str) -> bool: input_n += 1 except KeyboardInterrupt: r = _get_reader() - if r.last_command and 'isearch' in r.last_command.__name__: - r.isearch_direction = '' - r.console.forgetinput() - r.pop_input_trans() + if r.input_trans is r.isearch_trans: + r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) r.dirty = True r.refresh() diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index e816de3720670f..0f3e9996e77e45 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -8,7 +8,7 @@ import subprocess import sys import tempfile -from unittest import TestCase, skipUnless +from unittest import TestCase, skipUnless, skipIf from unittest.mock import patch from test.support import force_not_colorized from test.support import SHORT_TIMEOUT @@ -35,6 +35,94 @@ except ImportError: pty = None + +class ReplTestCase(TestCase): + def run_repl( + self, + repl_input: str | list[str], + env: dict | None = None, + *, + cmdline_args: list[str] | None = None, + cwd: str | None = None, + ) -> tuple[str, int]: + temp_dir = None + if cwd is None: + temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) + cwd = temp_dir.name + try: + return self._run_repl( + repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd + ) + finally: + if temp_dir is not None: + temp_dir.cleanup() + + def _run_repl( + self, + repl_input: str | list[str], + *, + env: dict | None, + cmdline_args: list[str] | None, + cwd: str, + ) -> tuple[str, int]: + assert pty + master_fd, slave_fd = pty.openpty() + cmd = [sys.executable, "-i", "-u"] + if env is None: + cmd.append("-I") + elif "PYTHON_HISTORY" not in env: + env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") + if cmdline_args is not None: + cmd.extend(cmdline_args) + + try: + import termios + except ModuleNotFoundError: + pass + else: + term_attr = termios.tcgetattr(slave_fd) + term_attr[6][termios.VREPRINT] = 0 # pass through CTRL-R + term_attr[6][termios.VINTR] = 0 # pass through CTRL-C + termios.tcsetattr(slave_fd, termios.TCSANOW, term_attr) + + process = subprocess.Popen( + cmd, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + cwd=cwd, + text=True, + close_fds=True, + env=env if env else os.environ, + ) + os.close(slave_fd) + if isinstance(repl_input, list): + repl_input = "\n".join(repl_input) + "\n" + os.write(master_fd, repl_input.encode("utf-8")) + + output = [] + while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: + try: + data = os.read(master_fd, 1024).decode("utf-8") + if not data: + break + except OSError: + break + output.append(data) + else: + os.close(master_fd) + process.kill() + self.fail(f"Timeout while waiting for output, got: {''.join(output)}") + + os.close(master_fd) + try: + exit_code = process.wait(timeout=SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + process.kill() + exit_code = process.wait() + return "".join(output), exit_code + + class TestCursorPosition(TestCase): def prepare_reader(self, events): console = FakeConsole(events) @@ -968,7 +1056,20 @@ def test_bracketed_paste_single_line(self): @skipUnless(pty, "requires pty") -class TestMain(TestCase): +class TestDumbTerminal(ReplTestCase): + def test_dumb_terminal_exits_cleanly(self): + env = os.environ.copy() + env.update({"TERM": "dumb"}) + output, exit_code = self.run_repl("exit()\n", env=env) + self.assertEqual(exit_code, 0) + self.assertIn("warning: can't use pyrepl", output) + self.assertNotIn("Exception", output) + self.assertNotIn("Traceback", output) + + +@skipUnless(pty, "requires pty") +@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +class TestMain(ReplTestCase): def setUp(self): # Cleanup from PYTHON* variables to isolate from local # user settings, see #121359. Such variables should be @@ -1078,15 +1179,6 @@ def test_inspect_keeps_globals_from_inspected_module(self): } self._run_repl_globals_test(expectations, as_module=True) - def test_dumb_terminal_exits_cleanly(self): - env = os.environ.copy() - env.update({"TERM": "dumb"}) - output, exit_code = self.run_repl("exit()\n", env=env) - self.assertEqual(exit_code, 0) - self.assertIn("warning: can't use pyrepl", output) - self.assertNotIn("Exception", output) - self.assertNotIn("Traceback", output) - @force_not_colorized def test_python_basic_repl(self): env = os.environ.copy() @@ -1209,80 +1301,6 @@ def test_proper_tracebacklimit(self): self.assertIn("in x3", output) self.assertIn("in ", output) - def run_repl( - self, - repl_input: str | list[str], - env: dict | None = None, - *, - cmdline_args: list[str] | None = None, - cwd: str | None = None, - ) -> tuple[str, int]: - temp_dir = None - if cwd is None: - temp_dir = tempfile.TemporaryDirectory(ignore_cleanup_errors=True) - cwd = temp_dir.name - try: - return self._run_repl( - repl_input, env=env, cmdline_args=cmdline_args, cwd=cwd - ) - finally: - if temp_dir is not None: - temp_dir.cleanup() - - def _run_repl( - self, - repl_input: str | list[str], - *, - env: dict | None, - cmdline_args: list[str] | None, - cwd: str, - ) -> tuple[str, int]: - assert pty - master_fd, slave_fd = pty.openpty() - cmd = [sys.executable, "-i", "-u"] - if env is None: - cmd.append("-I") - elif "PYTHON_HISTORY" not in env: - env["PYTHON_HISTORY"] = os.path.join(cwd, ".regrtest_history") - if cmdline_args is not None: - cmd.extend(cmdline_args) - process = subprocess.Popen( - cmd, - stdin=slave_fd, - stdout=slave_fd, - stderr=slave_fd, - cwd=cwd, - text=True, - close_fds=True, - env=env if env else os.environ, - ) - os.close(slave_fd) - if isinstance(repl_input, list): - repl_input = "\n".join(repl_input) + "\n" - os.write(master_fd, repl_input.encode("utf-8")) - - output = [] - while select.select([master_fd], [], [], SHORT_TIMEOUT)[0]: - try: - data = os.read(master_fd, 1024).decode("utf-8") - if not data: - break - except OSError: - break - output.append(data) - else: - os.close(master_fd) - process.kill() - self.fail(f"Timeout while waiting for output, got: {''.join(output)}") - - os.close(master_fd) - try: - exit_code = process.wait(timeout=SHORT_TIMEOUT) - except subprocess.TimeoutExpired: - process.kill() - exit_code = process.wait() - return "".join(output), exit_code - def test_readline_history_file(self): # skip, if readline module is not available readline = import_module('readline') @@ -1305,3 +1323,7 @@ def test_readline_history_file(self): output, exit_code = self.run_repl("exit\n", env=env) self.assertEqual(exit_code, 0) self.assertNotIn("\\040", pathlib.Path(hfile.name).read_text()) + + def test_keyboard_interrupt_after_isearch(self): + output, exit_code = self.run_repl(["\x12", "\x03", "exit"]) + self.assertEqual(exit_code, 0) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst new file mode 100644 index 00000000000000..b5f423f3ff1c96 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-09-23-15-23-14.gh-issue-123856.yrgJ9m.rst @@ -0,0 +1,2 @@ +Fix PyREPL failure when a keyboard interrupt is triggered after using a +history search From 198756b0f653e9f487076df2c6f943e374def6cb Mon Sep 17 00:00:00 2001 From: Ken Jin Date: Thu, 26 Sep 2024 02:41:07 +0800 Subject: [PATCH 38/39] gh-117376: Fix off-by-ones in conversion functions (GH-124301) Fix off-by-ones in conversion function --- Python/bytecodes.c | 4 ++-- Python/executor_cases.c.h | 4 ++-- Python/generated_cases.c.h | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bf8f6af83fc56e..0fd396f1319e78 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3928,7 +3928,7 @@ dummy_func( PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); @@ -4009,7 +4009,7 @@ dummy_func( (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { DECREF_INPUTS(); ERROR_IF(true, error); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 7285acec0bacaf..7a9c6ab89c38cc 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4596,7 +4596,7 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null[0]); @@ -4713,7 +4713,7 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null[0]); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 58792a2101ab28..1201fe82efb919 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -2252,7 +2252,7 @@ PyCFunctionFast cfunc = (PyCFunctionFast)(void(*)(void))meth->ml_meth; int nargs = total_args - 1; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null[0]); @@ -2333,7 +2333,7 @@ int nargs = total_args - 1; PyCFunctionFastWithKeywords cfunc = (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - STACKREFS_TO_PYOBJECTS(args, nargs, args_o); + STACKREFS_TO_PYOBJECTS(args, total_args, args_o); if (CONVERSION_FAILED(args_o)) { PyStackRef_CLOSE(callable); PyStackRef_CLOSE(self_or_null[0]); From 6318ffcba21f8fc155f5558237ab03aa45f0e174 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 25 Sep 2024 20:06:54 +0100 Subject: [PATCH 39/39] Doc: Use ``major.minor`` for documentation distribution archive filenames (#124489) --- Doc/tools/extensions/patchlevel.py | 2 +- Doc/tools/templates/download.html | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py index f2df6db47a2227..53ea1bf47b8fd3 100644 --- a/Doc/tools/extensions/patchlevel.py +++ b/Doc/tools/extensions/patchlevel.py @@ -74,4 +74,4 @@ def get_version_info(): if __name__ == "__main__": - print(format_version_info(get_header_version_info())[1]) + print(format_version_info(get_header_version_info())[0]) diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index b4217908cc63c9..c978e61b16a49e 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -1,13 +1,15 @@ {% extends "layout.html" %} {% set title = _('Download') %} {% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} + {% set dl_base = pathto('archives', resource=True) %} + {% set dl_version = version %} {% else %} {# The link below returns HTTP 404 until the first related alpha release. This is expected; use daily documentation builds for CPython development. #} - {% set dlbase = 'https://www.python.org/ftp/python/doc/' + release %} + {% set dl_base = 'https://www.python.org/ftp/python/doc/' + release %} + {% set dl_version = release %} {% endif %} {% block body %} @@ -26,27 +28,27 @@

{% trans %}Download Python {{ release }} Documentation{% endtrans %}

{% trans %}PDF{% endtrans %} - {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="17" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}HTML{% endtrans %} - {% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="13" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="8" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}Plain text{% endtrans %} - {% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="4" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="3" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}Texinfo{% endtrans %} - {% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %} - {% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="9" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="7" %}Download (ca. {{ download_size }} MiB){% endtrans %} {% trans %}EPUB{% endtrans %} - {% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %} + {% trans download_size="6" %}Download (ca. {{ download_size }} MiB){% endtrans %}