From a192547dfe7c4f184cc8b579c3eff2f61f642483 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 23 May 2024 18:01:37 +0200 Subject: [PATCH 01/43] gh-117142: Slightly hacky fix for memory leak of StgInfo (GH-119424) Add a funciton that inlines PyObject_GetTypeData and skips type-checking, so it doesn't need access to the CType_Type object. This will break if the memory layout changes, but should be an acceptable solution to enable ctypes in subinterpreters in Python 3.13. Mark _ctypes as safe for multiple interpreters Co-authored-by: neonene <53406459+neonene@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 70 +++++++++++++++++---------------------- Modules/_ctypes/ctypes.h | 28 ++++++++-------- 2 files changed, 45 insertions(+), 53 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 574cb8014493c4..6c1e5f58b95657 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -454,20 +454,17 @@ class _ctypes.CType_Type "PyObject *" "clinic_state()->CType_Type" static int CType_Type_traverse(PyObject *self, visitproc visit, void *arg) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - Py_VISIT(info->proto); - Py_VISIT(info->argtypes); - Py_VISIT(info->converters); - Py_VISIT(info->restype); - Py_VISIT(info->checker); - Py_VISIT(info->module); - } + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + Py_VISIT(info->proto); + Py_VISIT(info->argtypes); + Py_VISIT(info->converters); + Py_VISIT(info->restype); + Py_VISIT(info->checker); + Py_VISIT(info->module); } Py_VISIT(Py_TYPE(self)); return PyType_Type.tp_traverse(self, visit, arg); @@ -488,15 +485,12 @@ ctype_clear_stginfo(StgInfo *info) static int CType_Type_clear(PyObject *self) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - ctype_clear_stginfo(info); - } + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + ctype_clear_stginfo(info); } return PyType_Type.tp_clear(self); } @@ -504,22 +498,20 @@ CType_Type_clear(PyObject *self) static void CType_Type_dealloc(PyObject *self) { - ctypes_state *st = get_module_state_by_def_final(Py_TYPE(self)); - if (st && st->PyCType_Type) { - StgInfo *info; - if (PyStgInfo_FromType(st, self, &info) < 0) { - PyErr_WriteUnraisable(self); - } - if (info) { - PyMem_Free(info->ffi_type_pointer.elements); - info->ffi_type_pointer.elements = NULL; - PyMem_Free(info->format); - info->format = NULL; - PyMem_Free(info->shape); - info->shape = NULL; - ctype_clear_stginfo(info); - } + StgInfo *info = _PyStgInfo_FromType_NoState(self); + if (!info) { + PyErr_WriteUnraisable(self); + } + if (info) { + PyMem_Free(info->ffi_type_pointer.elements); + info->ffi_type_pointer.elements = NULL; + PyMem_Free(info->format); + info->format = NULL; + PyMem_Free(info->shape); + info->shape = NULL; + ctype_clear_stginfo(info); } + PyTypeObject *tp = Py_TYPE(self); PyType_Type.tp_dealloc(self); Py_DECREF(tp); @@ -5947,7 +5939,7 @@ module_free(void *module) static PyModuleDef_Slot module_slots[] = { {Py_mod_exec, _ctypes_mod_exec}, - {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, {0, NULL} }; diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 7784020e9af45a..423120f3460113 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -101,20 +101,6 @@ get_module_state_by_def(PyTypeObject *cls) return get_module_state(mod); } -static inline ctypes_state * -get_module_state_by_def_final(PyTypeObject *cls) -{ - if (cls->tp_mro == NULL) { - return NULL; - } - PyObject *mod = PyType_GetModuleByDef(cls, &_ctypesmodule); - if (mod == NULL) { - PyErr_Clear(); - return NULL; - } - return get_module_state(mod); -} - extern PyType_Spec carg_spec; extern PyType_Spec cfield_spec; @@ -502,6 +488,20 @@ PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) return _stginfo_from_type(state, Py_TYPE(obj), result); } +/* A variant of PyStgInfo_FromType that doesn't need the state, + * so it can be called from finalization functions when the module + * state is torn down. Does no checks; cannot fail. + * This inlines the current implementation PyObject_GetTypeData, + * so it might break in the future. + */ +static inline StgInfo * +_PyStgInfo_FromType_NoState(PyObject *type) +{ + size_t type_basicsize =_Py_SIZE_ROUND_UP(PyType_Type.tp_basicsize, + ALIGNOF_MAX_ALIGN_T); + return (StgInfo *)((char *)type + type_basicsize); +} + // Initialize StgInfo on a newly created type static inline StgInfo * PyStgInfo_Init(ctypes_state *state, PyTypeObject *type) From 6e012ced6cc07a7502278e1849c5618d1ab54a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Thu, 23 May 2024 13:28:31 -0400 Subject: [PATCH 02/43] gh-119469: Fix _pyrepl reference leaks (#119470) --- Lib/test/test_pyrepl/test_interact.py | 6 +++- Lib/test/test_pyrepl/test_unix_console.py | 36 ++++++++++++++++------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index b3dc07c063f90e..10e34045bcf92d 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -27,9 +27,11 @@ def bar(self): a """) console = InteractiveColoredConsole(namespace, filename="") + f = io.StringIO() with ( patch.object(InteractiveColoredConsole, "showsyntaxerror") as showsyntaxerror, patch.object(InteractiveColoredConsole, "runsource", wraps=console.runsource) as runsource, + contextlib.redirect_stdout(f), ): more = console.push(code, filename="", _symbol="single") # type: ignore[call-arg] self.assertFalse(more) @@ -71,7 +73,9 @@ def test_runsource_compiles_and_runs_code(self): def test_runsource_returns_false_for_successful_compilation(self): console = InteractiveColoredConsole() source = "print('Hello, world!')" - result = console.runsource(source) + f = io.StringIO() + with contextlib.redirect_stdout(f): + result = console.runsource(source) self.assertFalse(result) @force_not_colorized diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index cec3ae033325ac..e1faa00caafc27 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -112,17 +112,18 @@ class TestConsole(TestCase): def test_simple_addition(self, _os_write): code = "12+34" events = code_to_events(code) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, b"1") _os_write.assert_any_call(ANY, b"2") _os_write.assert_any_call(ANY, b"+") _os_write.assert_any_call(ANY, b"3") _os_write.assert_any_call(ANY, b"4") + con.restore() def test_wrap(self, _os_write): code = "12+34" events = code_to_events(code) - _, _ = handle_events_narrow_unix_console(events) + _, con = handle_events_narrow_unix_console(events) _os_write.assert_any_call(ANY, b"1") _os_write.assert_any_call(ANY, b"2") _os_write.assert_any_call(ANY, b"+") @@ -130,6 +131,8 @@ def test_wrap(self, _os_write): _os_write.assert_any_call(ANY, b"\\") _os_write.assert_any_call(ANY, b"\n") _os_write.assert_any_call(ANY, b"4") + con.restore() + def test_cursor_left(self, _os_write): code = "1" @@ -137,8 +140,9 @@ def test_cursor_left(self, _os_write): code_to_events(code), [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") + con.restore() def test_cursor_left_right(self, _os_write): code = "1" @@ -149,9 +153,10 @@ def test_cursor_left_right(self, _os_write): Event(evt="key", data="right", raw=bytearray(b"\x1bOC")), ], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuf"] + b":1") + con.restore() def test_cursor_up(self, _os_write): code = "1\n2+3" @@ -159,8 +164,9 @@ def test_cursor_up(self, _os_write): code_to_events(code), [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") + con.restore() def test_cursor_up_down(self, _os_write): code = "1\n2+3" @@ -171,9 +177,10 @@ def test_cursor_up_down(self, _os_write): Event(evt="key", data="down", raw=bytearray(b"\x1bOB")), ], ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cuu"] + b":1") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cud"] + b":1") + con.restore() def test_cursor_back_write(self, _os_write): events = itertools.chain( @@ -181,10 +188,11 @@ def test_cursor_back_write(self, _os_write): [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], code_to_events("2"), ) - _, _ = handle_events_unix_console(events) + _, con = handle_events_unix_console(events) _os_write.assert_any_call(ANY, b"1") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["cub"] + b":1") _os_write.assert_any_call(ANY, b"2") + con.restore() def test_multiline_function_move_up_short_terminal(self, _os_write): # fmt: off @@ -201,8 +209,9 @@ def test_multiline_function_move_up_short_terminal(self, _os_write): Event(evt="scroll", data=None), ], ) - _, _ = handle_events_short_unix_console(events) + _, con = handle_events_short_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") + con.restore() def test_multiline_function_move_up_down_short_terminal(self, _os_write): # fmt: off @@ -221,9 +230,10 @@ def test_multiline_function_move_up_down_short_terminal(self, _os_write): Event(evt="scroll", data=None), ], ) - _, _ = handle_events_short_unix_console(events) + _, con = handle_events_short_unix_console(events) _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ri"] + b":") _os_write.assert_any_call(ANY, TERM_CAPABILITIES["ind"] + b":") + con.restore() def test_resize_bigger_on_multiline_function(self, _os_write): # fmt: off @@ -246,7 +256,7 @@ def same_console(events): console.get_event = MagicMock(side_effect=events) return console - _, _ = handle_all_events( + _, con = handle_all_events( [Event(evt="resize", data=None)], prepare_reader=same_reader, prepare_console=same_console, @@ -258,6 +268,8 @@ def same_console(events): call(ANY, b"def f():"), ] ) + console.restore() + con.restore() def test_resize_smaller_on_multiline_function(self, _os_write): # fmt: off @@ -280,7 +292,7 @@ def same_console(events): console.get_event = MagicMock(side_effect=events) return console - _, _ = handle_all_events( + _, con = handle_all_events( [Event(evt="resize", data=None)], prepare_reader=same_reader, prepare_console=same_console, @@ -292,3 +304,5 @@ def same_console(events): call(ANY, b" foo"), ] ) + console.restore() + con.restore() From b30d30c747df2bf9f1614df8e76db2ffdb24fcd8 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 23 May 2024 15:15:52 -0400 Subject: [PATCH 03/43] gh-117398: Statically Allocate the Datetime C-API (GH-119472) --- ...-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst | 3 + Modules/_datetimemodule.c | 118 ++++++++++++------ Tools/c-analyzer/cpython/globals-to-fix.tsv | 3 + 3 files changed, 89 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst b/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst new file mode 100644 index 00000000000000..ac595f1b7fc84c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-11-52-36.gh-issue-117398.2FG1Mk.rst @@ -0,0 +1,3 @@ +Objects in the datetime C-API are now all statically allocated, which means +better memory safety, especially when the module is reloaded. This should be +transparent to users. diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9a66f0358179b5..3c6d270b8d1331 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1178,6 +1178,8 @@ new_time_subclass_fold_ex(int hour, int minute, int second, int usecond, return t; } +static PyDateTime_Delta * look_up_delta(int, int, int, PyTypeObject *); + /* Create a timedelta instance. Normalize the members iff normalize is * true. Passing false is a speed optimization, if you know for sure * that seconds and microseconds are already in their proper ranges. In any @@ -1198,6 +1200,12 @@ new_delta_ex(int days, int seconds, int microseconds, int normalize, if (check_delta_day_range(days) < 0) return NULL; + self = look_up_delta(days, seconds, microseconds, type); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_Delta *) (type->tp_alloc(type, 0)); if (self != NULL) { self->hashcode = -1; @@ -1219,6 +1227,8 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; +static PyDateTime_TimeZone * look_up_timezone(PyObject *offset, PyObject *name); + /* Create new timezone instance checking offset range. This function does not check the name argument. Caller must assure that offset is a timedelta instance and name is either NULL @@ -1234,6 +1244,12 @@ create_timezone(PyObject *offset, PyObject *name) assert(PyDelta_Check(offset)); assert(name == NULL || PyUnicode_Check(name)); + self = look_up_timezone(offset, name); + if (self != NULL) { + return (PyObject *)self; + } + assert(!PyErr_Occurred()); + self = (PyDateTime_TimeZone *)(type->tp_alloc(type, 0)); if (self == NULL) { return NULL; @@ -2892,6 +2908,25 @@ static PyTypeObject PyDateTime_DeltaType = { 0, /* tp_free */ }; +// XXX Can we make this const? +static PyDateTime_Delta zero_delta = { + PyObject_HEAD_INIT(&PyDateTime_DeltaType) + /* Letting this be set lazily is a benign race. */ + .hashcode = -1, +}; + +static PyDateTime_Delta * +look_up_delta(int days, int seconds, int microseconds, PyTypeObject *type) +{ + if (days == 0 && seconds == 0 && microseconds == 0 + && type == zero_delta.ob_base.ob_type) + { + return &zero_delta; + } + return NULL; +} + + /* * PyDateTime_Date implementation. */ @@ -4184,6 +4219,23 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_new, /* tp_new */ }; +// XXX Can we make this const? +static PyDateTime_TimeZone utc_timezone = { + PyObject_HEAD_INIT(&PyDateTime_TimeZoneType) + .offset = (PyObject *)&zero_delta, + .name = NULL, +}; + +static PyDateTime_TimeZone * +look_up_timezone(PyObject *offset, PyObject *name) +{ + if (offset == utc_timezone.offset && name == NULL) { + return &utc_timezone; + } + return NULL; +} + + /* * PyDateTime_Time implementation. */ @@ -6719,45 +6771,42 @@ static PyMethodDef module_methods[] = { {NULL, NULL} }; + +/* The C-API is process-global. This violates interpreter isolation + * due to the objects stored here. Thus each of those objects must + * be managed carefully. */ +// XXX Can we make this const? +static PyDateTime_CAPI capi = { + /* The classes must be readied before used here. + * That will happen the first time the module is loaded. + * They aren't safe to be shared between interpreters, + * but that's okay as long as the module is single-phase init. */ + .DateType = &PyDateTime_DateType, + .DateTimeType = &PyDateTime_DateTimeType, + .TimeType = &PyDateTime_TimeType, + .DeltaType = &PyDateTime_DeltaType, + .TZInfoType = &PyDateTime_TZInfoType, + + .TimeZone_UTC = (PyObject *)&utc_timezone, + + .Date_FromDate = new_date_ex, + .DateTime_FromDateAndTime = new_datetime_ex, + .Time_FromTime = new_time_ex, + .Delta_FromDelta = new_delta_ex, + .TimeZone_FromTimeZone = new_timezone, + .DateTime_FromTimestamp = datetime_fromtimestamp, + .Date_FromTimestamp = datetime_date_fromtimestamp_capi, + .DateTime_FromDateAndTimeAndFold = new_datetime_ex2, + .Time_FromTimeAndFold = new_time_ex2, +}; + /* Get a new C API by calling this function. * Clients get at C API via PyDateTime_IMPORT, defined in datetime.h. */ static inline PyDateTime_CAPI * get_datetime_capi(void) { - datetime_state *st = get_datetime_state(); - - PyDateTime_CAPI *capi = PyMem_Malloc(sizeof(PyDateTime_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->DateType = st->date_type; - capi->DateTimeType = st->datetime_type; - capi->TimeType = st->time_type; - capi->DeltaType = st->delta_type; - capi->TZInfoType = st->tzinfo_type; - capi->Date_FromDate = new_date_ex; - capi->DateTime_FromDateAndTime = new_datetime_ex; - capi->Time_FromTime = new_time_ex; - capi->Delta_FromDelta = new_delta_ex; - capi->TimeZone_FromTimeZone = new_timezone; - capi->DateTime_FromTimestamp = datetime_fromtimestamp; - capi->Date_FromTimestamp = datetime_date_fromtimestamp_capi; - capi->DateTime_FromDateAndTimeAndFold = new_datetime_ex2; - capi->Time_FromTimeAndFold = new_time_ex2; - // Make sure this function is called after utc has - // been initialized. - assert(st->utc != NULL); - capi->TimeZone_UTC = st->utc; // borrowed ref - return capi; -} - -static void -datetime_destructor(PyObject *op) -{ - void *ptr = PyCapsule_GetPointer(op, PyDateTime_CAPSULE_NAME); - PyMem_Free(ptr); + return &capi; } static int @@ -6955,8 +7004,7 @@ _datetime_exec(PyObject *module) if (capi == NULL) { goto error; } - PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, - datetime_destructor); + PyObject *capsule = PyCapsule_New(capi, PyDateTime_CAPSULE_NAME, NULL); if (capsule == NULL) { PyMem_Free(capi); goto error; diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 8b6fe94e3afc52..711ae343a8d876 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -304,6 +304,9 @@ Python/crossinterp_exceptions.h - PyExc_InterpreterNotFoundError - ##----------------------- ## singletons +Modules/_datetimemodule.c - zero_delta - +Modules/_datetimemodule.c - utc_timezone - +Modules/_datetimemodule.c - capi - Objects/boolobject.c - _Py_FalseStruct - Objects/boolobject.c - _Py_TrueStruct - Objects/dictobject.c - empty_keys_struct - From be1dfccdf2c5c7671b8a549e969b8cf7d60d9936 Mon Sep 17 00:00:00 2001 From: Brett Simmers Date: Thu, 23 May 2024 16:59:35 -0400 Subject: [PATCH 04/43] gh-118727: Don't drop the GIL in `drop_gil()` unless the current thread holds it (#118745) `drop_gil()` assumes that its caller is attached, which means that the current thread holds the GIL if and only if the GIL is enabled, and the enabled-state of the GIL won't change. This isn't true, though, because `detach_thread()` calls `_PyEval_ReleaseLock()` after detaching and `_PyThreadState_DeleteCurrent()` calls it after removing the current thread from consideration for stop-the-world requests (effectively detaching it). Fix this by remembering whether or not a thread acquired the GIL when it last attached, in `PyThreadState._status.holds_gil`, and check this in `drop_gil()` instead of `gil->enabled`. This fixes a crash in `test_multiprocessing_pool_circular_import()`, so I've reenabled it. --- Include/cpython/pystate.h | 4 +- Include/internal/pycore_ceval.h | 7 +- .../test_importlib/test_threaded_import.py | 5 +- Python/ceval_gil.c | 96 +++++++++++-------- Python/pystate.c | 11 +-- 5 files changed, 68 insertions(+), 55 deletions(-) diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 2df9ecd6d52084..ed3ee090ae53db 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -83,6 +83,8 @@ struct _ts { unsigned int bound_gilstate:1; /* Currently in use (maybe holds the GIL). */ unsigned int active:1; + /* Currently holds the GIL. */ + unsigned int holds_gil:1; /* various stages of finalization */ unsigned int finalizing:1; @@ -90,7 +92,7 @@ struct _ts { unsigned int finalized:1; /* padding to align to 4 bytes */ - unsigned int :24; + unsigned int :23; } _status; #ifdef Py_BUILD_CORE # define _PyThreadState_WHENCE_NOTSET -1 diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 48ad0678995904..bd3ba1225f2597 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -131,11 +131,10 @@ extern int _PyEval_ThreadsInitialized(void); extern void _PyEval_InitGIL(PyThreadState *tstate, int own_gil); extern void _PyEval_FiniGIL(PyInterpreterState *interp); -// Acquire the GIL and return 1. In free-threaded builds, this function may -// return 0 to indicate that the GIL was disabled and therefore not acquired. -extern int _PyEval_AcquireLock(PyThreadState *tstate); +extern void _PyEval_AcquireLock(PyThreadState *tstate); -extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *); +extern void _PyEval_ReleaseLock(PyInterpreterState *, PyThreadState *, + int final_release); #ifdef Py_GIL_DISABLED // Returns 0 or 1 if the GIL for the given thread's interpreter is disabled or diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 3477112927a0b6..9af1e4d505c66e 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -17,7 +17,7 @@ from test.support import verbose from test.support.import_helper import forget, mock_register_at_fork from test.support.os_helper import (TESTFN, unlink, rmtree) -from test.support import script_helper, threading_helper, requires_gil_enabled +from test.support import script_helper, threading_helper threading_helper.requires_working_threading(module=True) @@ -248,9 +248,6 @@ def test_concurrent_futures_circular_import(self): 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - # gh-118727 and gh-118729: pool_in_threads.py may crash in free-threaded - # builds, which can hang the Tsan test so temporarily skip it for now. - @requires_gil_enabled("gh-118727: test may crash in free-threaded builds") def test_multiprocessing_pool_circular_import(self): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 7a54c185303cd1..5617504a495686 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -205,32 +205,39 @@ static void recreate_gil(struct _gil_runtime_state *gil) } #endif -static void -drop_gil_impl(struct _gil_runtime_state *gil) +static inline void +drop_gil_impl(PyThreadState *tstate, struct _gil_runtime_state *gil) { MUTEX_LOCK(gil->mutex); _Py_ANNOTATE_RWLOCK_RELEASED(&gil->locked, /*is_write=*/1); _Py_atomic_store_int_relaxed(&gil->locked, 0); + if (tstate != NULL) { + tstate->_status.holds_gil = 0; + } COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); } static void -drop_gil(PyInterpreterState *interp, PyThreadState *tstate) +drop_gil(PyInterpreterState *interp, PyThreadState *tstate, int final_release) { struct _ceval_state *ceval = &interp->ceval; - /* If tstate is NULL, the caller is indicating that we're releasing + /* If final_release is true, the caller is indicating that we're releasing the GIL for the last time in this thread. This is particularly relevant when the current thread state is finalizing or its interpreter is finalizing (either may be in an inconsistent state). In that case the current thread will definitely never try to acquire the GIL again. */ // XXX It may be more correct to check tstate->_status.finalizing. - // XXX assert(tstate == NULL || !tstate->_status.cleared); + // XXX assert(final_release || !tstate->_status.cleared); + assert(final_release || tstate != NULL); struct _gil_runtime_state *gil = ceval->gil; #ifdef Py_GIL_DISABLED - if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { + // Check if we have the GIL before dropping it. tstate will be NULL if + // take_gil() detected that this thread has been destroyed, in which case + // we know we have the GIL. + if (tstate != NULL && !tstate->_status.holds_gil) { return; } #endif @@ -238,26 +245,23 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) Py_FatalError("drop_gil: GIL is not locked"); } - /* tstate is allowed to be NULL (early interpreter init) */ - if (tstate != NULL) { + if (!final_release) { /* Sub-interpreter support: threads might have been switched under our feet using PyThreadState_Swap(). Fix the GIL last holder variable so that our heuristics work. */ _Py_atomic_store_ptr_relaxed(&gil->last_holder, tstate); } - drop_gil_impl(gil); + drop_gil_impl(tstate, gil); #ifdef FORCE_SWITCHING - /* We check tstate first in case we might be releasing the GIL for - the last time in this thread. In that case there's a possible - race with tstate->interp getting deleted after gil->mutex is - unlocked and before the following code runs, leading to a crash. - We can use (tstate == NULL) to indicate the thread is done with - the GIL, and that's the only time we might delete the - interpreter, so checking tstate first prevents the crash. - See https://github.com/python/cpython/issues/104341. */ - if (tstate != NULL && + /* We might be releasing the GIL for the last time in this thread. In that + case there's a possible race with tstate->interp getting deleted after + gil->mutex is unlocked and before the following code runs, leading to a + crash. We can use final_release to indicate the thread is done with the + GIL, and that's the only time we might delete the interpreter. See + https://github.com/python/cpython/issues/104341. */ + if (!final_release && _Py_eval_breaker_bit_is_set(tstate, _PY_GIL_DROP_REQUEST_BIT)) { MUTEX_LOCK(gil->switch_mutex); /* Not switched yet => wait */ @@ -284,7 +288,7 @@ drop_gil(PyInterpreterState *interp, PyThreadState *tstate) tstate must be non-NULL. Returns 1 if the GIL was acquired, or 0 if not. */ -static int +static void take_gil(PyThreadState *tstate) { int err = errno; @@ -309,7 +313,7 @@ take_gil(PyThreadState *tstate) struct _gil_runtime_state *gil = interp->ceval.gil; #ifdef Py_GIL_DISABLED if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { - return 0; + return; } #endif @@ -358,10 +362,10 @@ take_gil(PyThreadState *tstate) if (!_Py_atomic_load_int_relaxed(&gil->enabled)) { // Another thread disabled the GIL between our check above and // now. Don't take the GIL, signal any other waiting threads, and - // return 0. + // return. COND_SIGNAL(gil->cond); MUTEX_UNLOCK(gil->mutex); - return 0; + return; } #endif @@ -393,20 +397,21 @@ take_gil(PyThreadState *tstate) in take_gil() while the main thread called wait_for_thread_shutdown() from Py_Finalize(). */ MUTEX_UNLOCK(gil->mutex); - /* Passing NULL to drop_gil() indicates that this thread is about to - terminate and will never hold the GIL again. */ - drop_gil(interp, NULL); + /* tstate could be a dangling pointer, so don't pass it to + drop_gil(). */ + drop_gil(interp, NULL, 1); PyThread_exit_thread(); } assert(_PyThreadState_CheckConsistency(tstate)); + tstate->_status.holds_gil = 1; _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); update_eval_breaker_for_thread(interp, tstate); MUTEX_UNLOCK(gil->mutex); errno = err; - return 1; + return; } void _PyEval_SetSwitchInterval(unsigned long microseconds) @@ -451,10 +456,17 @@ PyEval_ThreadsInitialized(void) static inline int current_thread_holds_gil(struct _gil_runtime_state *gil, PyThreadState *tstate) { - if (((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) != tstate) { - return 0; - } - return _Py_atomic_load_int_relaxed(&gil->locked); + int holds_gil = tstate->_status.holds_gil; + + // holds_gil is the source of truth; check that last_holder and gil->locked + // are consistent with it. + int locked = _Py_atomic_load_int_relaxed(&gil->locked); + int is_last_holder = + ((PyThreadState*)_Py_atomic_load_ptr_relaxed(&gil->last_holder)) == tstate; + assert(!holds_gil || locked); + assert(!holds_gil || is_last_holder); + + return holds_gil; } #endif @@ -563,23 +575,24 @@ PyEval_ReleaseLock(void) /* This function must succeed when the current thread state is NULL. We therefore avoid PyThreadState_Get() which dumps a fatal error in debug mode. */ - drop_gil(tstate->interp, tstate); + drop_gil(tstate->interp, tstate, 0); } -int +void _PyEval_AcquireLock(PyThreadState *tstate) { _Py_EnsureTstateNotNULL(tstate); - return take_gil(tstate); + take_gil(tstate); } void -_PyEval_ReleaseLock(PyInterpreterState *interp, PyThreadState *tstate) +_PyEval_ReleaseLock(PyInterpreterState *interp, + PyThreadState *tstate, + int final_release) { - /* If tstate is NULL then we do not expect the current thread - to acquire the GIL ever again. */ - assert(tstate == NULL || tstate->interp == interp); - drop_gil(interp, tstate); + assert(tstate != NULL); + assert(tstate->interp == interp); + drop_gil(interp, tstate, final_release); } void @@ -1136,7 +1149,12 @@ _PyEval_DisableGIL(PyThreadState *tstate) // // Drop the GIL, which will wake up any threads waiting in take_gil() // and let them resume execution without the GIL. - drop_gil_impl(gil); + drop_gil_impl(tstate, gil); + + // If another thread asked us to drop the GIL, they should be + // free-threading by now. Remove any such request so we have a clean + // slate if/when the GIL is enabled again. + _Py_unset_eval_breaker_bit(tstate, _PY_GIL_DROP_REQUEST_BIT); return 1; } return 0; diff --git a/Python/pystate.c b/Python/pystate.c index 0832b37c278c76..1ea1ad982a0ec9 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1843,7 +1843,7 @@ _PyThreadState_DeleteCurrent(PyThreadState *tstate) #endif current_fast_clear(tstate->interp->runtime); tstate_delete_common(tstate); - _PyEval_ReleaseLock(tstate->interp, NULL); + _PyEval_ReleaseLock(tstate->interp, tstate, 1); free_threadstate((_PyThreadStateImpl *)tstate); } @@ -2068,7 +2068,7 @@ _PyThreadState_Attach(PyThreadState *tstate) while (1) { - int acquired_gil = _PyEval_AcquireLock(tstate); + _PyEval_AcquireLock(tstate); // XXX assert(tstate_is_alive(tstate)); current_fast_set(&_PyRuntime, tstate); @@ -2079,20 +2079,17 @@ _PyThreadState_Attach(PyThreadState *tstate) } #ifdef Py_GIL_DISABLED - if (_PyEval_IsGILEnabled(tstate) != acquired_gil) { + if (_PyEval_IsGILEnabled(tstate) && !tstate->_status.holds_gil) { // The GIL was enabled between our call to _PyEval_AcquireLock() // and when we attached (the GIL can't go from enabled to disabled // here because only a thread holding the GIL can disable // it). Detach and try again. - assert(!acquired_gil); tstate_set_detached(tstate, _Py_THREAD_DETACHED); tstate_deactivate(tstate); current_fast_clear(&_PyRuntime); continue; } _Py_qsbr_attach(((_PyThreadStateImpl *)tstate)->qsbr); -#else - (void)acquired_gil; #endif break; } @@ -2123,7 +2120,7 @@ detach_thread(PyThreadState *tstate, int detached_state) tstate_deactivate(tstate); tstate_set_detached(tstate, detached_state); current_fast_clear(&_PyRuntime); - _PyEval_ReleaseLock(tstate->interp, tstate); + _PyEval_ReleaseLock(tstate->interp, tstate, 0); } void From e94dbe4ed83460f18bd72563c5f09f6cdc71f604 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 23 May 2024 23:26:09 +0200 Subject: [PATCH 05/43] gh-119461: Fix ThreadedVSOCKSocketStreamTest (#119465) Add socket.VMADDR_CID_LOCAL constant. Fix ThreadedVSOCKSocketStreamTest: if get_cid() returns the host address or the "any" address, use the local communication address (loopback): VMADDR_CID_LOCAL. On Linux 6.9, apparently, the /dev/vsock device is now available but get_cid() returns VMADDR_CID_ANY (-1). --- Lib/test/test_socket.py | 10 ++++++---- .../2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst | 1 + Modules/socketmodule.c | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 0c4b3bb2ad4d81..ce0f64b43ed49f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -160,8 +160,8 @@ def _have_socket_qipcrtr(): def _have_socket_vsock(): """Check whether AF_VSOCK sockets are supported on this host.""" - ret = get_cid() is not None - return ret + cid = get_cid() + return (cid is not None) def _have_socket_bluetooth(): @@ -520,8 +520,6 @@ def clientTearDown(self): @unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, - "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -543,6 +541,9 @@ def clientSetUp(self): self.cli = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.cli.close) cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + # gh-119461: Use the local communication address (loopback) + cid = socket.VMADDR_CID_LOCAL self.cli.connect((cid, VSOCKPORT)) def testStream(self): @@ -2515,6 +2516,7 @@ def testVSOCKConstants(self): socket.SO_VM_SOCKETS_BUFFER_MAX_SIZE socket.VMADDR_CID_ANY socket.VMADDR_PORT_ANY + socket.VMADDR_CID_LOCAL socket.VMADDR_CID_HOST socket.VM_SOCKETS_INVALID_VERSION socket.IOCTL_VM_SOCKETS_GET_LOCAL_CID diff --git a/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst b/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst new file mode 100644 index 00000000000000..48e18f42b5556a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-23-15-48-17.gh-issue-119461.82KqUW.rst @@ -0,0 +1 @@ +Add ``socket.VMADDR_CID_LOCAL`` constant. Patch by Victor Stinner. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index daec560ddfcac7..cb7dc25e23fb3d 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -7596,6 +7596,7 @@ socket_exec(PyObject *m) ADD_INT_CONST(m, "SO_VM_SOCKETS_BUFFER_MAX_SIZE", 2); ADD_INT_CONST(m, "VMADDR_CID_ANY", 0xffffffff); ADD_INT_CONST(m, "VMADDR_PORT_ANY", 0xffffffff); + ADD_INT_CONST(m, "VMADDR_CID_LOCAL", 1); ADD_INT_CONST(m, "VMADDR_CID_HOST", 2); ADD_INT_CONST(m, "VM_SOCKETS_INVALID_VERSION", 0xffffffff); ADD_INT_CONST(m, "IOCTL_VM_SOCKETS_GET_LOCAL_CID", _IO(7, 0xb9)); From ffa24aab107b5bc3c6ad31a6a245c226bf24b208 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 24 May 2024 00:11:45 +0200 Subject: [PATCH 06/43] Clarify base64.a85encode docs: *wrapcols* doesn't count the newline (GH-119409) --- Doc/library/base64.rst | 2 +- Lib/base64.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index cec9a6cef4bf7d..834ab2536e6c14 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -193,7 +193,7 @@ The modern interface provides: *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be - at most this many characters long. + at most this many characters long, excluding the trailing newline. *pad* controls whether the input is padded to a multiple of 4 before encoding. Note that the ``btoa`` implementation always pads. diff --git a/Lib/base64.py b/Lib/base64.py index 25164d1a1df4fc..5a7e790a193380 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -332,7 +332,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): wrapcol controls whether the output should have newline (b'\\n') characters added to it. If this is non-zero, each output line will be at most this - many characters long. + many characters long, excluding the trailing newline. pad controls whether the input is padded to a multiple of 4 before encoding. Note that the btoa implementation always pads. From 0867bce45768454ee31bee95ca33fdc2c9d8b0fa Mon Sep 17 00:00:00 2001 From: Carlos Meza Date: Thu, 23 May 2024 19:04:12 -0700 Subject: [PATCH 07/43] gh-119317: findall instead of traverse for docutils nodes (#119319) --- Doc/tools/extensions/glossary_search.py | 4 ++-- Doc/tools/extensions/pyspecific.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/tools/extensions/glossary_search.py b/Doc/tools/extensions/glossary_search.py index 232782093926f6..7c93b1e4990603 100644 --- a/Doc/tools/extensions/glossary_search.py +++ b/Doc/tools/extensions/glossary_search.py @@ -25,8 +25,8 @@ def process_glossary_nodes(app, doctree, fromdocname): terms = {} - for node in doctree.traverse(glossary): - for glossary_item in node.traverse(definition_list_item): + for node in doctree.findall(glossary): + for glossary_item in node.findall(definition_list_item): term = glossary_item[0].astext().lower() definition = glossary_item[1] diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 44db77af5d24d3..8b592d4b4adcea 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -604,7 +604,7 @@ def parse_monitoring_event(env, sig, signode): def process_audit_events(app, doctree, fromdocname): - for node in doctree.traverse(audit_event_list): + for node in doctree.findall(audit_event_list): break else: return @@ -663,7 +663,7 @@ def process_audit_events(app, doctree, fromdocname): body += row - for node in doctree.traverse(audit_event_list): + for node in doctree.findall(audit_event_list): node.replace_self(table) From b48a3dbff4d70e72797e67b46276564fc63ddb89 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 23 May 2024 23:13:41 -0400 Subject: [PATCH 08/43] GH-113464: Run the JIT interpreter before any other JIT CI (GH-119466) --- .github/workflows/jit.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index b7938a177c856f..8c760a81d52662 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -26,8 +26,22 @@ concurrency: cancel-in-progress: true jobs: + interpreter: + name: Interpreter (Debug) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + - name: Build tier two interpreter + run: | + ./configure --enable-experimental-jit=interpreter --with-pydebug + make all --jobs 4 + - name: Test tier two interpreter + run: | + ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 jit: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + needs: interpreter runs-on: ${{ matrix.runner }} timeout-minutes: 90 strategy: @@ -153,6 +167,7 @@ jobs: jit-with-disabled-gil: name: Free-Threaded (Debug) + needs: interpreter runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 92fab3356f4c61d4c73606e4fae705c6d8f6213b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 24 May 2024 14:31:40 +0200 Subject: [PATCH 09/43] gh-69214: Fix fcntl.ioctl() request type (#119498) Use an 'unsigned long' instead of an 'unsigned int' for the request parameter of fcntl.ioctl() to support requests larger than UINT_MAX. --- .../2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst | 3 +++ Modules/clinic/fcntlmodule.c.h | 11 ++++++----- Modules/fcntlmodule.c | 6 +++--- 3 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst diff --git a/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst b/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst new file mode 100644 index 00000000000000..8c3a36c9f56475 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-11-47-08.gh-issue-69214.Grl6zF.rst @@ -0,0 +1,3 @@ +Fix ``fcntl.ioctl()`` *request* parameter: use an ``unsigned long`` instead of +an ``unsigned int`` for the *request* parameter of :func:`fcntl.ioctl` to +support requests larger than ``UINT_MAX``. Patch by Victor Stinner. diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index d4846ddf8df7e4..53b139e09afdf1 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -96,7 +96,7 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg); static PyObject * @@ -104,7 +104,7 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int fd; - unsigned int code; + unsigned long code; PyObject *ob_arg = NULL; int mutate_arg = 1; @@ -120,10 +120,11 @@ fcntl_ioctl(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (fd < 0) { goto exit; } - code = (unsigned int)PyLong_AsUnsignedLongMask(args[1]); - if (code == (unsigned int)-1 && PyErr_Occurred()) { + if (!PyLong_Check(args[1])) { + PyErr_Format(PyExc_TypeError, "ioctl() argument 2 must be int, not %T", args[1]); goto exit; } + code = PyLong_AsUnsignedLongMask(args[1]); if (nargs < 3) { goto skip_optional; } @@ -263,4 +264,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=26793691ab1c75ba input=a9049054013a1b77]*/ +/*[clinic end generated code: output=45a56f53fd17ff3c input=a9049054013a1b77]*/ diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index b6eeec2c66f6e5..873bdf2ac0657a 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -112,7 +112,7 @@ fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) fcntl.ioctl fd: fildes - request as code: unsigned_int(bitwise=True) + request as code: unsigned_long(bitwise=True) arg as ob_arg: object(c_default='NULL') = 0 mutate_flag as mutate_arg: bool = True / @@ -148,9 +148,9 @@ code. [clinic start generated code]*/ static PyObject * -fcntl_ioctl_impl(PyObject *module, int fd, unsigned int code, +fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *ob_arg, int mutate_arg) -/*[clinic end generated code: output=7f7f5840c65991be input=967b4a4cbeceb0a8]*/ +/*[clinic end generated code: output=3d8eb6828666cea1 input=cee70f6a27311e58]*/ { #define IOCTL_BUFSZ 1024 /* We use the unsigned non-checked 'I' format for the 'code' parameter From bf5b6467f8cc06759f3396ab1a8ad64fe7d1db2e Mon Sep 17 00:00:00 2001 From: Alyssa Coghlan Date: Sat, 25 May 2024 00:29:19 +1000 Subject: [PATCH 10/43] GH-119496: accept UTF-8 BOM in .pth files (GH-119503) `Out-File -Encoding utf8` and similar commands in Windows Powershell 5.1 emit UTF-8 with a BOM marker, which the regular `utf-8` codec decodes incorrectly. `utf-8-sig` accepts a BOM, but also works correctly without one. This change also makes .pth files match the way Python source files are handled. Co-authored-by: Inada Naoki --- Lib/site.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/site.py b/Lib/site.py index f1a6d9cf66fdc3..7eace190f5ab21 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -185,7 +185,9 @@ def addpackage(sitedir, name, known_paths): return try: - pth_content = pth_content.decode() + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") except UnicodeDecodeError: # Fallback to locale encoding for backward compatibility. # We will deprecate this fallback in the future. From f0ed1863bd7a0b9d021fb59e156663a7ec553f0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Germ=C3=A1n=20M=C3=A9ndez=20Bravo?= Date: Fri, 24 May 2024 10:39:06 -0700 Subject: [PATCH 11/43] gh-112075: Fix dict thread safety issues (#119288) Fix dict thread safety issues --- Objects/dictobject.c | 66 +++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 6e1c3b93fd391b..a1ee32b7099f91 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -154,6 +154,11 @@ ASSERT_DICT_LOCKED(PyObject *op) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); } #define ASSERT_DICT_LOCKED(op) ASSERT_DICT_LOCKED(_Py_CAST(PyObject*, op)) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) \ + if (!_PyInterpreterState_GET()->stoptheworld.world_stopped) { \ + ASSERT_DICT_LOCKED(op); \ + } + #define IS_DICT_SHARED(mp) _PyObject_GC_IS_SHARED(mp) #define SET_DICT_SHARED(mp) _PyObject_GC_SET_SHARED(mp) #define LOAD_INDEX(keys, size, idx) _Py_atomic_load_int##size##_relaxed(&((const int##size##_t*)keys->dk_indices)[idx]); @@ -221,6 +226,7 @@ static inline void split_keys_entry_added(PyDictKeysObject *keys) #else /* Py_GIL_DISABLED */ #define ASSERT_DICT_LOCKED(op) +#define ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op) #define LOCK_KEYS(keys) #define UNLOCK_KEYS(keys) #define ASSERT_KEYS_LOCKED(keys) @@ -473,7 +479,7 @@ dictkeys_decref(PyInterpreterState *interp, PyDictKeysObject *dk, bool use_qsbr) if (FT_ATOMIC_LOAD_SSIZE_RELAXED(dk->dk_refcnt) == _Py_IMMORTAL_REFCNT) { return; } - assert(dk->dk_refcnt > 0); + assert(FT_ATOMIC_LOAD_SSIZE(dk->dk_refcnt) > 0); #ifdef Py_REF_DEBUG _Py_DecRefTotal(_PyThreadState_GET()); #endif @@ -670,6 +676,8 @@ dump_entries(PyDictKeysObject *dk) int _PyDict_CheckConsistency(PyObject *op, int check_content) { + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) @@ -1580,6 +1588,8 @@ _PyDict_MaybeUntrack(PyObject *op) PyObject *value; Py_ssize_t i, numentries; + ASSERT_WORLD_STOPPED_OR_DICT_LOCKED(op); + if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op)) return; @@ -1722,13 +1732,14 @@ static void insert_split_value(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, PyObject *value, Py_ssize_t ix) { assert(PyUnicode_CheckExact(key)); + ASSERT_DICT_LOCKED(mp); MAINTAIN_TRACKING(mp, key, value); PyObject *old_value = mp->ma_values->values[ix]; if (old_value == NULL) { uint64_t new_version = _PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value); STORE_SPLIT_VALUE(mp, ix, Py_NewRef(value)); _PyDictValues_AddToInsertionOrder(mp->ma_values, ix); - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; } else { @@ -1792,7 +1803,7 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, goto Fail; } mp->ma_version_tag = new_version; - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); ASSERT_CONSISTENT(mp); return 0; } @@ -1861,7 +1872,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, ep->me_hash = hash; STORE_VALUE(ep, value); } - FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, FT_ATOMIC_LOAD_SSIZE_RELAXED(mp->ma_used) + 1); + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; newkeys->dk_usable--; newkeys->dk_nentries++; @@ -1870,11 +1881,7 @@ insert_to_emptydict(PyInterpreterState *interp, PyDictObject *mp, // the case where we're inserting from the non-owner thread. We don't use // set_keys here because the transition from empty to non-empty is safe // as the empty keys will never be freed. -#ifdef Py_GIL_DISABLED - _Py_atomic_store_ptr_release(&mp->ma_keys, newkeys); -#else - mp->ma_keys = newkeys; -#endif + FT_ATOMIC_STORE_PTR_RELEASE(mp->ma_keys, newkeys); return 0; } @@ -2580,7 +2587,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix, Py_ssize_t hashpos = lookdict_index(mp->ma_keys, hash, ix); assert(hashpos >= 0); - FT_ATOMIC_STORE_SSIZE_RELAXED(mp->ma_used, FT_ATOMIC_LOAD_SSIZE(mp->ma_used) - 1); + STORE_USED(mp, mp->ma_used - 1); mp->ma_version_tag = new_version; if (_PyDict_HasSplitTable(mp)) { assert(old_value == mp->ma_values->values[ix]); @@ -2752,7 +2759,7 @@ clear_lock_held(PyObject *op) // We don't inc ref empty keys because they're immortal ensure_shared_on_resize(mp); mp->ma_version_tag = new_version; - mp->ma_used = 0; + STORE_USED(mp, 0); if (oldvalues == NULL) { set_keys(mp, Py_EMPTY_KEYS); assert(oldkeys->dk_refcnt == 1); @@ -3191,6 +3198,8 @@ dict_repr_lock_held(PyObject *self) _PyUnicodeWriter writer; int first; + ASSERT_DICT_LOCKED(mp); + i = Py_ReprEnter((PyObject *)mp); if (i != 0) { return i > 0 ? PyUnicode_FromString("{...}") : NULL; @@ -3279,8 +3288,7 @@ dict_repr(PyObject *self) static Py_ssize_t dict_length(PyObject *self) { - PyDictObject *mp = (PyDictObject *)self; - return _Py_atomic_load_ssize_relaxed(&mp->ma_used); + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)self)->ma_used); } static PyObject * @@ -3672,6 +3680,9 @@ PyDict_MergeFromSeq2(PyObject *d, PyObject *seq2, int override) static int dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *other, int override) { + ASSERT_DICT_LOCKED(mp); + ASSERT_DICT_LOCKED(other); + if (other == mp || other->ma_used == 0) /* a.update(a) or a.update({}); nothing to do */ return 0; @@ -3699,7 +3710,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe ensure_shared_on_resize(mp); dictkeys_decref(interp, mp->ma_keys, IS_DICT_SHARED(mp)); mp->ma_keys = keys; - mp->ma_used = other->ma_used; + STORE_USED(mp, other->ma_used); mp->ma_version_tag = new_version; ASSERT_CONSISTENT(mp); @@ -4034,7 +4045,7 @@ PyDict_Size(PyObject *mp) PyErr_BadInternalCall(); return -1; } - return ((PyDictObject *)mp)->ma_used; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)mp)->ma_used); } /* Return 1 if dicts equal, 0 if not, -1 if error. @@ -4291,7 +4302,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } MAINTAIN_TRACKING(mp, key, value); - mp->ma_used++; + STORE_USED(mp, mp->ma_used + 1); mp->ma_version_tag = new_version; assert(mp->ma_keys->dk_usable >= 0); ASSERT_CONSISTENT(mp); @@ -4413,6 +4424,8 @@ dict_popitem_impl(PyDictObject *self) uint64_t new_version; PyInterpreterState *interp = _PyInterpreterState_GET(); + ASSERT_DICT_LOCKED(self); + /* Allocate the result tuple before checking the size. Believe it * or not, this allocation could trigger a garbage collection which * could empty the dict, so if we checked the size first and that @@ -4952,19 +4965,21 @@ typedef struct { static PyObject * dictiter_new(PyDictObject *dict, PyTypeObject *itertype) { + Py_ssize_t used; dictiterobject *di; di = PyObject_GC_New(dictiterobject, itertype); if (di == NULL) { return NULL; } di->di_dict = (PyDictObject*)Py_NewRef(dict); - di->di_used = dict->ma_used; - di->len = dict->ma_used; + used = FT_ATOMIC_LOAD_SSIZE_RELAXED(dict->ma_used); + di->di_used = used; + di->len = used; if (itertype == &PyDictRevIterKey_Type || itertype == &PyDictRevIterItem_Type || itertype == &PyDictRevIterValue_Type) { if (_PyDict_HasSplitTable(dict)) { - di->di_pos = dict->ma_used - 1; + di->di_pos = used - 1; } else { di->di_pos = load_keys_nentries(dict) - 1; @@ -5013,8 +5028,8 @@ dictiter_len(PyObject *self, PyObject *Py_UNUSED(ignored)) { dictiterobject *di = (dictiterobject *)self; Py_ssize_t len = 0; - if (di->di_dict != NULL && di->di_used == di->di_dict->ma_used) - len = di->len; + if (di->di_dict != NULL && di->di_used == FT_ATOMIC_LOAD_SSIZE_RELAXED(di->di_dict->ma_used)) + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(di->len); return PyLong_FromSize_t(len); } @@ -5297,6 +5312,7 @@ dictiter_iternextitem_lock_held(PyDictObject *d, PyObject *self, Py_ssize_t i; assert (PyDict_Check(d)); + ASSERT_DICT_LOCKED(d); if (di->di_used != d->ma_used) { PyErr_SetString(PyExc_RuntimeError, @@ -5811,7 +5827,7 @@ dictview_len(PyObject *self) _PyDictViewObject *dv = (_PyDictViewObject *)self; Py_ssize_t len = 0; if (dv->dv_dict != NULL) - len = dv->dv_dict->ma_used; + len = FT_ATOMIC_LOAD_SSIZE_RELAXED(dv->dv_dict->ma_used); return len; } @@ -6820,7 +6836,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, _PyDictValues_AddToInsertionOrder(values, ix); if (dict) { assert(dict->ma_values == values); - dict->ma_used++; + STORE_USED(dict, dict->ma_used + 1); } } else { @@ -6828,7 +6844,7 @@ store_instance_attr_lock_held(PyObject *obj, PyDictValues *values, delete_index_from_values(values, ix); if (dict) { assert(dict->ma_values == values); - dict->ma_used--; + STORE_USED(dict, dict->ma_used - 1); } } Py_DECREF(old_value); @@ -7039,7 +7055,7 @@ _PyObject_IsInstanceDictEmpty(PyObject *obj) if (dict == NULL) { return 1; } - return ((PyDictObject *)dict)->ma_used == 0; + return FT_ATOMIC_LOAD_SSIZE_RELAXED(((PyDictObject *)dict)->ma_used) == 0; } int From 96b392df303b2cfaea823afcb462c0b455704ce8 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Fri, 24 May 2024 20:04:17 +0200 Subject: [PATCH 12/43] gh-118263: Add additional arguments to path_t (Argument Clinic type) in posixmodule (GH-118355) --- Lib/ntpath.py | 42 +- Lib/posixpath.py | 41 +- Lib/test/test_ntpath.py | 4 + Lib/test/test_posixpath.py | 13 +- ...-04-28-19-51-00.gh-issue-118263.Gaap3S.rst | 1 + Modules/clinic/posixmodule.c.h | 227 ++++++---- Modules/posixmodule.c | 420 ++++++++++-------- 7 files changed, 389 insertions(+), 359 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 8d972cd1d0eb72..83e2d3b865757c 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -168,19 +168,12 @@ def splitdrive(p): try: - from nt import _path_splitroot_ex + from nt import _path_splitroot_ex as splitroot except ImportError: def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: - - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ + """Split a pathname into drive, root and tail. + + The tail contains anything after the root.""" p = os.fspath(p) if isinstance(p, bytes): sep = b'\\' @@ -220,23 +213,6 @@ def splitroot(p): else: # Relative path, e.g. Windows return empty, empty, p -else: - def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: - - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - drive, root, tail = _path_splitroot_ex(os.fsdecode(p)) - return os.fsencode(drive), os.fsencode(root), os.fsencode(tail) - return _path_splitroot_ex(p) # Split a path in head (everything up to the last '/') and tail (the @@ -538,7 +514,7 @@ def expandvars(path): # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -577,14 +553,6 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case diff --git a/Lib/posixpath.py b/Lib/posixpath.py index c04c628de55ee2..47b2aa572e5c65 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -135,18 +135,12 @@ def splitdrive(p): try: - from posix import _path_splitroot_ex + from posix import _path_splitroot_ex as splitroot except ImportError: def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: - - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ + """Split a pathname into drive, root and tail. + + The tail contains anything after the root.""" p = os.fspath(p) if isinstance(p, bytes): sep = b'/' @@ -164,23 +158,6 @@ def splitroot(p): # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 return empty, p[:2], p[2:] -else: - def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: - - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - # Optimisation: the drive is always empty - _, root, tail = _path_splitroot_ex(os.fsdecode(p)) - return b'', os.fsencode(root), os.fsencode(tail) - return _path_splitroot_ex(p) # Return the tail (basename) part of a path, same as split(path)[1]. @@ -363,7 +340,7 @@ def expandvars(path): # if it contains symbolic links! try: - from posix import _path_normpath + from posix import _path_normpath as normpath except ImportError: def normpath(path): @@ -394,14 +371,6 @@ def normpath(path): path = initial_slashes + sep.join(comps) return path or dot -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def abspath(path): """Return an absolute path.""" diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 9aa116682f7480..64cbfaaaaa0690 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1129,6 +1129,10 @@ def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) self.assertTrue(os.path.isdir is nt._path_isdir) self.assertFalse(inspect.isfunction(os.path.isdir)) self.assertTrue(os.path.isfile is nt._path_isfile) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 238baed5efa264..57a24e9c70d5e5 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,3 +1,4 @@ +import inspect import os import posixpath import sys @@ -5,7 +6,7 @@ from posixpath import realpath, abspath, dirname, basename from test import test_genericpath from test.support import import_helper -from test.support import os_helper +from test.support import cpython_only, os_helper from test.support.os_helper import FakePath from unittest import mock @@ -283,6 +284,16 @@ def fake_lstat(path): def test_isjunction(self): self.assertFalse(posixpath.isjunction(ABSTFN)) + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst new file mode 100644 index 00000000000000..165a1ba69a811b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-28-19-51-00.gh-issue-118263.Gaap3S.rst @@ -0,0 +1 @@ +Speed up :func:`os.path.splitroot` & :func:`os.path.normpath` with a direct C call. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 5ec5635bae3f41..c7a447b455c594 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -72,7 +72,7 @@ os_stat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("stat", "path", 0, 1); + path_t path = PATH_T_INITIALIZE_P("stat", "path", 0, 0, 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -154,7 +154,7 @@ os_lstat(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("lstat", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lstat", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -250,7 +250,7 @@ os_access(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("access", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("access", "path", 0, 0, 0, 0); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; @@ -409,7 +409,7 @@ os_chdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chdir", "path", 0, PATH_HAVE_FCHDIR); + path_t path = PATH_T_INITIALIZE_P("chdir", "path", 0, 0, 0, PATH_HAVE_FCHDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -560,7 +560,7 @@ os_chmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chmod", "path", 0, PATH_HAVE_FCHMOD); + path_t path = PATH_T_INITIALIZE_P("chmod", "path", 0, 0, 0, PATH_HAVE_FCHMOD); int mode; int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = CHMOD_DEFAULT_FOLLOW_SYMLINKS; @@ -725,7 +725,7 @@ os_lchmod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchmod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchmod", "path", 0, 0, 0, 0); int mode; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -802,7 +802,7 @@ os_chflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("chflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chflags", "path", 0, 0, 0, 0); unsigned long flags; int follow_symlinks = 1; @@ -884,7 +884,7 @@ os_lchflags(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("lchflags", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchflags", "path", 0, 0, 0, 0); unsigned long flags; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -954,7 +954,7 @@ os_chroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("chroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("chroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1190,7 +1190,7 @@ os_chown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("chown", "path", 0, PATH_HAVE_FCHOWN); + path_t path = PATH_T_INITIALIZE_P("chown", "path", 0, 0, 0, PATH_HAVE_FCHOWN); uid_t uid; gid_t gid; int dir_fd = DEFAULT_DIR_FD; @@ -1355,7 +1355,7 @@ os_lchown(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("lchown", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("lchown", "path", 0, 0, 0, 0); uid_t uid; gid_t gid; @@ -1476,8 +1476,8 @@ os_link(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("link", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("link", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("link", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("link", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; @@ -1583,7 +1583,7 @@ os_listdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listdir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("listdir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -1699,7 +1699,7 @@ os_listmounts(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t volume = PATH_T_INITIALIZE("listmounts", "volume", 0, 0); + path_t volume = PATH_T_INITIALIZE_P("listmounts", "volume", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1763,7 +1763,7 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_isdevdrive", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1800,7 +1800,7 @@ static PyObject * os__getfullpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfullpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfullpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1834,7 +1834,7 @@ static PyObject * os__getfinalpathname(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfinalpathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getfinalpathname", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1868,7 +1868,7 @@ static PyObject * os__findfirstfile(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_findfirstfile", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_findfirstfile", "path", 0, 0, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -1928,7 +1928,7 @@ os__getvolumepathname(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getvolumepathname", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getvolumepathname", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -1992,7 +1992,7 @@ os__path_splitroot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_splitroot", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_path_splitroot", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -2024,21 +2024,28 @@ PyDoc_STRVAR(os__path_exists__doc__, {"_path_exists", (PyCFunction)os__path_exists, METH_O, os__path_exists__doc__}, static int -os__path_exists_impl(PyObject *module, PyObject *path); +os__path_exists_impl(PyObject *module, path_t *path); static PyObject * -os__path_exists(PyObject *module, PyObject *path) +os__path_exists(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE_P("_path_exists", "path", 0, 0, 1, 1); int _return_value; - _return_value = os__path_exists_impl(module, path); + if (!path_converter(arg, &path)) { + goto exit; + } + _return_value = os__path_exists_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2056,21 +2063,28 @@ PyDoc_STRVAR(os__path_lexists__doc__, {"_path_lexists", (PyCFunction)os__path_lexists, METH_O, os__path_lexists__doc__}, static int -os__path_lexists_impl(PyObject *module, PyObject *path); +os__path_lexists_impl(PyObject *module, path_t *path); static PyObject * -os__path_lexists(PyObject *module, PyObject *path) +os__path_lexists(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; + path_t path = PATH_T_INITIALIZE_P("_path_lexists", "path", 0, 0, 1, 1); int _return_value; - _return_value = os__path_lexists_impl(module, path); + if (!path_converter(arg, &path)) { + goto exit; + } + _return_value = os__path_lexists_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2088,7 +2102,7 @@ PyDoc_STRVAR(os__path_isdir__doc__, {"_path_isdir", _PyCFunction_CAST(os__path_isdir), METH_FASTCALL|METH_KEYWORDS, os__path_isdir__doc__}, static int -os__path_isdir_impl(PyObject *module, PyObject *path); +os__path_isdir_impl(PyObject *module, path_t *path); static PyObject * os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2120,21 +2134,26 @@ os__path_isdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isdir", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_isdir_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isdir_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2152,7 +2171,7 @@ PyDoc_STRVAR(os__path_isfile__doc__, {"_path_isfile", _PyCFunction_CAST(os__path_isfile), METH_FASTCALL|METH_KEYWORDS, os__path_isfile__doc__}, static int -os__path_isfile_impl(PyObject *module, PyObject *path); +os__path_isfile_impl(PyObject *module, path_t *path); static PyObject * os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2184,21 +2203,26 @@ os__path_isfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isfile", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_isfile_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isfile_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2216,7 +2240,7 @@ PyDoc_STRVAR(os__path_islink__doc__, {"_path_islink", _PyCFunction_CAST(os__path_islink), METH_FASTCALL|METH_KEYWORDS, os__path_islink__doc__}, static int -os__path_islink_impl(PyObject *module, PyObject *path); +os__path_islink_impl(PyObject *module, path_t *path); static PyObject * os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2248,21 +2272,26 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_islink", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_islink_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_islink_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2280,7 +2309,7 @@ PyDoc_STRVAR(os__path_isjunction__doc__, {"_path_isjunction", _PyCFunction_CAST(os__path_isjunction), METH_FASTCALL|METH_KEYWORDS, os__path_isjunction__doc__}, static int -os__path_isjunction_impl(PyObject *module, PyObject *path); +os__path_isjunction_impl(PyObject *module, path_t *path); static PyObject * os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2312,21 +2341,26 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE_P("_path_isjunction", "path", 0, 0, 1, 1); int _return_value; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - _return_value = os__path_isjunction_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + _return_value = os__path_isjunction_impl(module, &path); if ((_return_value == -1) && PyErr_Occurred()) { goto exit; } return_value = PyBool_FromLong((long)_return_value); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2335,13 +2369,16 @@ os__path_isjunction(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P PyDoc_STRVAR(os__path_splitroot_ex__doc__, "_path_splitroot_ex($module, /, path)\n" "--\n" -"\n"); +"\n" +"Split a pathname into drive, root and tail.\n" +"\n" +"The tail contains anything after the root."); #define OS__PATH_SPLITROOT_EX_METHODDEF \ {"_path_splitroot_ex", _PyCFunction_CAST(os__path_splitroot_ex), METH_FASTCALL|METH_KEYWORDS, os__path_splitroot_ex__doc__}, static PyObject * -os__path_splitroot_ex_impl(PyObject *module, PyObject *path); +os__path_splitroot_ex_impl(PyObject *module, path_t *path); static PyObject * os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2373,20 +2410,21 @@ os__path_splitroot_ex(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_splitroot_ex", "path", 0, 1, 1, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("_path_splitroot_ex", "argument 'path'", "str", args[0]); + if (!path_converter(args[0], &path)) { goto exit; } - path = args[0]; - return_value = os__path_splitroot_ex_impl(module, path); + return_value = os__path_splitroot_ex_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2394,13 +2432,13 @@ PyDoc_STRVAR(os__path_normpath__doc__, "_path_normpath($module, /, path)\n" "--\n" "\n" -"Basic path normalization."); +"Normalize path, eliminating double slashes, etc."); #define OS__PATH_NORMPATH_METHODDEF \ {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path); +os__path_normpath_impl(PyObject *module, path_t *path); static PyObject * os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -2432,16 +2470,21 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_normpath", "path", 0, 1, 1, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_normpath_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_normpath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -2496,7 +2539,7 @@ os_mkdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkdir", "path", 0, 0, 0, 0); int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -2757,8 +2800,8 @@ os_rename(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("rename", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("rename", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("rename", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("rename", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2848,8 +2891,8 @@ os_replace(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("replace", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("replace", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("replace", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("replace", "dst", 0, 0, 0, 0); int src_dir_fd = DEFAULT_DIR_FD; int dst_dir_fd = DEFAULT_DIR_FD; @@ -2937,7 +2980,7 @@ os_rmdir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("rmdir", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("rmdir", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3186,7 +3229,7 @@ os_unlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("unlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("unlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3260,7 +3303,7 @@ os_remove(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("remove", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("remove", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -3378,7 +3421,7 @@ os_utime(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("utime", "path", 0, PATH_UTIME_HAVE_FD); + path_t path = PATH_T_INITIALIZE_P("utime", "path", 0, 0, 0, PATH_UTIME_HAVE_FD); PyObject *times = Py_None; PyObject *ns = NULL; int dir_fd = DEFAULT_DIR_FD; @@ -3513,7 +3556,7 @@ static PyObject * os_execv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("execv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("execv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("execv", nargs, 2, 2)) { @@ -3585,7 +3628,7 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k }; #undef KWTUPLE PyObject *argsbuf[3]; - path_t path = PATH_T_INITIALIZE("execve", "path", 0, PATH_HAVE_FEXECVE); + path_t path = PATH_T_INITIALIZE_P("execve", "path", 0, 0, 0, PATH_HAVE_FEXECVE); PyObject *argv; PyObject *env; @@ -3681,7 +3724,7 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawn", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawn", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3831,7 +3874,7 @@ os_posix_spawnp(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #undef KWTUPLE PyObject *argsbuf[10]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("posix_spawnp", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("posix_spawnp", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; PyObject *file_actions = NULL; @@ -3935,7 +3978,7 @@ os_spawnv(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnv", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnv", "path", 0, 0, 0, 0); PyObject *argv; if (!_PyArg_CheckPositional("spawnv", nargs, 3, 3)) { @@ -3989,7 +4032,7 @@ os_spawnve(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; int mode; - path_t path = PATH_T_INITIALIZE("spawnve", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("spawnve", "path", 0, 0, 0, 0); PyObject *argv; PyObject *env; @@ -6165,7 +6208,7 @@ os_readlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("readlink", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("readlink", "path", 0, 0, 0, 0); int dir_fd = DEFAULT_DIR_FD; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); @@ -6249,8 +6292,8 @@ os_symlink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t src = PATH_T_INITIALIZE("symlink", "src", 0, 0); - path_t dst = PATH_T_INITIALIZE("symlink", "dst", 0, 0); + path_t src = PATH_T_INITIALIZE_P("symlink", "src", 0, 0, 0, 0); + path_t dst = PATH_T_INITIALIZE_P("symlink", "dst", 0, 0, 0, 0); int target_is_directory = 0; int dir_fd = DEFAULT_DIR_FD; @@ -6892,7 +6935,7 @@ os_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("open", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("open", "path", 0, 0, 0, 0); int flags; int mode = 511; int dir_fd = DEFAULT_DIR_FD; @@ -8480,7 +8523,7 @@ os_mkfifo(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mkfifo", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mkfifo", "path", 0, 0, 0, 0); int mode = 438; int dir_fd = DEFAULT_DIR_FD; @@ -8580,7 +8623,7 @@ os_mknod(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw #undef KWTUPLE PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t path = PATH_T_INITIALIZE("mknod", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("mknod", "path", 0, 0, 0, 0); int mode = 384; dev_t device = 0; int dir_fd = DEFAULT_DIR_FD; @@ -8834,7 +8877,7 @@ os_truncate(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("truncate", "path", 0, PATH_HAVE_FTRUNCATE); + path_t path = PATH_T_INITIALIZE_P("truncate", "path", 0, 0, 0, PATH_HAVE_FTRUNCATE); Py_off_t length; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -9733,7 +9776,7 @@ os_statvfs(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("statvfs", "path", 0, PATH_HAVE_FSTATVFS); + path_t path = PATH_T_INITIALIZE_P("statvfs", "path", 0, 0, 0, PATH_HAVE_FSTATVFS); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9797,7 +9840,7 @@ os__getdiskusage(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_getdiskusage", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_getdiskusage", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -9911,7 +9954,7 @@ os_pathconf(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject }; #undef KWTUPLE PyObject *argsbuf[2]; - path_t path = PATH_T_INITIALIZE("pathconf", "path", 0, PATH_HAVE_FPATHCONF); + path_t path = PATH_T_INITIALIZE_P("pathconf", "path", 0, 0, 0, PATH_HAVE_FPATHCONF); int name; long _return_value; @@ -10101,10 +10144,10 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - path_t filepath = PATH_T_INITIALIZE("startfile", "filepath", 0, 0); + path_t filepath = PATH_T_INITIALIZE_P("startfile", "filepath", 0, 0, 0, 0); const wchar_t *operation = NULL; const wchar_t *arguments = NULL; - path_t cwd = PATH_T_INITIALIZE("startfile", "cwd", 1, 0); + path_t cwd = PATH_T_INITIALIZE_P("startfile", "cwd", 1, 0, 0, 0); int show_cmd = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 5, 0, argsbuf); @@ -10439,8 +10482,8 @@ os_getxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("getxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("getxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("getxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("getxattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10526,8 +10569,8 @@ os_setxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[5]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; - path_t path = PATH_T_INITIALIZE("setxattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("setxattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("setxattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("setxattr", "attribute", 0, 0, 0, 0); Py_buffer value = {NULL, NULL}; int flags = 0; int follow_symlinks = 1; @@ -10634,8 +10677,8 @@ os_removexattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #undef KWTUPLE PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 2; - path_t path = PATH_T_INITIALIZE("removexattr", "path", 0, 1); - path_t attribute = PATH_T_INITIALIZE("removexattr", "attribute", 0, 0); + path_t path = PATH_T_INITIALIZE_P("removexattr", "path", 0, 0, 0, 1); + path_t attribute = PATH_T_INITIALIZE_P("removexattr", "attribute", 0, 0, 0, 0); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 2, 2, 0, argsbuf); @@ -10720,7 +10763,7 @@ os_listxattr(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #undef KWTUPLE PyObject *argsbuf[2]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("listxattr", "path", 1, 1); + path_t path = PATH_T_INITIALIZE_P("listxattr", "path", 1, 0, 0, 1); int follow_symlinks = 1; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); @@ -11697,7 +11740,7 @@ os_scandir(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - path_t path = PATH_T_INITIALIZE("scandir", "path", 1, PATH_HAVE_FDOPENDIR); + path_t path = PATH_T_INITIALIZE_P("scandir", "path", 1, 0, 0, PATH_HAVE_FDOPENDIR); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); if (!args) { @@ -11909,7 +11952,7 @@ os__add_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nargs, }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_add_dll_directory", "path", 0, 0); + path_t path = PATH_T_INITIALIZE_P("_add_dll_directory", "path", 0, 0, 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { @@ -12752,4 +12795,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=af5074c4ce4b19f1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=300bd1c54dc43765 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 07fec35cb32d90..bb35cfd9cdb138 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1092,16 +1092,15 @@ get_posix_state(PyObject *module) * * path_converter accepts (Unicode) strings and their * subclasses, and bytes and their subclasses. What - * it does with the argument depends on the platform: + * it does with the argument depends on path.make_wide: * - * * On Windows, if we get a (Unicode) string we - * extract the wchar_t * and return it; if we get - * bytes we decode to wchar_t * and return that. + * * If path.make_wide is nonzero, if we get a (Unicode) + * string we extract the wchar_t * and return it; if we + * get bytes we decode to wchar_t * and return that. * - * * On all other platforms, strings are encoded - * to bytes using PyUnicode_FSConverter, then we - * extract the char * from the bytes object and - * return that. + * * If path.make_wide is zero, if we get bytes we extract + * the char_t * and return it; if we get a (Unicode) + * string we encode to char_t * and return that. * * path_converter also optionally accepts signed * integers (representing open file descriptors) instead @@ -1110,6 +1109,15 @@ get_posix_state(PyObject *module) * Input fields: * path.nullable * If nonzero, the path is permitted to be None. + * path.nonstrict + * If nonzero, the path is permitted to contain + * embedded null characters and have any length. + * path.make_wide + * If nonzero, the converter always uses wide, decoding if necessary, else + * it always uses narrow, encoding if necessary. The default value is + * nonzero on Windows, else zero. + * path.suppress_value_error + * If nonzero, raising ValueError is suppressed. * path.allow_fd * If nonzero, the path is permitted to be a file handle * (a signed int) instead of a string. @@ -1125,12 +1133,10 @@ get_posix_state(PyObject *module) * Output fields: * path.wide * Points to the path if it was expressed as Unicode - * and was not encoded. (Only used on Windows.) + * or if it was bytes and decoded to Unicode. * path.narrow * Points to the path if it was expressed as bytes, - * or it was Unicode and was encoded to bytes. (On Windows, - * is a non-zero integer if the path was expressed as bytes. - * The type is deliberately incompatible to prevent misuse.) + * or if it was Unicode and encoded to bytes. * path.fd * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any @@ -1140,6 +1146,9 @@ get_posix_state(PyObject *module) * unspecified, path_converter will never get called. * So if you set allow_fd, you *MUST* initialize path.fd = -1 * yourself! + * path.value_error + * If nonzero, then suppress_value_error was specified and a ValueError + * occurred. * path.length * The length of the path in characters, if specified as * a string. @@ -1172,28 +1181,38 @@ get_posix_state(PyObject *module) * path_cleanup(). However it is safe to do so.) */ typedef struct { + // Input fields const char *function_name; const char *argument_name; int nullable; + int nonstrict; + int make_wide; + int suppress_value_error; int allow_fd; + // Output fields const wchar_t *wide; -#ifdef MS_WINDOWS - BOOL narrow; -#else const char *narrow; -#endif int fd; + int value_error; Py_ssize_t length; PyObject *object; PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ + make_wide, suppress_value_error, allow_fd) \ + {function_name, argument_name, nullable, nonstrict, make_wide, \ + suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} #ifdef MS_WINDOWS -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, FALSE, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 1, \ + suppress_value_error, allow_fd) #else -#define PATH_T_INITIALIZE(function_name, argument_name, nullable, allow_fd) \ - {function_name, argument_name, nullable, allow_fd, NULL, NULL, -1, 0, NULL, NULL} +#define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ + nonstrict, suppress_value_error, allow_fd) \ + PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, 0, \ + suppress_value_error, allow_fd) #endif static void @@ -1214,10 +1233,8 @@ path_converter(PyObject *o, void *p) Py_ssize_t length = 0; int is_index, is_bytes, is_unicode; const char *narrow; -#ifdef MS_WINDOWS PyObject *wo = NULL; wchar_t *wide = NULL; -#endif #define FORMAT_EXCEPTION(exc, fmt) \ PyErr_Format(exc, "%s%s" fmt, \ @@ -1238,11 +1255,7 @@ path_converter(PyObject *o, void *p) if ((o == Py_None) && path->nullable) { path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif path->fd = -1; goto success_exit; } @@ -1286,30 +1299,33 @@ path_converter(PyObject *o, void *p) } if (is_unicode) { + if (path->make_wide) { + wide = PyUnicode_AsWideCharString(o, &length); + if (!wide) { + goto error_exit; + } #ifdef MS_WINDOWS - wide = PyUnicode_AsWideCharString(o, &length); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } - path->wide = wide; - path->narrow = FALSE; - path->fd = -1; - wide = NULL; - goto success_exit; -#else - if (!PyUnicode_FSConverter(o, &bytes)) { + path->wide = wide; + path->narrow = NULL; + path->fd = -1; + wide = NULL; + goto success_exit; + } + bytes = PyUnicode_EncodeFSDefault(o); + if (!bytes) { goto error_exit; } -#endif } else if (is_bytes) { bytes = Py_NewRef(o); @@ -1319,11 +1335,7 @@ path_converter(PyObject *o, void *p) goto error_exit; } path->wide = NULL; -#ifdef MS_WINDOWS - path->narrow = FALSE; -#else path->narrow = NULL; -#endif goto success_exit; } else { @@ -1343,52 +1355,54 @@ path_converter(PyObject *o, void *p) length = PyBytes_GET_SIZE(bytes); narrow = PyBytes_AS_STRING(bytes); - if ((size_t)length != strlen(narrow)) { + if (!path->nonstrict && strlen(narrow) != (size_t)length) { FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); goto error_exit; } -#ifdef MS_WINDOWS - wo = PyUnicode_DecodeFSDefaultAndSize( - narrow, - length - ); - if (!wo) { - goto error_exit; - } + if (path->make_wide) { + wo = PyUnicode_DecodeFSDefaultAndSize(narrow, length); + if (!wo) { + goto error_exit; + } - wide = PyUnicode_AsWideCharString(wo, &length); - Py_DECREF(wo); - if (!wide) { - goto error_exit; - } - if (length > 32767) { - FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); - goto error_exit; - } - if (wcslen(wide) != length) { - FORMAT_EXCEPTION(PyExc_ValueError, "embedded null character in %s"); - goto error_exit; - } - path->wide = wide; - path->narrow = TRUE; - Py_DECREF(bytes); - wide = NULL; -#else - path->wide = NULL; - path->narrow = narrow; - if (bytes == o) { - /* Still a reference owned by path->object, don't have to - worry about path->narrow is used after free. */ + wide = PyUnicode_AsWideCharString(wo, &length); + Py_DECREF(wo); + if (!wide) { + goto error_exit; + } +#ifdef MS_WINDOWS + if (!path->nonstrict && length > 32767) { + FORMAT_EXCEPTION(PyExc_ValueError, "%s too long for Windows"); + goto error_exit; + } +#endif + if (!path->nonstrict && wcslen(wide) != (size_t)length) { + FORMAT_EXCEPTION(PyExc_ValueError, + "embedded null character in %s"); + goto error_exit; + } + path->wide = wide; + path->narrow = NULL; Py_DECREF(bytes); + wide = NULL; } else { - path->cleanup = bytes; + path->wide = NULL; + path->narrow = narrow; + if (bytes == o) { + /* Still a reference owned by path->object, don't have to + worry about path->narrow is used after free. */ + Py_DECREF(bytes); + } + else { + path->cleanup = bytes; + } } -#endif path->fd = -1; success_exit: + path->value_error = 0; path->length = length; path->object = o; return Py_CLEANUP_SUPPORTED; @@ -1396,10 +1410,20 @@ path_converter(PyObject *o, void *p) error_exit: Py_XDECREF(o); Py_XDECREF(bytes); -#ifdef MS_WINDOWS PyMem_Free(wide); -#endif - return 0; + if (!path->suppress_value_error || + !PyErr_ExceptionMatches(PyExc_ValueError)) + { + return 0; + } + PyErr_Clear(); + path->wide = NULL; + path->narrow = NULL; + path->fd = -1; + path->value_error = 1; + path->length = 0; + path->object = NULL; + return Py_CLEANUP_SUPPORTED; } static void @@ -1449,11 +1473,7 @@ follow_symlinks_specified(const char *function_name, int follow_symlinks) static int path_and_dir_fd_invalid(const char *function_name, path_t *path, int dir_fd) { - if (!path->wide && (dir_fd != DEFAULT_DIR_FD) -#ifndef MS_WINDOWS - && !path->narrow -#endif - ) { + if (!path->wide && (dir_fd != DEFAULT_DIR_FD) && !path->narrow) { PyErr_Format(PyExc_ValueError, "%s: can't specify dir_fd without matching path", function_name); @@ -2913,7 +2933,9 @@ class path_t_converter(CConverter): converter = 'path_converter' - def converter_init(self, *, allow_fd=False, nullable=False): + def converter_init(self, *, allow_fd=False, make_wide=None, + nonstrict=False, nullable=False, + suppress_value_error=False): # right now path_t doesn't support default values. # to support a default value, you'll need to override initialize(). if self.default not in (unspecified, None): @@ -2923,6 +2945,9 @@ class path_t_converter(CConverter): raise RuntimeError("Can't specify a c_default to the path_t converter!") self.nullable = nullable + self.nonstrict = nonstrict + self.make_wide = make_wide + self.suppress_value_error = suppress_value_error self.allow_fd = allow_fd def pre_render(self): @@ -2932,11 +2957,24 @@ class path_t_converter(CConverter): return str(int(bool(value))) # add self.py_name here when merging with posixmodule conversion - self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {})'.format( - self.function.name, - self.name, - strify(self.nullable), - strify(self.allow_fd), + if self.make_wide is None: + self.c_default = 'PATH_T_INITIALIZE_P("{}", "{}", {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.suppress_value_error), + strify(self.allow_fd), + ) + else: + self.c_default = 'PATH_T_INITIALIZE("{}", "{}", {}, {}, {}, {}, {})'.format( + self.function.name, + self.name, + strify(self.nullable), + strify(self.nonstrict), + strify(self.make_wide), + strify(self.suppress_value_error), + strify(self.allow_fd), ) def cleanup(self): @@ -3016,7 +3054,7 @@ class sysconf_confname_converter(path_confname_converter): converter="conv_sysconf_confname" [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3338733161aa7879]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=577cb476e5d64960]*/ /*[clinic input] @@ -4285,7 +4323,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) { PyObject *v; HANDLE hFindFile = INVALID_HANDLE_VALUE; - BOOL result; + BOOL result, return_bytes; wchar_t namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ /* only claim to have space for MAX_PATH */ Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; @@ -4297,9 +4335,11 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; len = 1; + return_bytes = 0; } else { po_wchars = path->wide; len = wcslen(path->wide); + return_bytes = PyBytes_Check(path->object); } /* The +5 is so we can append "\\*.*\0" */ wnamebuf = PyMem_New(wchar_t, len + 5); @@ -4334,7 +4374,7 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) wcscmp(wFileData.cFileName, L"..") != 0) { v = PyUnicode_FromWideChar(wFileData.cFileName, wcslen(wFileData.cFileName)); - if (path->narrow && v) { + if (return_bytes && v) { Py_SETREF(v, PyUnicode_EncodeFSDefault(v)); } if (v == NULL) { @@ -4877,7 +4917,7 @@ os__getfullpathname_impl(PyObject *module, path_t *path) if (str == NULL) { return NULL; } - if (path->narrow) { + if (PyBytes_Check(path->object)) { Py_SETREF(str, PyUnicode_EncodeFSDefault(str)); } return str; @@ -4950,7 +4990,7 @@ os__getfinalpathname_impl(PyObject *module, path_t *path) } result = PyUnicode_FromWideChar(target_path, result_length); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -5033,7 +5073,7 @@ os__getvolumepathname_impl(PyObject *module, path_t *path) goto exit; } result = PyUnicode_FromWideChar(mountpath, wcslen(mountpath)); - if (path->narrow) + if (PyBytes_Check(path->object)) Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); exit: @@ -5267,64 +5307,52 @@ _testFileExistsByName(LPCWSTR path, BOOL followLinks) } -static int -_testFileExists(path_t *_path, PyObject *path, BOOL followLinks) +static BOOL +_testFileExists(path_t *path, BOOL followLinks) { BOOL result = FALSE; - if (!path_converter(path, _path)) { - path_cleanup(_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - return FALSE; - } - return -1; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path->fd != -1) { - HANDLE hfile = _Py_get_osfhandle_noraise(_path->fd); + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { if (GetFileType(hfile) != FILE_TYPE_UNKNOWN || !GetLastError()) { result = TRUE; } } } - else if (_path->wide) { - result = _testFileExistsByName(_path->wide, followLinks); + else if (path->wide) { + result = _testFileExistsByName(path->wide, followLinks); } Py_END_ALLOW_THREADS - path_cleanup(_path); return result; } -static int -_testFileType(path_t *_path, PyObject *path, int testedType) +static BOOL +_testFileType(path_t *path, int testedType) { BOOL result = FALSE; - if (!path_converter(path, _path)) { - path_cleanup(_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - return FALSE; - } - return -1; + if (path->value_error) { + return FALSE; } Py_BEGIN_ALLOW_THREADS - if (_path->fd != -1) { - HANDLE hfile = _Py_get_osfhandle_noraise(_path->fd); + if (path->fd != -1) { + HANDLE hfile = _Py_get_osfhandle_noraise(path->fd); if (hfile != INVALID_HANDLE_VALUE) { result = _testFileTypeByHandle(hfile, testedType, TRUE); } } - else if (_path->wide) { - result = _testFileTypeByName(_path->wide, testedType); + else if (path->wide) { + result = _testFileTypeByName(path->wide, testedType); } Py_END_ALLOW_THREADS - path_cleanup(_path); return result; } @@ -5332,7 +5360,7 @@ _testFileType(path_t *_path, PyObject *path, int testedType) /*[clinic input] os._path_exists -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) / Test whether a path exists. Returns False for broken symbolic links. @@ -5340,18 +5368,17 @@ Test whether a path exists. Returns False for broken symbolic links. [clinic start generated code]*/ static int -os__path_exists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=8f784b3abf9f8588 input=2777da15bc4ba5a3]*/ +os__path_exists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=8da13acf666e16ba input=29198507a6082a57]*/ { - path_t _path = PATH_T_INITIALIZE("_path_exists", "path", 0, 1); - return _testFileExists(&_path, path, TRUE); + return _testFileExists(path, TRUE); } /*[clinic input] os._path_lexists -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) / Test whether a path exists. Returns True for broken symbolic links. @@ -5359,83 +5386,78 @@ Test whether a path exists. Returns True for broken symbolic links. [clinic start generated code]*/ static int -os__path_lexists_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=fec4a91cf4ffccf1 input=8843d4d6d4e7c779]*/ +os__path_lexists_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e7240ed5fc45bff3 input=03d9fed8bc6ce96f]*/ { - path_t _path = PATH_T_INITIALIZE("_path_lexists", "path", 0, 1); - return _testFileExists(&_path, path, FALSE); + return _testFileExists(path, FALSE); } /*[clinic input] os._path_isdir -> bool - s as path: object + s as path: path_t(allow_fd=True, suppress_value_error=True) Return true if the pathname refers to an existing directory. [clinic start generated code]*/ static int -os__path_isdir_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=0504fd403f369701 input=2cb54dd97eb970f7]*/ +os__path_isdir_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d5786196f9e2fa7a input=132a3b5301aecf79]*/ { - path_t _path = PATH_T_INITIALIZE("_path_isdir", "s", 0, 1); - return _testFileType(&_path, path, PY_IFDIR); + return _testFileType(path, PY_IFDIR); } /*[clinic input] os._path_isfile -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a regular file [clinic start generated code]*/ static int -os__path_isfile_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b40d620efe5a896f input=54b428a310debaea]*/ +os__path_isfile_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=5c3073bc212b9863 input=4ac1fd350b30a39e]*/ { - path_t _path = PATH_T_INITIALIZE("_path_isfile", "path", 0, 1); - return _testFileType(&_path, path, PY_IFREG); + return _testFileType(path, PY_IFREG); } /*[clinic input] os._path_islink -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a symbolic link [clinic start generated code]*/ static int -os__path_islink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=9d0cf8e4c640dfe6 input=b71fed60b9b2cd73]*/ +os__path_islink_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=30da7bda8296adcc input=7510ce05b547debb]*/ { - path_t _path = PATH_T_INITIALIZE("_path_islink", "path", 0, 1); - return _testFileType(&_path, path, PY_IFLNK); + return _testFileType(path, PY_IFLNK); } /*[clinic input] os._path_isjunction -> bool - path: object + path: path_t(allow_fd=True, suppress_value_error=True) Test whether a path is a junction [clinic start generated code]*/ static int -os__path_isjunction_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=f1d51682a077654d input=103ccedcdb714f11]*/ +os__path_isjunction_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=e1d17a9dd18a9945 input=7dcb8bc4e972fcaf]*/ { - path_t _path = PATH_T_INITIALIZE("_path_isjunction", "path", 0, 1); - return _testFileType(&_path, path, PY_IFMNT); + return _testFileType(path, PY_IFMNT); } #undef PY_IFREG @@ -5451,23 +5473,22 @@ os__path_isjunction_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_splitroot_ex - path: unicode + path: path_t(make_wide=True, nonstrict=True) +Split a pathname into drive, root and tail. + +The tail contains anything after the root. [clinic start generated code]*/ static PyObject * -os__path_splitroot_ex_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=de97403d3dfebc40 input=f1470e12d899f9ac]*/ +os__path_splitroot_ex_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=4b0072b6cdf4b611 input=6eb76e9173412c92]*/ { - Py_ssize_t len, drvsize, rootsize; + Py_ssize_t drvsize, rootsize; PyObject *drv = NULL, *root = NULL, *tail = NULL, *result = NULL; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - goto exit; - } - - _Py_skiproot(buffer, len, &drvsize, &rootsize); + const wchar_t *buffer = path->wide; + _Py_skiproot(buffer, path->length, &drvsize, &rootsize); drv = PyUnicode_FromWideChar(buffer, drvsize); if (drv == NULL) { goto exit; @@ -5477,13 +5498,26 @@ os__path_splitroot_ex_impl(PyObject *module, PyObject *path) goto exit; } tail = PyUnicode_FromWideChar(&buffer[drvsize + rootsize], - len - drvsize - rootsize); + path->length - drvsize - rootsize); if (tail == NULL) { goto exit; } + if (PyBytes_Check(path->object)) { + Py_SETREF(drv, PyUnicode_EncodeFSDefault(drv)); + if (drv == NULL) { + goto exit; + } + Py_SETREF(root, PyUnicode_EncodeFSDefault(root)); + if (root == NULL) { + goto exit; + } + Py_SETREF(tail, PyUnicode_EncodeFSDefault(tail)); + if (tail == NULL) { + goto exit; + } + } result = PyTuple_Pack(3, drv, root, tail); exit: - PyMem_Free(buffer); Py_XDECREF(drv); Py_XDECREF(root); Py_XDECREF(tail); @@ -5494,29 +5528,28 @@ os__path_splitroot_ex_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_normpath - path: object + path: path_t(make_wide=True, nonstrict=True) -Basic path normalization. +Normalize path, eliminating double slashes, etc. [clinic start generated code]*/ static PyObject * -os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +os__path_normpath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=d353e7ed9410c044 input=3d4ac23b06332dcb]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; + PyObject *result; + Py_ssize_t norm_len; + wchar_t *norm_path = _Py_normpath_and_size((wchar_t *)path->wide, + path->length, &norm_len); + if (!norm_len) { + result = PyUnicode_FromOrdinal('.'); } - Py_ssize_t len; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { - return NULL; + else { + result = PyUnicode_FromWideChar(norm_path, norm_len); + } + if (PyBytes_Check(path->object)) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } - Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); - PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); - PyMem_Free(buffer); return result; } @@ -10243,7 +10276,7 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) name[1] = L'\\'; } result = PyUnicode_FromWideChar(name, nameLen); - if (result && path->narrow) { + if (result && PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } } @@ -15864,7 +15897,8 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) entry->name = PyUnicode_FromWideChar(dataW->cFileName, -1); if (!entry->name) goto error; - if (path->narrow) { + int return_bytes = path->wide && PyBytes_Check(path->object); + if (return_bytes) { Py_SETREF(entry->name, PyUnicode_EncodeFSDefault(entry->name)); if (!entry->name) goto error; @@ -15878,7 +15912,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) PyMem_Free(joined_path); if (!entry->path) goto error; - if (path->narrow) { + if (return_bytes) { Py_SETREF(entry->path, PyUnicode_EncodeFSDefault(entry->path)); if (!entry->path) goto error; From 045e195c76f33c77c339284b13f81102e4b9abe2 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Fri, 24 May 2024 22:30:32 +0300 Subject: [PATCH 13/43] Regen ``Doc/requirements-oldest-sphinx.txt`` (#119520) --- Doc/requirements-oldest-sphinx.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/requirements-oldest-sphinx.txt b/Doc/requirements-oldest-sphinx.txt index fe45c91501a56a..3ae65bc944da26 100644 --- a/Doc/requirements-oldest-sphinx.txt +++ b/Doc/requirements-oldest-sphinx.txt @@ -23,7 +23,7 @@ Jinja2==3.1.4 MarkupSafe==2.1.5 packaging==24.0 Pygments==2.18.0 -requests==2.31.0 +requests==2.32.2 snowballstemmer==2.2.0 Sphinx==6.2.1 sphinxcontrib-applehelp==1.0.8 From 81d63362302187e5cb838c9a7cd857181142e530 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Fri, 24 May 2024 20:35:13 +0100 Subject: [PATCH 14/43] GH-119054: Add "Querying file type and status" section to pathlib docs (#119055) Add a dedicated subsection for `Path.stat()`-related methods, specifically `stat()`, `lstat()`, `exists()`, `is_*()`, and `samefile()`. --- Doc/library/pathlib.rst | 338 ++++++++++++++++++++-------------------- 1 file changed, 171 insertions(+), 167 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 27ed0a32e801cc..71e2e5452d1754 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -808,8 +808,8 @@ bugs or failures in your application):: UnsupportedOperation: cannot instantiate 'WindowsPath' on your system -File URIs -^^^^^^^^^ +Parsing and generating URIs +^^^^^^^^^^^^^^^^^^^^^^^^^^^ Concrete path objects can be created from, and represented as, 'file' URIs conforming to :rfc:`8089`. @@ -869,12 +869,8 @@ conforming to :rfc:`8089`. it strictly impure. -Methods -^^^^^^^ - -Concrete paths provide the following methods in addition to pure paths -methods. Some of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist). +Querying file type and status +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. versionchanged:: 3.8 @@ -895,29 +891,6 @@ call fails (for example because the path doesn't exist). status without suppressing exceptions. -.. classmethod:: Path.cwd() - - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: - - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') - - -.. classmethod:: Path.home() - - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct). If the home - directory can't be resolved, :exc:`RuntimeError` is raised. - - :: - - >>> Path.home() - PosixPath('/home/antoine') - - .. versionadded:: 3.5 - - .. method:: Path.stat(*, follow_symlinks=True) Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. @@ -937,25 +910,12 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.10 The *follow_symlinks* parameter was added. -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - :: +.. method:: Path.lstat() - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 + Like :meth:`Path.stat` but, if the path points to a symbolic link, return + the symbolic link's information rather than its target's. - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. .. method:: Path.exists(*, follow_symlinks=True) @@ -980,6 +940,170 @@ call fails (for example because the path doesn't exist). .. versionchanged:: 3.12 The *follow_symlinks* parameter was added. + +.. method:: Path.is_file(*, follow_symlinks=True) + + Return ``True`` if the path points to a regular file. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a regular file. Use :meth:`Path.stat` to + distinguish between these cases. + + This method normally follows symlinks; to exclude symlinks, add the + argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + +.. method:: Path.is_dir(*, follow_symlinks=True) + + Return ``True`` if the path points to a directory. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a directory. Use :meth:`Path.stat` to distinguish + between these cases. + + This method normally follows symlinks; to exclude symlinks to directories, + add the argument ``follow_symlinks=False``. + + .. versionchanged:: 3.13 + The *follow_symlinks* parameter was added. + + +.. method:: Path.is_symlink() + + Return ``True`` if the path points to a symbolic link, even if that symlink + is broken. ``False`` will be returned if the path is invalid, inaccessible + or missing, or if it points to something other than a symbolic link. Use + :meth:`Path.stat` to distinguish between these cases. + + +.. method:: Path.is_junction() + + Return ``True`` if the path points to a junction, and ``False`` for any other + type of file. Currently only Windows supports junctions. + + .. versionadded:: 3.12 + + +.. method:: Path.is_mount() + + Return ``True`` if the path is a :dfn:`mount point`: a point in a + file system where a different file system has been mounted. On POSIX, the + function checks whether *path*'s parent, :file:`path/..`, is on a different + device than *path*, or whether :file:`path/..` and *path* point to the same + i-node on the same device --- this should detect mount points for all Unix + and POSIX variants. On Windows, a mount point is considered to be a drive + letter root (e.g. ``c:\``), a UNC share (e.g. ``\\server\share``), or a + mounted filesystem directory. + + .. versionadded:: 3.7 + + .. versionchanged:: 3.12 + Windows support was added. + +.. method:: Path.is_socket() + + Return ``True`` if the path points to a Unix socket. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a Unix socket. Use :meth:`Path.stat` to + distinguish between these cases. + + +.. method:: Path.is_fifo() + + Return ``True`` if the path points to a FIFO. ``False`` will be returned if + the path is invalid, inaccessible or missing, or if it points to something + other than a FIFO. Use :meth:`Path.stat` to distinguish between these + cases. + + +.. method:: Path.is_block_device() + + Return ``True`` if the path points to a block device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a block device. Use :meth:`Path.stat` to + distinguish between these cases. + + +.. method:: Path.is_char_device() + + Return ``True`` if the path points to a character device. ``False`` will be + returned if the path is invalid, inaccessible or missing, or if it points + to something other than a character device. Use :meth:`Path.stat` to + distinguish between these cases. + + +.. method:: Path.samefile(other_path) + + Return whether this path points to the same file as *other_path*, which + can be either a Path object, or a string. The semantics are similar + to :func:`os.path.samefile` and :func:`os.path.samestat`. + + An :exc:`OSError` can be raised if either file cannot be accessed for some + reason. + + :: + + >>> p = Path('spam') + >>> q = Path('eggs') + >>> p.samefile(q) + False + >>> p.samefile('spam') + True + + .. versionadded:: 3.5 + + +Other methods +^^^^^^^^^^^^^ + +Some of these methods can raise an :exc:`OSError` if a system call fails (for +example because the path doesn't exist). + + +.. classmethod:: Path.cwd() + + Return a new path object representing the current directory (as returned + by :func:`os.getcwd`):: + + >>> Path.cwd() + PosixPath('/home/antoine/pathlib') + + +.. classmethod:: Path.home() + + Return a new path object representing the user's home directory (as + returned by :func:`os.path.expanduser` with ``~`` construct). If the home + directory can't be resolved, :exc:`RuntimeError` is raised. + + :: + + >>> Path.home() + PosixPath('/home/antoine') + + .. versionadded:: 3.5 + + +.. method:: Path.chmod(mode, *, follow_symlinks=True) + + Change the file mode and permissions, like :func:`os.chmod`. + + This method normally follows symlinks. Some Unix flavours support changing + permissions on the symlink itself; on these platforms you may add the + argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. + + :: + + >>> p = Path('setup.py') + >>> p.stat().st_mode + 33277 + >>> p.chmod(0o444) + >>> p.stat().st_mode + 33060 + + .. versionchanged:: 3.10 + The *follow_symlinks* parameter was added. + .. method:: Path.expanduser() Return a new path with expanded ``~`` and ``~user`` constructs, @@ -1076,99 +1200,6 @@ call fails (for example because the path doesn't exist). The *follow_symlinks* parameter was added. -.. method:: Path.is_dir(*, follow_symlinks=True) - - Return ``True`` if the path points to a directory. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a directory. Use :meth:`Path.stat` to distinguish - between these cases. - - This method normally follows symlinks; to exclude symlinks to directories, - add the argument ``follow_symlinks=False``. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - -.. method:: Path.is_file(*, follow_symlinks=True) - - Return ``True`` if the path points to a regular file. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a regular file. Use :meth:`Path.stat` to - distinguish between these cases. - - This method normally follows symlinks; to exclude symlinks, add the - argument ``follow_symlinks=False``. - - .. versionchanged:: 3.13 - The *follow_symlinks* parameter was added. - - -.. method:: Path.is_junction() - - Return ``True`` if the path points to a junction, and ``False`` for any other - type of file. Currently only Windows supports junctions. - - .. versionadded:: 3.12 - - -.. method:: Path.is_mount() - - Return ``True`` if the path is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`path/..`, is on a different - device than *path*, or whether :file:`path/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. On Windows, a mount point is considered to be a drive - letter root (e.g. ``c:\``), a UNC share (e.g. ``\\server\share``), or a - mounted filesystem directory. - - .. versionadded:: 3.7 - - .. versionchanged:: 3.12 - Windows support was added. - - -.. method:: Path.is_symlink() - - Return ``True`` if the path points to a symbolic link, even if that symlink - is broken. ``False`` will be returned if the path is invalid, inaccessible - or missing, or if it points to something other than a symbolic link. Use - :meth:`Path.stat` to distinguish between these cases. - - -.. method:: Path.is_socket() - - Return ``True`` if the path points to a Unix socket. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a Unix socket. Use :meth:`Path.stat` to - distinguish between these cases. - - -.. method:: Path.is_fifo() - - Return ``True`` if the path points to a FIFO. ``False`` will be returned if - the path is invalid, inaccessible or missing, or if it points to something - other than a FIFO. Use :meth:`Path.stat` to distinguish between these - cases. - - -.. method:: Path.is_block_device() - - Return ``True`` if the path points to a block device. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a block device. Use :meth:`Path.stat` to - distinguish between these cases. - - -.. method:: Path.is_char_device() - - Return ``True`` if the path points to a character device. ``False`` will be - returned if the path is invalid, inaccessible or missing, or if it points - to something other than a character device. Use :meth:`Path.stat` to - distinguish between these cases. - - .. method:: Path.iterdir() When the path points to a directory, yield path objects of the directory @@ -1291,12 +1322,6 @@ call fails (for example because the path doesn't exist). symbolic link's mode is changed rather than its target's. -.. method:: Path.lstat() - - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. - - .. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) Create a new directory at this given path. If *mode* is given, it is @@ -1486,27 +1511,6 @@ call fails (for example because the path doesn't exist). Remove this directory. The directory must be empty. -.. method:: Path.samefile(other_path) - - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. - - :: - - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 - - .. method:: Path.symlink_to(target, target_is_directory=False) Make this path a symbolic link pointing to *target*. From 49c3ade4f3ceae2f8fcbe03ebaaad5eddf8de0bf Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 24 May 2024 16:58:24 -0500 Subject: [PATCH 15/43] Misc improvement to the docs for itertools (gh-119529) --- Doc/library/itertools.rst | 110 +++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 6d33748898361d..43432dae1623ce 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -134,7 +134,7 @@ loops that truncate the stream. total = func(total, element) yield total - There are a number of uses for the *func* argument. It can be set to + The *func* argument can be set to :func:`min` for a running minimum, :func:`max` for a running maximum, or :func:`operator.mul` for a running product. Amortization tables can be built by accumulating interest and applying payments: @@ -184,21 +184,14 @@ loops that truncate the stream. >>> unflattened [('roses', 'red'), ('violets', 'blue'), ('sugar', 'sweet')] - >>> for batch in batched('ABCDEFG', 3): - ... print(batch) - ... - ('A', 'B', 'C') - ('D', 'E', 'F') - ('G',) - Roughly equivalent to:: def batched(iterable, n, *, strict=False): # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): + iterable = iter(iterable) + while batch := tuple(islice(iterable, n)): if strict and len(batch) != n: raise ValueError('batched(): incomplete batch') yield batch @@ -237,13 +230,13 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable*. - The combination tuples are emitted in lexicographic ordering according to + The combination tuples are emitted in lexicographic order according to the order of the input *iterable*. So, if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values in each combination. + value. So, if the input elements are unique, there will be no repeated + values within each combination. Roughly equivalent to:: @@ -286,12 +279,12 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - The combination tuples are emitted in lexicographic ordering according to + The combination tuples are emitted in lexicographic order according to the order of the input *iterable*. So, if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations + value. So, if the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: @@ -332,13 +325,13 @@ loops that truncate the stream. .. function:: compress(data, selectors) Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. + have a corresponding element in *selectors* is true. + Stops when either the *data* or *selectors* iterables have been exhausted. Roughly equivalent to:: def compress(data, selectors): # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F - return (d for d, s in zip(data, selectors) if s) + return (datum for datum, selector in zip(data, selectors) if selector) .. versionadded:: 3.1 @@ -392,7 +385,7 @@ loops that truncate the stream. start-up time. Roughly equivalent to:: def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) → 6 4 1 + # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 iterable = iter(iterable) for x in iterable: if not predicate(x): @@ -408,7 +401,7 @@ loops that truncate the stream. that are false. Roughly equivalent to:: def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) → 0 2 4 6 8 + # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 if predicate is None: predicate = bool for x in iterable: @@ -444,36 +437,37 @@ loops that truncate the stream. :func:`groupby` is roughly equivalent to:: - class groupby: + def groupby(iterable, key=None): # [k for k, g in groupby('AAAABBBCCDAABBB')] → A B C D A B # [list(g) for k, g in groupby('AAAABBBCCD')] → AAAA BBB CC D - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - - def __iter__(self): - return self - - def __next__(self): - self.id = object() - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey, self.id)) - - def _grouper(self, tgtkey, id): - while self.id is id and self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: + keyfunc = (lambda x: x) if key is None else key + iterator = iter(iterable) + exhausted = False + + def _grouper(target_key): + nonlocal curr_value, curr_key, exhausted + yield curr_value + for curr_value in iterator: + curr_key = keyfunc(curr_value) + if curr_key != target_key: return - self.currkey = self.keyfunc(self.currvalue) + yield curr_value + exhausted = True + + try: + curr_value = next(iterator) + except StopIteration: + return + curr_key = keyfunc(curr_value) + + while not exhausted: + target_key = curr_key + curr_group = _grouper(target_key) + yield curr_key, curr_group + if curr_key == target_key: + for _ in curr_group: + pass .. function:: islice(iterable, stop) @@ -501,13 +495,15 @@ loops that truncate the stream. # islice('ABCDEFG', 2, 4) → C D # islice('ABCDEFG', 2, None) → C D E F G # islice('ABCDEFG', 0, None, 2) → A C E G + s = slice(*args) start = 0 if s.start is None else s.start stop = s.stop step = 1 if s.step is None else s.step if start < 0 or (stop is not None and stop < 0) or step <= 0: raise ValueError - indices = count() if stop is None else range(max(stop, start)) + + indices = count() if stop is None else range(max(start, stop)) next_i = start for i, element in zip(indices, iterable): if i == next_i: @@ -549,7 +545,7 @@ loops that truncate the stream. the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated + value. So, if the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: @@ -557,14 +553,17 @@ loops that truncate the stream. def permutations(iterable, r=None): # permutations('ABCD', 2) → AB AC AD BA BC BD CA CB CD DA DB DC # permutations(range(3)) → 012 021 102 120 201 210 + pool = tuple(iterable) n = len(pool) r = n if r is None else r if r > n: return + indices = list(range(n)) cycles = list(range(n, n-r, -1)) yield tuple(pool[i] for i in indices[:r]) + while n: for i in reversed(range(r)): cycles[i] -= 1 @@ -580,7 +579,7 @@ loops that truncate the stream. return The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those + :func:`product` filtered to exclude entries with repeated elements (those from the same position in the input pool):: def permutations(iterable, r=None): @@ -674,17 +673,16 @@ loops that truncate the stream. predicate is true. Roughly equivalent to:: def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) → 1 4 + # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 for x in iterable: - if predicate(x): - yield x - else: + if not predicate(x): break + yield x Note, the element that first fails the predicate condition is consumed from the input iterator and there is no way to access it. This could be an issue if an application wants to further consume the - input iterator after takewhile has been run to exhaustion. To work + input iterator after *takewhile* has been run to exhaustion. To work around this problem, consider using `more-iterools before_and_after() `_ instead. @@ -734,10 +732,12 @@ loops that truncate the stream. def zip_longest(*iterables, fillvalue=None): # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- - iterators = [iter(it) for it in iterables] + + iterators = list(map(iter, iterables)) num_active = len(iterators) if not num_active: return + while True: values = [] for i, iterator in enumerate(iterators): From 84be5244de75c92904fb41326c9a69f19051e7ab Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Fri, 24 May 2024 17:32:17 -0700 Subject: [PATCH 16/43] gh-119180: Update the magic number (#119397) PR #119321 added a comment about the magic number bump but did not actually apply the new magic number. --- Lib/importlib/_bootstrap_external.py | 2 +- PC/launcher.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index b3abf380a82b11..68469863e7f774 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -489,7 +489,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3571).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3600).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/PC/launcher.c b/PC/launcher.c index 8e60ab9303cb95..47fafbc3bf6bad 100644 --- a/PC/launcher.c +++ b/PC/launcher.c @@ -1271,6 +1271,7 @@ static PYC_MAGIC magic_values[] = { { 3450, 3499, L"3.11" }, { 3500, 3549, L"3.12" }, { 3550, 3599, L"3.13" }, + { 3600, 3649, L"3.14" }, { 0 } }; From de19694cfbcaa1c85c3a4b7184a24ff21b1c0919 Mon Sep 17 00:00:00 2001 From: Tim Peters Date: Fri, 24 May 2024 22:08:21 -0500 Subject: [PATCH 17/43] gh-119105: Differ.compare is too slow [for degenerate cases] (#119492) ``_fancy_replace()`` is no longer recursive. and a single call does a worst-case linear number of ratio() computations instead of quadratic. This renders toothless a universe of pathological cases. Some inputs may produce different output, but that's rare, and I didn't find a case where the final diff appeared to be of materially worse quality. To the contrary, by refusing to even consider synching on lines "far apart", there was more easy-to-digest locality in the output. --- Lib/difflib.py | 116 +++++++----------- ...-05-24-04-05-37.gh-issue-119105.aDSRFn.rst | 1 + 2 files changed, 46 insertions(+), 71 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst diff --git a/Lib/difflib.py b/Lib/difflib.py index 79b446c2afbdc6..0443963b4fd697 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -908,79 +908,52 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): + abcdefGhijkl ? ^ ^ ^ """ - from operator import ge, gt - # Don't synch up unless the lines have a similarity score of at - # least cutoff; best_ratio tracks the best score seen so far. - # Keep track of all index pairs achieving the best ratio and - # deal with them here. Previously only the smallest pair was - # handled here, and if there are many pairs with the best ratio, - # recursion could grow very deep, and runtime cubic. See: + # Don't synch up unless the lines have a similarity score above + # cutoff. Previously only the smallest pair was handled here, + # and if there are many pairs with the best ratio, recursion + # could grow very deep, and runtime cubic. See: # https://github.com/python/cpython/issues/119105 - best_ratio, cutoff = 0.74, 0.75 + # + # Later, more pathological cases prompted removing recursion + # entirely. + cutoff = 0.74999 cruncher = SequenceMatcher(self.charjunk) - eqi, eqj = None, None # 1st indices of equal lines (if any) - # List of index pairs achieving best_ratio. Strictly increasing - # in both index positions. - max_pairs = [] - maxi = -1 # `i` index of last pair in max_pairs - - # search for the pair that matches best without being identical - # (identical lines must be junk lines, & we don't want to synch - # up on junk -- unless we have to) crqr = cruncher.real_quick_ratio cqr = cruncher.quick_ratio cr = cruncher.ratio + + WINDOW = 10 + best_i = best_j = None + dump_i, dump_j = alo, blo # smallest indices not yet resolved for j in range(blo, bhi): - bj = b[j] - cruncher.set_seq2(bj) - # Find new best, if possible. Else search for the smallest i - # (if any) > maxi that equals the best ratio - search_equal = True - for i in range(alo, ahi): - ai = a[i] - if ai == bj: - if eqi is None: - eqi, eqj = i, j - continue - cruncher.set_seq1(ai) - # computing similarity is expensive, so use the quick - # upper bounds first -- have seen this speed up messy - # compares by a factor of 3. - cmp = ge if search_equal and i > maxi else gt - if (cmp(crqr(), best_ratio) - and cmp(cqr(), best_ratio) - and cmp((ratio := cr()), best_ratio)): - if ratio > best_ratio: - best_ratio = ratio - max_pairs.clear() - else: - assert best_ratio == ratio and search_equal - assert i > maxi - max_pairs.append((i, j)) - maxi = i - search_equal = False - if best_ratio < cutoff: - assert not max_pairs - # no non-identical "pretty close" pair - if eqi is None: - # no identical pair either -- treat it as a straight replace - yield from self._plain_replace(a, alo, ahi, b, blo, bhi) - return - # no close pair, but an identical pair -- synch up on that - max_pairs = [(eqi, eqj)] - else: - # there's a close pair, so forget the identical pair (if any) - assert max_pairs - eqi = None - - last_i, last_j = alo, blo - for this_i, this_j in max_pairs: - # pump out diffs from before the synch point - yield from self._fancy_helper(a, last_i, this_i, - b, last_j, this_j) + cruncher.set_seq2(b[j]) + # Search the corresponding i's within WINDOW for rhe highest + # ratio greater than `cutoff`. + aequiv = alo + (j - blo) + arange = range(max(aequiv - WINDOW, dump_i), + min(aequiv + WINDOW + 1, ahi)) + if not arange: # likely exit if `a` is shorter than `b` + break + best_ratio = cutoff + for i in arange: + cruncher.set_seq1(a[i]) + # Ordering by cheapest to most expensive ratio is very + # valuable, most often getting out early. + if (crqr() > best_ratio + and cqr() > best_ratio + and cr() > best_ratio): + best_i, best_j, best_ratio = i, j, cr() + + if best_i is None: + # found nothing to synch on yet - move to next j + continue + + # pump out straight replace from before this synch pair + yield from self._fancy_helper(a, dump_i, best_i, + b, dump_j, best_j) # do intraline marking on the synch pair - aelt, belt = a[this_i], b[this_j] - if eqi is None: + aelt, belt = a[best_i], b[best_j] + if aelt != belt: # pump out a '-', '?', '+', '?' quad for the synched lines atags = btags = "" cruncher.set_seqs(aelt, belt) @@ -1002,17 +975,18 @@ def _fancy_replace(self, a, alo, ahi, b, blo, bhi): else: # the synch pair is identical yield ' ' + aelt - last_i, last_j = this_i + 1, this_j + 1 + dump_i, dump_j = best_i + 1, best_j + 1 + best_i = best_j = None - # pump out diffs from after the last synch point - yield from self._fancy_helper(a, last_i, ahi, - b, last_j, bhi) + # pump out straight replace from after the last synch pair + yield from self._fancy_helper(a, dump_i, ahi, + b, dump_j, bhi) def _fancy_helper(self, a, alo, ahi, b, blo, bhi): g = [] if alo < ahi: if blo < bhi: - g = self._fancy_replace(a, alo, ahi, b, blo, bhi) + g = self._plain_replace(a, alo, ahi, b, blo, bhi) else: g = self._dump('-', a, alo, ahi) elif blo < bhi: diff --git a/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst b/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst new file mode 100644 index 00000000000000..3205061a68ce7f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-24-04-05-37.gh-issue-119105.aDSRFn.rst @@ -0,0 +1 @@ +``difflib``'s ``DIffer.compare()`` (and so also ``ndiff``) can no longer be provoked into cubic-time behavior, or into unbounded recursion, and should generally be faster in ordinary cases too. Results may change in some cases, although that should be rare. Correctness of diffs is not affected. Some similar lines far apart may be reported as deleting one and adding the other, where before they were displayed on adjacent output lines with markup showing the intraline differences. From 08e65430aafa1047029e6f132a5f748c415bda14 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 25 May 2024 16:21:11 +0300 Subject: [PATCH 18/43] gh-111999: Fix the signature of str.format_map() (#119540) --- Doc/library/stdtypes.rst | 2 +- .../2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst | 1 + Objects/unicodeobject.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c0a3d0b3a2a49e..c8acde8b57dcdb 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1768,7 +1768,7 @@ expression support in the :mod:`re` module). cases. -.. method:: str.format_map(mapping) +.. method:: str.format_map(mapping, /) Similar to ``str.format(**mapping)``, except that ``mapping`` is used directly and not copied to a :class:`dict`. This is useful diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst new file mode 100644 index 00000000000000..4b1ca6ca5b0765 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-13-51-48.gh-issue-111999.L0q1gh.rst @@ -0,0 +1 @@ +Fix the signature of :meth:`str.format_map`. diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index afff37467caf32..048f9a814c30af 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -13490,7 +13490,7 @@ Return a formatted version of the string, using substitutions from args and kwar The substitutions are identified by braces ('{' and '}')."); PyDoc_STRVAR(format_map__doc__, - "format_map($self, /, mapping)\n\ + "format_map($self, mapping, /)\n\ --\n\ \n\ Return a formatted version of the string, using substitutions from mapping.\n\ From a531fd7fdb45d13825cb0c38d97fd38246cf9634 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Sat, 25 May 2024 17:13:17 +0300 Subject: [PATCH 19/43] FAQ: Add reference to Python version numbering scheme (#119225) --- Doc/faq/general.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index ec7c2897594999..eb859c5d5992da 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -122,6 +122,8 @@ available. Consult `the Python Package Index `_ to find packages of interest to you. +.. _faq-version-numbering-scheme: + How does the Python version numbering scheme work? -------------------------------------------------- @@ -183,8 +185,6 @@ information on getting the source code and compiling it. How do I get documentation on Python? ------------------------------------- -.. XXX mention py3k - The standard documentation for the current stable version of Python is available at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. From e3bac04c37f6823cebc74d97feae0e0c25818b31 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 25 May 2024 17:15:54 +0100 Subject: [PATCH 20/43] gh-119548: Add a 'clear' command to the REPL (#119549) --- Lib/_pyrepl/reader.py | 5 +++++ Lib/_pyrepl/simple_interact.py | 7 ++++++- .../2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 81df0c925ee6cb..d2960bbb6121b3 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -238,6 +238,7 @@ class Reader: cxy: tuple[int, int] = field(init=False) lxy: tuple[int, int] = field(init=False) calc_screen: CalcScreen = field(init=False) + scheduled_commands: list[str] = field(default_factory=list) def __post_init__(self) -> None: # Enable the use of `insert` without a `prepare` call - necessary to @@ -557,6 +558,10 @@ def prepare(self) -> None: self.restore() raise + while self.scheduled_commands: + cmd = self.scheduled_commands.pop() + self.do_cmd((cmd, [])) + def last_command_is(self, cls: type) -> bool: if not self.last_command: return False diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index 8ab4dab757685e..975533a425be23 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -57,12 +57,17 @@ def _strip_final_indent(text: str) -> str: return text +def _clear_screen(): + reader = _get_reader() + reader.scheduled_commands.append("clear_screen") + + REPL_COMMANDS = { "exit": _sitebuiltins.Quitter('exit', ''), "quit": _sitebuiltins.Quitter('quit' ,''), "copyright": _sitebuiltins._Printer('copyright', sys.copyright), "help": "help", - "clear": "clear_screen", + "clear": _clear_screen, } class InteractiveColoredConsole(code.InteractiveConsole): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst new file mode 100644 index 00000000000000..0318790d46f0a3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-16-45-27.gh-issue-119548.pqF9Y6.rst @@ -0,0 +1 @@ +Add a ``clear`` command to the REPL. Patch by Pablo Galindo From 6b6c1a904f6d6237a05057727360fe4b80e98d4c Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sat, 25 May 2024 17:21:07 +0100 Subject: [PATCH 21/43] Add codeowners for PYREPL (#119550) --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1f5f7e57dc4859..e955567ec0b0f8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -39,9 +39,11 @@ Python/bytecodes.c @markshannon @gvanrossum Python/optimizer*.c @markshannon @gvanrossum Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner +Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently # dbm From 4b7eb321bc43e41371df86fce47bd999ee51a793 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sat, 25 May 2024 18:08:32 +0100 Subject: [PATCH 22/43] gh-99180: Make `StackSummary.should_show_carets` private (#119554) --- Lib/traceback.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/traceback.py b/Lib/traceback.py index 280d92d04cac9b..6ee1a50ca6804a 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -580,7 +580,7 @@ def format_frame_summary(self, frame_summary, **kwargs): show_carets = False with suppress(Exception): anchors = _extract_caret_anchors_from_line_segment(segment) - show_carets = self.should_show_carets(start_offset, end_offset, all_lines, anchors) + show_carets = self._should_show_carets(start_offset, end_offset, all_lines, anchors) result = [] @@ -694,7 +694,7 @@ def output_line(lineno): return ''.join(row) - def should_show_carets(self, start_offset, end_offset, all_lines, anchors): + def _should_show_carets(self, start_offset, end_offset, all_lines, anchors): with suppress(SyntaxError, ImportError): import ast tree = ast.parse('\n'.join(all_lines)) From 0c5ebe13e9937c446e9947c44f2570737ecca135 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 25 May 2024 15:30:48 -0400 Subject: [PATCH 23/43] gh-119560: Drop an Invalid Assert in PyState_FindModule() (gh-119561) The assertion was added in gh-118532 but was based on the invalid assumption that PyState_FindModule() would only be called with an already-initialized module def. I've added a test to make sure we don't make that assumption again. --- Lib/test/test_import/__init__.py | 7 ++ ...-05-25-12-52-25.gh-issue-119560.wSlm8q.rst | 3 + Modules/_testsinglephase.c | 91 ++++++++++++++++++- Python/import.c | 3 +- 4 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 64282d0f2d0bcf..11eaae5e47e97a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2887,6 +2887,13 @@ def test_with_reinit_reloaded(self): self.assertIs(reloaded.snapshot.cached, reloaded.module) + def test_check_state_first(self): + for variant in ['', '_with_reinit', '_with_state']: + name = f'{self.NAME}{variant}_check_cache_first' + with self.subTest(name): + mod = self._load_dynamic(name, self.ORIGIN) + self.assertEqual(mod.__name__, name) + # Currently, for every single-phrase init module loaded # in multiple interpreters, those interpreters share a # PyModuleDef for that object, which can be a problem. diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst new file mode 100644 index 00000000000000..3a28a94df0f7cf --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-25-12-52-25.gh-issue-119560.wSlm8q.rst @@ -0,0 +1,3 @@ +An invalid assert in beta 1 has been removed. The assert would fail if +``PyState_FindModule()`` was used in an extension module's init function +before the module def had been initialized. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 448be502466e79..bcdb5ba31842fd 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -1,7 +1,7 @@ /* Testing module for single-phase initialization of extension modules -This file contains 5 distinct modules, meaning each as its own name +This file contains 8 distinct modules, meaning each as its own name and its own init function (PyInit_...). The default import system will only find the one matching the filename: _testsinglephase. To load the others you must do so manually. For example: @@ -14,7 +14,7 @@ spec = importlib.util.spec_from_file_location(name, filename, loader=loader) mod = importlib._bootstrap._load(spec) ``` -Here are the 5 modules: +Here are the 8 modules: * _testsinglephase * def: _testsinglephase_basic, @@ -136,6 +136,32 @@ Here are the 5 modules: 5. increment .initialized_count * functions: see common functions below * import system: same as _testsinglephase_basic_copy +* _testsinglephase_check_cache_first + * def: _testsinglepahse_check_cache_first + * m_name: "_testsinglephase_check_cache_first" + * m_size: -1 + * state: none + * init function: + * tries PyState_FindModule() first + * otherwise creates empty module + * functions: none + * import system: same as _testsinglephase +* _testsinglephase_with_reinit_check_cache_first + * def: _testsinglepahse_with_reinit_check_cache_first + * m_name: "_testsinglephase_with_reinit_check_cache_first" + * m_size: 0 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_reinit +* _testsinglephase_with_state_check_cache_first + * def: _testsinglepahse_with_state_check_cache_first + * m_name: "_testsinglephase_with_state_check_cache_first" + * m_size: 42 + * state: none + * init function: same as _testsinglephase_check_cache_first + * functions: none + * import system: same as _testsinglephase_with_state Module state: @@ -650,3 +676,64 @@ PyInit__testsinglephase_with_state(void) finally: return module; } + + +/****************************************************/ +/* the _testsinglephase_*_check_cache_first modules */ +/****************************************************/ + +static struct PyModuleDef _testsinglephase_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_check_cache_first"), + .m_size = -1, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_check_cache_first(void) +{ + assert(_testsinglephase_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_reinit_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_reinit_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_reinit_check_cache_first"), + .m_size = 0, // no module state +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_reinit_check_cache_first(void) +{ + assert(_testsinglephase_with_reinit_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_reinit_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_reinit_check_cache_first); +} + + +static struct PyModuleDef _testsinglephase_with_state_check_cache_first = { + PyModuleDef_HEAD_INIT, + .m_name = "_testsinglephase_with_state_check_cache_first", + .m_doc = PyDoc_STR("Test module _testsinglephase_with_state_check_cache_first"), + .m_size = 42, // not used +}; + +PyMODINIT_FUNC +PyInit__testsinglephase_with_state_check_cache_first(void) +{ + assert(_testsinglephase_with_state_check_cache_first.m_base.m_index == 0); + PyObject *mod = PyState_FindModule(&_testsinglephase_with_state_check_cache_first); + if (mod != NULL) { + return Py_NewRef(mod); + } + return PyModule_Create(&_testsinglephase_with_state_check_cache_first); +} diff --git a/Python/import.c b/Python/import.c index ba44477318d473..4f3325aa67bd0a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -457,7 +457,6 @@ static Py_ssize_t _get_module_index_from_def(PyModuleDef *def) { Py_ssize_t index = def->m_base.m_index; - assert(index > 0); #ifndef NDEBUG struct extensions_cache_value *cached = _find_cached_def(def); assert(cached == NULL || index == _get_cached_module_index(cached)); @@ -489,7 +488,7 @@ _set_module_index(PyModuleDef *def, Py_ssize_t index) static const char * _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) { - if (index == 0) { + if (index <= 0) { return "invalid module index"; } if (MODULES_BY_INDEX(interp) == NULL) { From e418fc3a6e7bade68ab5dfe72f14ddba28e6acb5 Mon Sep 17 00:00:00 2001 From: Barney Gale Date: Sat, 25 May 2024 21:01:36 +0100 Subject: [PATCH 24/43] GH-82805: Fix handling of single-dot file extensions in pathlib (#118952) pathlib now treats "`.`" as a valid file extension (suffix). This brings it in line with `os.path.splitext()`. In the (private) pathlib ABCs, we add a new `ParserBase.splitext()` method that splits a path into a `(root, ext)` pair, like `os.path.splitext()`. This method is called by `PurePathBase.stem`, `suffix`, etc. In a future version of pathlib, we might make these base classes public, and so users will be able to define their own `splitext()` method to control file extension splitting. In `pathlib.PurePath` we add optimised `stem`, `suffix` and `suffixes` properties that don't use `splitext()`, which avoids computing the path base name twice. --- Doc/library/pathlib.rst | 13 +++++ Lib/pathlib/_abc.py | 34 ++++++------- Lib/pathlib/_local.py | 34 +++++++++++++ Lib/test/test_pathlib/test_pathlib_abc.py | 50 ++++++++++++------- ...4-05-11-20-23-45.gh-issue-82805.F9bz4J.rst | 5 ++ 5 files changed, 101 insertions(+), 35 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 71e2e5452d1754..c72d409a8eb2d6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -449,6 +449,10 @@ Pure paths provide the following methods and properties: This is commonly called the file extension. + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.suffixes A list of the path's suffixes, often called file extensions:: @@ -460,6 +464,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').suffixes [] + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. attribute:: PurePath.stem @@ -713,6 +721,11 @@ Pure paths provide the following methods and properties: >>> p.with_suffix('') PureWindowsPath('README') + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. In previous + versions, :exc:`ValueError` is raised if a single dot is supplied. + .. method:: PurePath.with_segments(*pathsegments) diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 3cdbb735096edb..6b5d9fc2a0c560 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -68,6 +68,12 @@ def splitdrive(self, path): drive. Either part may be empty.""" raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) + def splitext(self, path): + """Split the path into a pair (root, ext), where *ext* is empty or + begins with a begins with a period and contains at most one period, + and *root* is everything before the extension.""" + raise UnsupportedOperation(self._unsupported_msg('splitext()')) + def normcase(self, path): """Normalize the case of the path.""" raise UnsupportedOperation(self._unsupported_msg('normcase()')) @@ -151,12 +157,7 @@ def suffix(self): This includes the leading period. For example: '.txt' """ - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[i:] - else: - return '' + return self.parser.splitext(self.name)[1] @property def suffixes(self): @@ -165,21 +166,18 @@ def suffixes(self): These include the leading periods. For example: ['.tar', '.gz'] """ - name = self.name - if name.endswith('.'): - return [] - name = name.lstrip('.') - return ['.' + suffix for suffix in name.split('.')[1:]] + split = self.parser.splitext + stem, suffix = split(self.name) + suffixes = [] + while suffix: + suffixes.append(suffix) + stem, suffix = split(stem) + return suffixes[::-1] @property def stem(self): """The final path component, minus its last suffix.""" - name = self.name - i = name.rfind('.') - if 0 < i < len(name) - 1: - return name[:i] - else: - return name + return self.parser.splitext(self.name)[0] def with_name(self, name): """Return a new path with the file name changed.""" @@ -208,7 +206,7 @@ def with_suffix(self, suffix): if not stem: # If the stem is empty, we can't make the suffix non-empty. raise ValueError(f"{self!r} has an empty name") - elif suffix and not (suffix.startswith('.') and len(suffix) > 1): + elif suffix and not suffix.startswith('.'): raise ValueError(f"Invalid suffix {suffix!r}") else: return self.with_name(stem + suffix) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index f2776b1d20a2ea..49d9f813c54c23 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -361,6 +361,40 @@ def with_name(self, name): tail[-1] = name return self._from_parsed_parts(self.drive, self.root, tail) + @property + def stem(self): + """The final path component, minus its last suffix.""" + name = self.name + i = name.rfind('.') + if i != -1: + stem = name[:i] + # Stem must contain at least one non-dot character. + if stem.lstrip('.'): + return stem + return name + + @property + def suffix(self): + """ + The final component's last suffix, if any. + + This includes the leading period. For example: '.txt' + """ + name = self.name.lstrip('.') + i = name.rfind('.') + if i != -1: + return name[i:] + return '' + + @property + def suffixes(self): + """ + A list of the final component's suffixes, if any. + + These include the leading periods. For example: ['.tar', '.gz'] + """ + return ['.' + ext for ext in self.name.lstrip('.').split('.')[1:]] + def relative_to(self, other, *, walk_up=False): """Return the relative path to another path identified by the passed arguments. If the operation is not possible (because this is not diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index d9e51c0e3d6411..57cc1612c03468 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -50,6 +50,7 @@ def test_unsupported_operation(self): self.assertRaises(e, m.join, 'foo') self.assertRaises(e, m.split, 'foo') self.assertRaises(e, m.splitdrive, 'foo') + self.assertRaises(e, m.splitext, 'foo') self.assertRaises(e, m.normcase, 'foo') self.assertRaises(e, m.isabs, 'foo') @@ -789,8 +790,12 @@ def test_suffix_common(self): self.assertEqual(P('/a/.hg.rc').suffix, '.rc') self.assertEqual(P('a/b.tar.gz').suffix, '.gz') self.assertEqual(P('/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('a/trailing.dot.').suffix, '.') + self.assertEqual(P('/a/trailing.dot.').suffix, '.') + self.assertEqual(P('a/..d.o.t..').suffix, '.') + self.assertEqual(P('a/inn.er..dots').suffix, '.dots') + self.assertEqual(P('photo').suffix, '') + self.assertEqual(P('photo.jpg').suffix, '.jpg') @needs_windows def test_suffix_windows(self): @@ -807,8 +812,8 @@ def test_suffix_windows(self): self.assertEqual(P('c:/a/.hg.rc').suffix, '.rc') self.assertEqual(P('c:a/b.tar.gz').suffix, '.gz') self.assertEqual(P('c:/a/b.tar.gz').suffix, '.gz') - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffix, '') - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffix, '') + self.assertEqual(P('c:a/trailing.dot.').suffix, '.') + self.assertEqual(P('c:/a/trailing.dot.').suffix, '.') self.assertEqual(P('//My.py/Share.php').suffix, '') self.assertEqual(P('//My.py/Share.php/a/b').suffix, '') @@ -828,8 +833,12 @@ def test_suffixes_common(self): self.assertEqual(P('/a/.hg.rc').suffixes, ['.rc']) self.assertEqual(P('a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('/a/b.tar.gz').suffixes, ['.tar', '.gz']) - self.assertEqual(P('a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('/a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('a/..d.o.t..').suffixes, ['.o', '.t', '.', '.']) + self.assertEqual(P('a/inn.er..dots').suffixes, ['.er', '.', '.dots']) + self.assertEqual(P('photo').suffixes, []) + self.assertEqual(P('photo.jpg').suffixes, ['.jpg']) @needs_windows def test_suffixes_windows(self): @@ -848,8 +857,8 @@ def test_suffixes_windows(self): self.assertEqual(P('c:/a/b.tar.gz').suffixes, ['.tar', '.gz']) self.assertEqual(P('//My.py/Share.php').suffixes, []) self.assertEqual(P('//My.py/Share.php/a/b').suffixes, []) - self.assertEqual(P('c:a/Some name. Ending with a dot.').suffixes, []) - self.assertEqual(P('c:/a/Some name. Ending with a dot.').suffixes, []) + self.assertEqual(P('c:a/trailing.dot.').suffixes, ['.dot', '.']) + self.assertEqual(P('c:/a/trailing.dot.').suffixes, ['.dot', '.']) def test_stem_empty(self): P = self.cls @@ -865,8 +874,11 @@ def test_stem_common(self): self.assertEqual(P('a/.hgrc').stem, '.hgrc') self.assertEqual(P('a/.hg.rc').stem, '.hg') self.assertEqual(P('a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('a/trailing.dot.').stem, 'trailing.dot') + self.assertEqual(P('a/..d.o.t..').stem, '..d.o.t.') + self.assertEqual(P('a/inn.er..dots').stem, 'inn.er.') + self.assertEqual(P('photo').stem, 'photo') + self.assertEqual(P('photo.jpg').stem, 'photo') @needs_windows def test_stem_windows(self): @@ -880,8 +892,8 @@ def test_stem_windows(self): self.assertEqual(P('c:a/.hgrc').stem, '.hgrc') self.assertEqual(P('c:a/.hg.rc').stem, '.hg') self.assertEqual(P('c:a/b.tar.gz').stem, 'b.tar') - self.assertEqual(P('c:a/Some name. Ending with a dot.').stem, - 'Some name. Ending with a dot.') + self.assertEqual(P('c:a/trailing.dot.').stem, 'trailing.dot') + def test_with_name_common(self): P = self.cls self.assertEqual(P('a/b').with_name('d.xml'), P('a/d.xml')) @@ -929,16 +941,16 @@ def test_with_stem_common(self): self.assertEqual(P('a/b.py').with_stem('d'), P('a/d.py')) self.assertEqual(P('/a/b.py').with_stem('d'), P('/a/d.py')) self.assertEqual(P('/a/b.tar.gz').with_stem('d'), P('/a/d.gz')) - self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d')) - self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d')) + self.assertEqual(P('a/Dot ending.').with_stem('d'), P('a/d.')) + self.assertEqual(P('/a/Dot ending.').with_stem('d'), P('/a/d.')) @needs_windows def test_with_stem_windows(self): P = self.cls self.assertEqual(P('c:a/b').with_stem('d'), P('c:a/d')) self.assertEqual(P('c:/a/b').with_stem('d'), P('c:/a/d')) - self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d')) - self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d')) + self.assertEqual(P('c:a/Dot ending.').with_stem('d'), P('c:a/d.')) + self.assertEqual(P('c:/a/Dot ending.').with_stem('d'), P('c:/a/d.')) self.assertRaises(ValueError, P('c:').with_stem, 'd') self.assertRaises(ValueError, P('c:/').with_stem, 'd') self.assertRaises(ValueError, P('//My/Share').with_stem, 'd') @@ -974,6 +986,11 @@ def test_with_suffix_common(self): # Stripping suffix. self.assertEqual(P('a/b.py').with_suffix(''), P('a/b')) self.assertEqual(P('/a/b').with_suffix(''), P('/a/b')) + # Single dot + self.assertEqual(P('a/b').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b').with_suffix('.'), P('/a/b.')) + self.assertEqual(P('a/b.py').with_suffix('.'), P('a/b.')) + self.assertEqual(P('/a/b.py').with_suffix('.'), P('/a/b.')) @needs_windows def test_with_suffix_windows(self): @@ -1012,7 +1029,6 @@ def test_with_suffix_invalid(self): # Invalid suffix. self.assertRaises(ValueError, P('a/b').with_suffix, 'gz') self.assertRaises(ValueError, P('a/b').with_suffix, '/') - self.assertRaises(ValueError, P('a/b').with_suffix, '.') self.assertRaises(ValueError, P('a/b').with_suffix, '/.gz') self.assertRaises(ValueError, P('a/b').with_suffix, 'c/d') self.assertRaises(ValueError, P('a/b').with_suffix, '.c/.d') diff --git a/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst b/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst new file mode 100644 index 00000000000000..8715deda7d9c41 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-11-20-23-45.gh-issue-82805.F9bz4J.rst @@ -0,0 +1,5 @@ +Support single-dot file extensions in :attr:`pathlib.PurePath.suffix` and +related attributes and methods. For example, the +:attr:`~pathlib.PurePath.suffixes` of ``PurePath('foo.bar.')`` are now +``['.bar', '.']`` rather than ``[]``. This brings file extension splitting +in line with :func:`os.path.splitext`. From d25954dff5409c8926d2a4053d3e892462f8b8b5 Mon Sep 17 00:00:00 2001 From: Ned Batchelder Date: Sat, 25 May 2024 21:13:31 -0400 Subject: [PATCH 25/43] docs: fix a few typos identified by codespell (#119516) --- Doc/c-api/weakref.rst | 4 ++-- Doc/extending/extending.rst | 2 +- Doc/extending/newtypes.rst | 2 +- Doc/howto/mro.rst | 2 +- Doc/library/curses.rst | 2 +- Doc/library/numbers.rst | 4 ++-- Doc/library/optparse.rst | 2 +- Doc/library/ssl.rst | 4 ++-- Doc/library/textwrap.rst | 2 +- Doc/library/turtle.rst | 2 +- Doc/tutorial/venv.rst | 2 +- Doc/using/ios.rst | 4 ++-- Doc/whatsnew/2.2.rst | 2 +- Doc/whatsnew/2.7.rst | 2 +- Doc/whatsnew/3.12.rst | 2 +- Doc/whatsnew/3.4.rst | 2 +- Doc/whatsnew/3.6.rst | 4 ++-- Doc/whatsnew/3.9.rst | 2 +- 18 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index 038f54a9751fd1..ae0699383900c4 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -35,7 +35,7 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or + weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. @@ -47,7 +47,7 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, + is not a weakly referenceable object, or if *callback* is not callable, ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b70e1b1fe57e67..b0493bed75b151 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -868,7 +868,7 @@ It is important to call :c:func:`free` at the right time. If a block's address is forgotten but :c:func:`free` is not called for it, the memory it occupies cannot be reused until the program terminates. This is called a :dfn:`memory leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block +continues to use the block, it creates a conflict with reuse of the block through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. It has the same bad consequences as referencing uninitialized data --- core dumps, wrong results, mysterious crashes. diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 473a418809cff1..fd05c82b41629a 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -545,7 +545,7 @@ performance-critical objects (such as numbers). .. seealso:: Documentation for the :mod:`weakref` module. -For an object to be weakly referencable, the extension type must set the +For an object to be weakly referenceable, the extension type must set the ``Py_TPFLAGS_MANAGED_WEAKREF`` bit of the :c:member:`~PyTypeObject.tp_flags` field. The legacy :c:member:`~PyTypeObject.tp_weaklistoffset` field should be left as zero. diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index a44ef6848af4f3..f44b4f98e570bd 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -426,7 +426,7 @@ In this case the MRO is GFEF and the local precedence ordering is preserved. As a general rule, hierarchies such as the previous one should be -avoided, since it is unclear if F should override E or viceversa. +avoided, since it is unclear if F should override E or vice-versa. Python 2.3 solves the ambiguity by raising an exception in the creation of class G, effectively stopping the programmer from generating ambiguous hierarchies. The reason for that is that the C3 algorithm diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 883150e91378cc..91ea6150fb15ba 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -924,7 +924,7 @@ the following methods and attributes: .. method:: window.getbegyx() - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. + Return a tuple ``(y, x)`` of coordinates of upper-left corner. .. method:: window.getbkgd() diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 5f59746fa59812..d0ae79c7a3df76 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -84,10 +84,10 @@ The numeric tower ``~``. -Notes for type implementors +Notes for type implementers --------------------------- -Implementors should be careful to make equal numbers equal and hash +Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different extensions of the real numbers. For example, :class:`fractions.Fraction` implements :func:`hash` as follows:: diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 8c7d77d369b44c..3e96259f94d47b 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -1739,7 +1739,7 @@ seen, but blow up if it comes after ``-b`` in the command-line. :: Callback example 3: check option order (generalized) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you want to re-use this callback for several similar options (set a flag, but +If you want to reuse this callback for several similar options (set a flag, but blow up if ``-b`` has already been seen), it needs a bit of work: the error message and the flag that it sets must be generalized. :: diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 9c757ce1b8efc4..99abf45469018e 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -781,7 +781,7 @@ Constants .. data:: OP_SINGLE_DH_USE - Prevents re-use of the same DH key for distinct SSL sessions. This + Prevents reuse of the same DH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. @@ -789,7 +789,7 @@ Constants .. data:: OP_SINGLE_ECDH_USE - Prevents re-use of the same ECDH key for distinct SSL sessions. This + Prevents reuse of the same ECDH key for distinct SSL sessions. This improves forward secrecy but requires more computational resources. This option only applies to server sockets. diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index deaefeee7b8c99..a58b460fef409c 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -154,7 +154,7 @@ hyphenated words; only then will long words be broken if necessary, unless wrapper = TextWrapper() wrapper.initial_indent = "* " - You can re-use the same :class:`TextWrapper` object many times, and you can + You can reuse the same :class:`TextWrapper` object many times, and you can change any of its options through direct assignment to instance attributes between uses. diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index 2941201332a715..afda3685d606bb 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -120,7 +120,7 @@ off-screen):: home() The home position is at the center of the turtle's screen. If you ever need to -know them, get the turtle's x-y co-ordinates with:: +know them, get the turtle's x-y coordinates with:: pos() diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index 6cca3f1b25aadc..91e4ce18acef1d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -38,7 +38,7 @@ Creating Virtual Environments The module used to create and manage virtual environments is called :mod:`venv`. :mod:`venv` will install the Python version from which the command was run (as reported by the :option:`--version` option). -For instance, excuting the command with ``python3.12`` will install +For instance, executing the command with ``python3.12`` will install version 3.12. To create a virtual environment, decide upon a directory where you want to diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index da8f42048c0faf..71fc29c450c8eb 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -303,8 +303,8 @@ modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are either associated with the app target, or copied in as part of step 8. Step 8 should also purge any binaries that are not appropriate for the platform a - specific build is targetting (i.e., delete any device binaries if you're - building app app targeting the simulator). + specific build is targeting (i.e., delete any device binaries if you're + building an app targeting the simulator). * Any folders that contain third-party binaries must be processed into framework form by step 9. The invocation of ``install_dylib`` that processes diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index e6c13f957b8d54..d4dbe0570fbda5 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -1062,7 +1062,7 @@ code, none of the changes described here will affect you very much. simply been changed to use the new C-level interface. (Contributed by Fred L. Drake, Jr.) -* Another low-level API, primarily of interest to implementors of Python +* Another low-level API, primarily of interest to implementers of Python debuggers and development tools, was added. :c:func:`PyInterpreterState_Head` and :c:func:`PyInterpreterState_Next` let a caller walk through all the existing interpreter objects; :c:func:`PyInterpreterState_ThreadHead` and diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index 031777b9cf6413..c45f0887b41f4f 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -1738,7 +1738,7 @@ New module: importlib Python 3.1 includes the :mod:`importlib` package, a re-implementation of the logic underlying Python's :keyword:`import` statement. -:mod:`importlib` is useful for implementors of Python interpreters and +:mod:`importlib` is useful for implementers of Python interpreters and to users who wish to write new importers that can participate in the import process. Python 2.7 doesn't contain the complete :mod:`importlib` package, but instead has a tiny subset that contains diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index b64e4e205fe8c1..f99489fb53db74 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1251,7 +1251,7 @@ Deprecated :exc:`DeprecationWarning` when it can detect being called from a multithreaded process. There has always been a fundamental incompatibility with the POSIX platform when doing so. Even if such code *appeared* to work. - We added the warning to to raise awareness as issues encounted by code doing + We added the warning to raise awareness as issues encountered by code doing this are becoming more frequent. See the :func:`os.fork` documentation for more details along with `this discussion on fork being incompatible with threads `_ for *why* we're now surfacing this diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index 3dd400c3771ed2..8aef0f5ac26728 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2413,7 +2413,7 @@ Changes in the Python API formal public interface the naming has been made consistent (:issue:`18532`). * Because :mod:`unittest.TestSuite` now drops references to tests after they - are run, test harnesses that re-use a :class:`~unittest.TestSuite` to re-run + are run, test harnesses that reuse a :class:`~unittest.TestSuite` to re-run a set of tests may fail. Test suites should not be re-used in this fashion since it means state is retained between test runs, breaking the test isolation that :mod:`unittest` is designed to provide. However, if the lack diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index d62beb0bdc8672..68ab43462b743a 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -2336,10 +2336,10 @@ Changes in the Python API * With the introduction of :exc:`ModuleNotFoundError`, import system consumers may start expecting import system replacements to raise that more specific exception when appropriate, rather than the less-specific :exc:`ImportError`. - To provide future compatibility with such consumers, implementors of + To provide future compatibility with such consumers, implementers of alternative import systems that completely replace :func:`__import__` will need to update their implementations to raise the new subclass when a module - can't be found at all. Implementors of compliant plugins to the default + can't be found at all. Implementers of compliant plugins to the default import system shouldn't need to make any changes, as the default import system will raise the new subclass when appropriate. diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index e29d37ca120b76..90bdcf9541613c 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -891,7 +891,7 @@ Deprecated * Deprecated the ``split()`` method of :class:`!_tkinter.TkappType` in favour of the ``splitlist()`` method which has more consistent and - predicable behavior. + predictable behavior. (Contributed by Serhiy Storchaka in :issue:`38371`.) * The explicit passing of coroutine objects to :func:`asyncio.wait` has been From b5b7dc98c94100e992a5409d24bf035d88c7b2cd Mon Sep 17 00:00:00 2001 From: Wulian233 <71213467+Wulian233@users.noreply.github.com> Date: Sun, 26 May 2024 19:26:59 +0800 Subject: [PATCH 26/43] Update README and layout.html from 3.13 to 3.14 (#119539) Co-authored-by: Nice Zombies Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/tools/templates/layout.html | 8 +++++++- README.rst | 10 +++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index e931147813ae03..e96cbf70b1239e 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -61,7 +61,13 @@ // since all the "active, built and not hidden" versions will be shown automatically. let versions = config.versions.active.concat([ { - slug: "dev (3.13)", + slug: "dev (3.14)", + urls: { + documentation: "https://docs.python.org/3.14/", + } + }, + { + slug: "pre (3.13)", urls: { documentation: "https://docs.python.org/3.13/", } diff --git a/README.rst b/README.rst index 44b020e0c7ed5f..e3163c5ff636ab 100644 --- a/README.rst +++ b/README.rst @@ -136,7 +136,7 @@ What's New ---------- We have a comprehensive overview of the changes in the `What's New in Python -3.13 `_ document. For a more +3.14 `_ document. For a more detailed change log, read `Misc/NEWS `_, but a full accounting of changes can only be gleaned from the `commit history @@ -149,7 +149,7 @@ entitled "Installing multiple versions". Documentation ------------- -`Documentation for Python 3.13 `_ is online, +`Documentation for Python 3.14 `_ is online, updated daily. It can also be downloaded in many formats for faster access. The documentation @@ -200,15 +200,15 @@ intend to install multiple versions using the same prefix you must decide which version (if any) is your "primary" version. Install that version using ``make install``. Install all other versions using ``make altinstall``. -For example, if you want to install Python 2.7, 3.6, and 3.13 with 3.13 being the -primary version, you would execute ``make install`` in your 3.13 build directory +For example, if you want to install Python 2.7, 3.6, and 3.14 with 3.14 being the +primary version, you would execute ``make install`` in your 3.14 build directory and ``make altinstall`` in the others. Release Schedule ---------------- -See :pep:`719` for Python 3.13 release details. +See `PEP 745 `__ for Python 3.14 release details. Copyright and License Information From 008bc04dcb3b1fa6d7c11ed8050467dfad3090a9 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 26 May 2024 13:34:48 +0100 Subject: [PATCH 27/43] gh-119562: Remove AST nodes deprecated since Python 3.8 (#119563) --- Doc/whatsnew/3.14.rst | 30 ++ Lib/ast.py | 174 +------- Lib/test/test_ast.py | 416 +----------------- ...-05-25-20-20-42.gh-issue-119562.DyplWc.rst | 3 + 4 files changed, 38 insertions(+), 585 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 39172ac60cf1e0..bc12d4b3b590dd 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -108,6 +108,36 @@ Deprecated Removed ======= +ast +--- + +* Remove the following classes. They were all deprecated since Python 3.8, + and have emitted deprecation warnings since Python 3.12: + + * :class:`!ast.Num` + * :class:`!ast.Str` + * :class:`!ast.Bytes` + * :class:`!ast.NameConstant` + * :class:`!ast.Ellipsis` + + Use :class:`ast.Constant` instead. As a consequence of these removals, + user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, + ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom + :class:`ast.NodeVisitor` subclasses will no longer be called when the + ``NodeVisitor`` subclass is visiting an AST. Define a ``visit_Constant`` + method instead. + + Also, remove the following deprecated properties on :class:`ast.Constant`, + which were present for compatibility with the now-removed AST classes: + + * :attr:`!ast.Constant.n` + * :attr:`!ast.Constant.s` + + Use :attr:`!ast.Constant.value` instead. + + (Contributed by Alex Waygood in :gh:`119562`.) + + argparse -------- diff --git a/Lib/ast.py b/Lib/ast.py index 031bab43df7579..c5d495ea1c8000 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -508,27 +508,6 @@ def generic_visit(self, node): elif isinstance(value, AST): self.visit(value) - def visit_Constant(self, node): - value = node.value - type_name = _const_node_type_names.get(type(value)) - if type_name is None: - for cls, name in _const_node_type_names.items(): - if isinstance(value, cls): - type_name = name - break - if type_name is not None: - method = 'visit_' + type_name - try: - visitor = getattr(self, method) - except AttributeError: - pass - else: - import warnings - warnings.warn(f"{method} is deprecated; add visit_Constant", - DeprecationWarning, 2) - return visitor(node) - return self.generic_visit(node) - class NodeTransformer(NodeVisitor): """ @@ -597,142 +576,6 @@ def generic_visit(self, node): "use ast.Constant instead" ) - -# If the ast module is loaded more than once, only add deprecated methods once -if not hasattr(Constant, 'n'): - # The following code is for backward compatibility. - # It will be removed in future. - - def _n_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _n_setter(self, value): - import warnings - warnings._deprecated( - "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - def _s_getter(self): - """Deprecated. Use value instead.""" - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - return self.value - - def _s_setter(self, value): - import warnings - warnings._deprecated( - "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14) - ) - self.value = value - - Constant.n = property(_n_getter, _n_setter) - Constant.s = property(_s_getter, _s_setter) - -class _ABC(type): - - def __init__(cls, *args): - cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead""" - - def __instancecheck__(cls, inst): - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", - message=_DEPRECATED_CLASS_MESSAGE, - remove=(3, 14) - ) - if not isinstance(inst, Constant): - return False - if cls in _const_types: - try: - value = inst.value - except AttributeError: - return False - else: - return ( - isinstance(value, _const_types[cls]) and - not isinstance(value, _const_types_not.get(cls, ())) - ) - return type.__instancecheck__(cls, inst) - -def _new(cls, *args, **kwargs): - for key in kwargs: - if key not in cls._fields: - # arbitrary keyword arguments are accepted - continue - pos = cls._fields.index(key) - if pos < len(args): - raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") - if cls in _const_types: - import warnings - warnings._deprecated( - f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(*args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -class Num(Constant, metaclass=_ABC): - _fields = ('n',) - __new__ = _new - -class Str(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class Bytes(Constant, metaclass=_ABC): - _fields = ('s',) - __new__ = _new - -class NameConstant(Constant, metaclass=_ABC): - __new__ = _new - -class Ellipsis(Constant, metaclass=_ABC): - _fields = () - - def __new__(cls, *args, **kwargs): - if cls is _ast_Ellipsis: - import warnings - warnings._deprecated( - "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return Constant(..., *args, **kwargs) - return Constant.__new__(cls, *args, **kwargs) - -# Keep another reference to Ellipsis in the global namespace -# so it can be referenced in Ellipsis.__new__ -# (The original "Ellipsis" name is removed from the global namespace later on) -_ast_Ellipsis = Ellipsis - -_const_types = { - Num: (int, float, complex), - Str: (str,), - Bytes: (bytes,), - NameConstant: (type(None), bool), - Ellipsis: (type(...),), -} -_const_types_not = { - Num: (bool,), -} - -_const_node_type_names = { - bool: 'NameConstant', # should be before int - type(None): 'NameConstant', - int: 'Num', - float: 'Num', - complex: 'Num', - str: 'Str', - bytes: 'Bytes', - type(...): 'Ellipsis', -} - class slice(AST): """Deprecated AST node class.""" @@ -1884,27 +1727,12 @@ def visit_MatchOr(self, node): self.set_precedence(_Precedence.BOR.next(), *node.patterns) self.interleave(lambda: self.write(" | "), self.traverse, node.patterns) + def unparse(ast_obj): unparser = _Unparser() return unparser.visit(ast_obj) -_deprecated_globals = { - name: globals().pop(name) - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis') -} - -def __getattr__(name): - if name in _deprecated_globals: - globals()[name] = value = _deprecated_globals[name] - import warnings - warnings._deprecated( - f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14) - ) - return value - raise AttributeError(f"module 'ast' has no attribute '{name}'") - - def main(): import argparse diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 8a4374c56cbc08..18b2f7ffca6083 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -8,9 +8,7 @@ import textwrap import types import unittest -import warnings import weakref -from functools import partial from textwrap import dedent try: import _testinternalcapi @@ -18,7 +16,6 @@ _testinternalcapi = None from test import support -from test.support.import_helper import import_fresh_module from test.support import os_helper, script_helper from test.support.ast_helper import ASTTestMixin @@ -223,7 +220,7 @@ def to_tuple(t): # These are compiled through "eval" # It should test all expressions eval_tests = [ - # None + # Constant(value=None) "None", # BoolOp "a and b", @@ -269,9 +266,9 @@ def to_tuple(t): "f(*[0, 1])", # Call with a generator argument "f(a for a in b)", - # Num + # Constant(value=int()) "10", - # Str + # Constant(value=str()) "'string'", # Attribute "a.b", @@ -498,35 +495,8 @@ def test_base_classes(self): self.assertTrue(issubclass(ast.comprehension, ast.AST)) self.assertTrue(issubclass(ast.Gt, ast.AST)) - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - def test_field_attr_existence(self): for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue # constructor has a different signature if name == 'Index': continue @@ -569,106 +539,12 @@ def test_arguments(self): self.assertEqual(x.args, 2) self.assertEqual(x.vararg, 3) - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - def test_field_attr_writable(self): x = ast.Constant(1) # We can assign to _fields x._fields = 666 self.assertEqual(x._fields, 666) - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - 'an error in Python 3.15.', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ missing 1 required positional argument: 'value'. This will become " - 'an error in Python 3.15.', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " - 'arbitrary keyword arguments is deprecated and will be removed in Python ' - '3.15.', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_classattrs(self): with self.assertWarns(DeprecationWarning): x = ast.Constant() @@ -714,190 +590,6 @@ def test_classattrs(self): self.assertIs(ast.Constant(None).value, None) self.assertIs(ast.Constant(...).value, ...) - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(1), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - def test_constant_subclasses(self): class N(ast.Constant): def __init__(self, *args, **kwargs): @@ -2223,32 +1915,6 @@ def test_call(self): call = ast.Call(func, args, bad_keywords) self.expr(call, "must have Load context") - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - def test_attribute(self): attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) self.expr(attr, "must have Load context") @@ -2288,19 +1954,6 @@ def test_list(self): def test_tuple(self): self._sequence(ast.Tuple) - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - @support.requires_resource('cpu') def test_stdlib_validates(self): for module in STDLIB_FILES: @@ -2953,69 +2606,8 @@ def test_source_segment_missing_info(self): self.assertIsNone(ast.get_source_segment(s, x)) self.assertIsNone(ast.get_source_segment(s, y)) -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer +class NodeTransformerTests(ASTTestMixin, unittest.TestCase): def assertASTTransformation(self, tranformer_class, initial_code, expected_code): initial_ast = ast.parse(dedent(initial_code)) diff --git a/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst b/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst new file mode 100644 index 00000000000000..dd23466b9d2cef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-05-25-20-20-42.gh-issue-119562.DyplWc.rst @@ -0,0 +1,3 @@ +Remove :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, +:class:`!ast.NameConstant` and :class:`!ast.Ellipsis`. They had all emitted +deprecation warnings since Python 3.12. Patch by Alex Waygood. From 70b07aa4153c1a914a3d69307d5b258cf7ed16ab Mon Sep 17 00:00:00 2001 From: scoder Date: Sun, 26 May 2024 14:37:33 +0200 Subject: [PATCH 28/43] gh-111997: Fix argument count for LINE event and clarify type of argument counts. (#119179) --- Python/instrumentation.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 9095fb981b7981..a5211ee5428cf8 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -893,7 +893,7 @@ remove_per_instruction_tools(PyCodeObject * code, int offset, int tools) static int call_one_instrument( PyInterpreterState *interp, PyThreadState *tstate, PyObject **args, - Py_ssize_t nargsf, int8_t tool, int event) + size_t nargsf, int8_t tool, int event) { assert(0 <= tool && tool < 8); assert(tstate->tracing == 0); @@ -1084,7 +1084,7 @@ call_instrumentation_vector( args[2] = offset_obj; PyInterpreterState *interp = tstate->interp; uint8_t tools = get_tools_for_instruction(code, interp, offset, event); - Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; PyObject **callargs = &args[1]; int err = 0; while (tools) { @@ -2439,13 +2439,15 @@ capi_call_instrumentation(PyMonitoringState *state, PyObject *codelike, int32_t PyErr_SetString(PyExc_ValueError, "offset must be non-negative"); return -1; } - PyObject *offset_obj = PyLong_FromLong(offset); - if (offset_obj == NULL) { - return -1; + if (event != PY_MONITORING_EVENT_LINE) { + PyObject *offset_obj = PyLong_FromLong(offset); + if (offset_obj == NULL) { + return -1; + } + assert(args[2] == NULL); + args[2] = offset_obj; } - assert(args[2] == NULL); - args[2] = offset_obj; - Py_ssize_t nargsf = nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; + size_t nargsf = (size_t) nargs | PY_VECTORCALL_ARGUMENTS_OFFSET; PyObject **callargs = &args[1]; int err = 0; @@ -2565,8 +2567,8 @@ _PyMonitoring_FireLineEvent(PyMonitoringState *state, PyObject *codelike, int32_ if (lno == NULL) { return -1; } - PyObject *args[4] = { NULL, NULL, NULL, lno }; - int res= capi_call_instrumentation(state, codelike, offset, args, 3, + PyObject *args[3] = { NULL, NULL, lno }; + int res= capi_call_instrumentation(state, codelike, offset, args, 2, PY_MONITORING_EVENT_LINE); Py_DECREF(lno); return res; From 0220663e26aa2a5322df092078c5a16cddcc5cf4 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Sun, 26 May 2024 14:31:02 +0100 Subject: [PATCH 29/43] gh-119562: Remove unused private string constants from `ast.py` (#119576) --- Lib/ast.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Lib/ast.py b/Lib/ast.py index c5d495ea1c8000..bc6c3347787d61 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -567,15 +567,6 @@ def generic_visit(self, node): setattr(node, field, new_node) return node - -_DEPRECATED_VALUE_ALIAS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; use value instead" -) -_DEPRECATED_CLASS_MESSAGE = ( - "{name} is deprecated and will be removed in Python {remove}; " - "use ast.Constant instead" -) - class slice(AST): """Deprecated AST node class.""" From 5d04cc50e51cb262ee189a6ef0e79f4b372d1583 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Sun, 26 May 2024 10:05:23 -0700 Subject: [PATCH 30/43] gh-102864: Add switching frame test for pdb (#119564) --- Lib/test/test_pdb.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 1b329b205d2d0f..cf69bc415c9b69 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2555,7 +2555,7 @@ def test_pdb_issue_gh_94215(): def test_pdb_issue_gh_101673(): """See GH-101673 - Make sure ll won't revert local variable assignment + Make sure ll and switching frames won't revert local variable assignment >>> def test_function(): ... a = 1 @@ -2565,6 +2565,10 @@ def test_pdb_issue_gh_101673(): ... '!a = 2', ... 'll', ... 'p a', + ... 'u', + ... 'p a', + ... 'd', + ... 'p a', ... 'continue' ... ]): ... test_function() @@ -2577,6 +2581,16 @@ def test_pdb_issue_gh_101673(): 3 -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() (Pdb) p a 2 + (Pdb) u + > (11)() + -> test_function() + (Pdb) p a + *** NameError: name 'a' is not defined + (Pdb) d + > (3)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) p a + 2 (Pdb) continue """ From 5482a939ac18f4cd861d212c759960af8fa2b19d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 26 May 2024 21:33:16 -0400 Subject: [PATCH 31/43] Re-order imports to align with zipp 3.18.2 (#119587) --- Lib/test/test_zipfile/_path/test_complexity.py | 2 +- Lib/test/test_zipfile/_path/test_path.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index fd7ce57551b7a5..b505dd7c376462 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -20,7 +20,7 @@ class TestComplexity(unittest.TestCase): @pytest.mark.flaky def test_implied_dirs_performance(self): best, others = big_o.big_o( - compose(consume, zipfile.CompleteDirs._implied_dirs), + compose(consume, zipfile._path.CompleteDirs._implied_dirs), lambda size: [ '/'.join(string.ascii_lowercase + str(n)) for n in range(size) ], diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index f6e2c8c289f6fd..e5d2acf39a10f8 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -8,13 +8,13 @@ import zipfile import zipfile._path +from test.support.os_helper import temp_dir, FakePath + from ._functools import compose from ._itertools import Counter from ._test_params import parameterize, Invoked -from test.support.os_helper import temp_dir, FakePath - class jaraco: class itertools: From 5ef5622543844bad1f9bc770ddaaddd2615b8466 Mon Sep 17 00:00:00 2001 From: Xie Yanbo Date: Mon, 27 May 2024 15:57:23 +0800 Subject: [PATCH 32/43] Fix typos in HISTORY documentation (#119453) --- Misc/ACKS | 1 + Misc/HISTORY | 70 ++++++++++++++++++++++++++-------------------------- 2 files changed, 36 insertions(+), 35 deletions(-) diff --git a/Misc/ACKS b/Misc/ACKS index eaa7453aaade3e..9c10a76f1df624 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -2054,6 +2054,7 @@ Doug Wyatt Xiang Zhang Robert Xiao Florent Xicluna +Yanbo, Xie Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/HISTORY b/Misc/HISTORY index b66413277259dc..8ca35e1af62c05 100644 --- a/Misc/HISTORY +++ b/Misc/HISTORY @@ -607,7 +607,7 @@ Library MemoryError. - Issue #18473: Fixed 2to3 and 3to2 compatible pickle mappings. Fixed - ambigious reverse mappings. Added many new mappings. Import mapping is no + ambiguous reverse mappings. Added many new mappings. Import mapping is no longer applied to modules already mapped with full name mapping. - Issue #23745: The new email header parser now handles duplicate MIME @@ -2030,7 +2030,7 @@ Library initialization of the unquote_to_bytes() table of the urllib.parse module, to not waste memory if these modules are not used. -- Issue #19157: Include the broadcast address in the usuable hosts for IPv6 +- Issue #19157: Include the broadcast address in the usable hosts for IPv6 in ipaddress. - Issue #11599: When an external command (e.g. compiler) fails, distutils now @@ -2620,7 +2620,7 @@ Library - asyncio: Various improvements and small changes not all covered by issues listed below. E.g. wait_for() now cancels the inner task if - the timeout occcurs; tweaked the set of exported symbols; renamed + the timeout occurs; tweaked the set of exported symbols; renamed Empty/Full to QueueEmpty/QueueFull; "with (yield from lock)" now uses a separate context manager; readexactly() raises if not enough data was read; PTY support tweaks. @@ -3944,7 +3944,7 @@ Library - Issue #18996: TestCase.assertEqual() now more cleverly shorten differing strings in error report. -- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string reperesentation. +- Issue #19034: repr() for tkinter.Tcl_Obj now exposes string representation. - Issue #18978: ``urllib.request.Request`` now allows the method to be indicated on the class and no longer sets it to None in ``__init__``. @@ -4191,7 +4191,7 @@ Library - Issue #18532: Change the builtin hash algorithms' names to lower case names as promised by hashlib's documentation. -- Issue #8713: add new spwan and forkserver start methods, and new functions +- Issue #8713: add new spawn and forkserver start methods, and new functions get_all_start_methods, get_start_method, and set_start_method, to multiprocessing. @@ -4524,7 +4524,7 @@ Core and Builtins - Issue #16613: Add *m* argument to ``collections.Chainmap.new_child`` to allow the new child map to be specified explicitly. -- Issue #16730: importlib.machinery.FileFinder now no longers raises an +- Issue #16730: importlib.machinery.FileFinder now no longer raises an exception when trying to populate its cache and it finds out the directory is unreadable or has turned into a file. Reported and diagnosed by David Pritchard. @@ -4832,7 +4832,7 @@ Library on Windows and adds no value over and above python -m pydoc ... - Issue #18155: The csv module now correctly handles csv files that use - a delimter character that has a special meaning in regexes, instead of + a delimiter character that has a special meaning in regexes, instead of throwing an exception. - Issue #14360: encode_quopri can now be successfully used as an encoder @@ -6329,7 +6329,7 @@ Documentation - Issue #15940: Specify effect of locale on time functions. -- Issue #17538: Document XML vulnerabilties +- Issue #17538: Document XML vulnerabilities - Issue #16642: sched.scheduler timefunc initial default is time.monotonic. Patch by Ramchandra Apte @@ -6676,7 +6676,7 @@ Library - Issue #14669: Fix pickling of connections and sockets on Mac OS X by sending/receiving an acknowledgment after file descriptor transfer. - TestPicklingConnection has been reenabled for Mac OS X. + TestPicklingConnection has been re-enabled for Mac OS X. - Issue #11062: Fix adding a message from file to Babyl mailbox. @@ -7114,7 +7114,7 @@ Build - Issue #14330: For cross builds, don't use host python, use host search paths for host compiler. -- Issue #15235: Allow Berkley DB versions up to 5.3 to build the dbm module. +- Issue #15235: Allow Berkeley DB versions up to 5.3 to build the dbm module. - Issue #15268: Search curses.h in /usr/include/ncursesw. @@ -7264,7 +7264,7 @@ Library called with no arguments. - Issue #14653: email.utils.mktime_tz() no longer relies on system - mktime() when timezone offest is supplied. + mktime() when timezone offset is supplied. - Issue #14684: zlib.compressobj() and zlib.decompressobj() now support the use of predefined compression dictionaries. Original patch by Sam Rushing. @@ -7606,7 +7606,7 @@ Library - Issue #14773: Fix os.fwalk() failing on dangling symlinks. - Issue #12541: Be lenient with quotes around Realm field of HTTP Basic - Authentation in urllib2. + Authentication in urllib2. - Issue #14807: move undocumented tarfile.filemode() to stat.filemode() and add doc entry. Add tarfile.filemode alias with deprecation warning. @@ -7673,7 +7673,7 @@ Library IDLE ---- -- Issue #14958: Change IDLE systax highlighting to recognize all string and +- Issue #14958: Change IDLE syntax highlighting to recognize all string and byte literals supported in Python 3.3. - Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. @@ -10176,7 +10176,7 @@ IDLE - Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. (Patch by Roger Serwy) -- Issue #9871: Prevent IDLE 3 crash when given byte stings +- Issue #9871: Prevent IDLE 3 crash when given byte strings with invalid hex escape sequences, like b'\x0'. (Original patch by Claudiu Popa.) @@ -12098,7 +12098,7 @@ Library - Issue #9632: Remove sys.setfilesystemencoding() function: use PYTHONFSENCODING environment variable to set the filesystem encoding at Python startup. sys.setfilesystemencoding() creates inconsistencies because it is unable to - reencode all filenames in all objects. + re-encode all filenames in all objects. - Issue #9410: Various optimizations to the pickle module, leading to speedups up to 4x (depending on the benchmark). Mostly ported from Unladen Swallow; @@ -12509,7 +12509,7 @@ Library - Issue #9605: posix.getlogin() decodes the username with file filesystem encoding and surrogateescape error handler. Patch written by David Watson. -- Issue #9604: posix.initgroups() encodes the username using the fileystem +- Issue #9604: posix.initgroups() encodes the username using the filesystem encoding and surrogateescape error handler. Patch written by David Watson. - Issue #9603: posix.ttyname() and posix.ctermid() decode the terminal name @@ -12667,7 +12667,7 @@ What's New in Python 3.2 Alpha 1? Core and Builtins ----------------- -- Issue #8991: convertbuffer() rejects discontigious buffers. +- Issue #8991: convertbuffer() rejects discontiguous buffers. - Issue #7616: Fix copying of overlapping memoryview slices with the Intel compiler. @@ -13211,7 +13211,7 @@ Library - Issue #7989: Added pure python implementation of the `datetime` module. The C module is renamed to `_datetime` and if available, overrides all classes - defined in datetime with fast C impementation. Python implementation is based + defined in datetime with fast C implementation. Python implementation is based on the original python prototype for the datetime module by Tim Peters with minor modifications by the PyPy project. The test suite now tests `datetime` module with and without `_datetime` acceleration using the same test cases. @@ -15049,7 +15049,7 @@ Extension Modules an error. The _PY_STRUCT_FLOAT_COERCE constant has been removed. The version number has been bumped to 0.3. -- Issue #5359: Readd the Berkeley DB detection code to allow _dbm be built +- Issue #5359: Re-add the Berkeley DB detection code to allow _dbm be built using Berkeley DB. Tests @@ -17028,7 +17028,7 @@ Extension Modules and renamed to filter(), map(), and zip(). Also, renamed izip_longest() to zip_longest() and ifilterfalse() to filterfalse(). -- Issue #1762972: Readded the reload() function as imp.reload(). +- Issue #1762972: Re-added the reload() function as imp.reload(). - Bug #2111: mmap segfaults when trying to write a block opened with PROT_READ. @@ -18448,7 +18448,7 @@ Core and builtins - Fixed bug #1459029 - unicode reprs were double-escaped. -- Patch #1396919: The system scope threads are reenabled on FreeBSD +- Patch #1396919: The system scope threads are re-enabled on FreeBSD 5.4 and later versions. - Bug #1115379: Compiling a Unicode string with an encoding declaration @@ -21803,7 +21803,7 @@ Library - New csv package makes it easy to read/write CSV files. - Module shlex has been extended to allow posix-like shell parsings, - including a split() function for easy spliting of quoted strings and + including a split() function for easy splitting of quoted strings and commands. An iterator interface was also implemented. Tools/Demos @@ -27751,7 +27751,7 @@ Fri Mar 12 22:15:43 1999 Guido van Rossum The filename to URL conversion didn't properly quote special characters. - The URL to filename didn't properly unquote special chatacters. + The URL to filename didn't properly unquote special characters. * Objects/floatobject.c: OK, try again. Vladimir gave me a fix for the alignment bus error, @@ -27807,7 +27807,7 @@ Wed Mar 10 22:55:47 1999 Guido van Rossum classes in selected module methods of selected class - Sinlge clicking in a directory, module or class item updates the next + Single clicking in a directory, module or class item updates the next column with info about the selected item. Double clicking in a module, class or method item opens the file (and selects the clicked item if it is a class or method). @@ -28130,7 +28130,7 @@ webchecker and other ftp retrieves. - ConfigParser's get() method now accepts an optional keyword argument (vars) that is substituted on top of the defaults that were setup in -__init__. You can now also have recusive references in your +__init__. You can now also have recursive references in your configuration file. - Some improvements to the Queue module, including a put_nowait() @@ -28209,7 +28209,7 @@ core. not. - The curses module implements an optional nlines argument to -w.scroll(). (It then calls wscrl(win, nlines) instead of scoll(win).) +w.scroll(). (It then calls wscrl(win, nlines) instead of scroll(win).) Changes to tools ---------------- @@ -28504,7 +28504,7 @@ PyEval_GetGlobals. - glmodule.c: check in the changed version after running the stubber again -- this solves the conflict with curses over the 'clear' entry point much nicer. (Jack Jansen had checked in the changes to cstubs -eons ago, but I never regenrated glmodule.c :-( ) +eons ago, but I never regenerated glmodule.c :-( ) - frameobject.c: fix reference count bug in PyFrame_New. Vladimir Marangozov. @@ -28581,7 +28581,7 @@ idiom L1[len(L1):] = L2. - Better error messages when a sequence is indexed with a non-integer. -- Bettter error message when calling a non-callable object (include +- Better error message when calling a non-callable object (include the type in the message). Python services @@ -28656,7 +28656,7 @@ Internet Protocols and Support - imaplib.py: new version from Piers Lauder. - smtplib.py: change sendmail() method to accept a single string or a -list or strings as the destination (commom newbie mistake). +list or strings as the destination (common newbie mistake). - poplib.py: LIST with a msg argument fixed. @@ -31109,7 +31109,7 @@ encoding/decoding CGI form arguments. Catch all errors from the ftp module. HTTP requests now add the Host: header line. The proxy variable names are now mapped to lower case, for Windows. The spliturl() function no longer erroneously throws away all data past -the first newline. The basejoin() function now intereprets "../" +the first newline. The basejoin() function now interprets "../" correctly. I *believe* that the problems with "exception raised in __del__" under certain circumstances have been fixed (mostly by changes elsewher in the interpreter). @@ -31397,7 +31397,7 @@ changes and fixes. - Added a bunch of new winfo options to Tkinter.py; we should now be up to date with Tk 4.2. The new winfo options supported are: -mananger, pointerx, pointerxy, pointery, server, viewable, visualid, +manager, pointerx, pointerxy, pointery, server, viewable, visualid, visualsavailable. - The broken bind() method on Canvas objects defined in the Canvas.py @@ -32552,7 +32552,7 @@ The same applies to posixfile.open() and the socket method makefile(). is being maintained and distributed separately. - Improved support for the Apple Macintosh, in part by Jack Jansen, -e.g. interfaces to (a few) resource mananger functions, get/set file +e.g. interfaces to (a few) resource manager functions, get/set file type and creator, gestalt, sound manager, speech manager, MacTCP, comm toolbox, and the think C console library. This is being maintained and distributed separately. @@ -33229,7 +33229,7 @@ sys.argv[0]; it can simply do "if __name__ == '__main__': main()". * When an object is printed by the print statement, its implementation of str() is used. This means that classes can define __str__(self) to direct how their instances are printed. This is different from -__repr__(self), which should define an unambigous string +__repr__(self), which should define an unambiguous string representation of the instance. (If __str__() is not defined, it defaults to __repr__().) @@ -34366,7 +34366,7 @@ eval_code) and ceval.h (which doesn't need compile.hand declares the rest) ceval.h defines macros BGN_SAVE / END_SAVE for use with threads (to -improve the parallellism of multi-threaded programs by letting other +improve the parallelism of multi-threaded programs by letting other Python code run when a blocking system call or something similar is made) @@ -34514,7 +34514,7 @@ names listed in a 'global' statement must not be used in the function before the statement is reached. Remember that you don't need to use 'global' if you only want to *use* -a global variable in a function; nor do you need ot for assignments to +a global variable in a function; nor do you need to for assignments to parts of global variables (e.g., list or dictionary items or attributes of class instances). This has not changed; in fact assignment to part of a global variable was the standard workaround. From c7a5e1e550a2a0bfa11dbf055ed4b7afb26b5fe9 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Mon, 27 May 2024 14:35:36 +0300 Subject: [PATCH 33/43] ``Include/internal/pycore_import.h``: Fix typo (#119586) Fix typo --- Include/internal/pycore_import.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index bd40707fed21a8..f8329a460d6cbf 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -50,7 +50,7 @@ struct _import_runtime_state { PyMutex mutex; /* The actual cache of (filename, name, PyModuleDef) for modules. Only legacy (single-phase init) extension modules are added - and only if they support multiple initialization (m_size >- 0) + and only if they support multiple initialization (m_size >= 0) or are imported in the main interpreter. This is initialized lazily in fix_up_extension() in import.c. Modules are added there and looked up in _imp.find_extension(). */ From 3b26cd8ca0e6c65e4b61effea9aa44d06e926797 Mon Sep 17 00:00:00 2001 From: Aditya Borikar Date: Mon, 27 May 2024 06:16:13 -0600 Subject: [PATCH 34/43] gh-119467: Fix Py_buffer.format type and correct documentation typo (#119475) --- Doc/c-api/buffer.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 1e1cabdf242bd1..9500fe465c7d94 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -147,9 +147,9 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char *format + .. c:member:: char *format - A *NUL* terminated string in :mod:`struct` module style syntax describing + A *NULL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) is assumed. From 041a566f3f987619cef7d6ae7915ba93e39d2d1e Mon Sep 17 00:00:00 2001 From: Savannah Ostrowski Date: Mon, 27 May 2024 05:20:28 -0700 Subject: [PATCH 35/43] GH-117283: Add doc warning for `PyTuple_SetItem` refcount > 1 (#117916) --- Doc/c-api/tuple.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 0d68a360f347f8..52668d16b74436 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -105,6 +105,12 @@ Tuple Objects is being replaced; any reference in the tuple at position *pos* will be leaked. + .. warning:: + + This macro should *only* be used on tuples that are newly created. + Using this macro on a tuple that is already in use (or in other words, has + a refcount > 1) could lead to undefined behavior. + .. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) From 59630f92d8223f80993e3646b0f734d27f4b8dd4 Mon Sep 17 00:00:00 2001 From: Rafael Fontenelle Date: Mon, 27 May 2024 09:39:59 -0300 Subject: [PATCH 36/43] Docs: Add class role for IPV{4,6}Address and fix a typo (#118059) Add class role for IPV{4,6}Address and fix a typo Co-authored-by: Kumar Aditya --- Doc/library/ipaddress.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index d7dccf1a86593d..ead841b0581e21 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -504,7 +504,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 32-bits integer, a 4-bytes packed - integer, or an existing IPv4Address object; and the netmask is either + integer, or an existing :class:`IPv4Address` object; and the netmask is either an integer representing the prefix length (e.g. ``24``) or a string representing the prefix mask (e.g. ``255.255.255.0``). @@ -725,7 +725,7 @@ dictionaries. 4. A two-tuple of an address description and a netmask, where the address description is either a string, a 128-bits integer, a 16-bytes packed - integer, or an existing IPv6Address object; and the netmask is an + integer, or an existing :class:`IPv6Address` object; and the netmask is an integer representing the prefix length. An :exc:`AddressValueError` is raised if *address* is not a valid IPv6 @@ -781,7 +781,7 @@ dictionaries. .. attribute:: is_site_local - These attribute is true for the network as a whole if it is true + This attribute is true for the network as a whole if it is true for both the network address and the broadcast address. From 88e3fee3f81f3470cf4fe2e2611441071779e884 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 27 May 2024 20:29:27 +0300 Subject: [PATCH 37/43] Docs: Only install sphinx-autobuild for `make htmllive` (#119607) --- Doc/Makefile | 6 +++++- Doc/requirements.txt | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/Makefile b/Doc/Makefile index eca574ec290af7..1cbfc722b010f5 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -150,10 +150,14 @@ gettext: build htmlview: html $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" +.PHONY: ensure-sphinx-autobuild +ensure-sphinx-autobuild: venv + $(VENVDIR)/bin/sphinx-autobuild --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install sphinx-autobuild + .PHONY: htmllive htmllive: SPHINXBUILD = $(VENVDIR)/bin/sphinx-autobuild htmllive: SPHINXOPTS = --re-ignore="/venv/" --open-browser --delay 0 -htmllive: html +htmllive: ensure-sphinx-autobuild html .PHONY: clean clean: clean-venv diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 15675ab45fea71..b47a9d8a8635ab 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -10,7 +10,6 @@ sphinx~=7.3.0 blurb -sphinx-autobuild sphinxext-opengraph==0.7.5 sphinx-notfound-page==1.0.0 From 3dfa364cf2ae94e797b25fe5cac74b016a5a7fe6 Mon Sep 17 00:00:00 2001 From: Tian Gao Date: Mon, 27 May 2024 10:54:23 -0700 Subject: [PATCH 38/43] gh-119580: Improve version added section for convenience variable (#119583) --- Doc/library/pdb.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 7d67e06434b799..cd6496203949ea 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -290,6 +290,8 @@ There are three preset *convenience variables*: .. versionadded:: 3.12 + Added the *convenience variable* feature. + .. index:: pair: .pdbrc; file triple: debugger; configuration; file From eea26c4a731ff9547d48a6761b209fee3f2f84df Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 27 May 2024 21:04:34 +0300 Subject: [PATCH 39/43] Docs: Move inline JavaScript to own file to reduce duplication (#119541) --- Doc/tools/static/rtd_switcher.js | 88 ++++++++++++++++++++++++++++++ Doc/tools/templates/layout.html | 93 +------------------------------- 2 files changed, 90 insertions(+), 91 deletions(-) create mode 100644 Doc/tools/static/rtd_switcher.js diff --git a/Doc/tools/static/rtd_switcher.js b/Doc/tools/static/rtd_switcher.js new file mode 100644 index 00000000000000..a67bb85505a9ca --- /dev/null +++ b/Doc/tools/static/rtd_switcher.js @@ -0,0 +1,88 @@ + function onSwitch(event) { + const option = event.target.selectedIndex; + const item = event.target.options[option]; + window.location.href = item.dataset.url; + } + + document.addEventListener("readthedocs-addons-data-ready", function(event) { + const config = event.detail.data() + + // Add some mocked hardcoded versions pointing to the official + // documentation while migrating to Read the Docs. + // These are only for testing purposes. + // TODO: remove them when managing all the versions on Read the Docs, + // since all the "active, built and not hidden" versions will be shown automatically. + let versions = config.versions.active.concat([ + { + slug: "dev (3.14)", + urls: { + documentation: "https://docs.python.org/3.14/", + } + }, + { + slug: "dev (3.13)", + urls: { + documentation: "https://docs.python.org/3.13/", + } + }, + { + slug: "3.12", + urls: { + documentation: "https://docs.python.org/3.12/", + } + }, + { + slug: "3.11", + urls: { + documentation: "https://docs.python.org/3.11/", + } + }, + ]); + + const versionSelect = ` + + `; + + // Prepend the current language to the options on the selector + let languages = config.projects.translations.concat(config.projects.current); + languages = languages.sort((a, b) => a.language.name.localeCompare(b.language.name)); + + const languageSelect = ` + + `; + + // Query all the placeholders because there are different ones for Desktop/Mobile + const versionPlaceholders = document.querySelectorAll(".version_switcher_placeholder"); + for (placeholder of versionPlaceholders) { + placeholder.innerHTML = versionSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + + const languagePlaceholders = document.querySelectorAll(".language_switcher_placeholder"); + for (placeholder of languagePlaceholders) { + placeholder.innerHTML = languageSelect; + let selectElement = placeholder.querySelector("select"); + selectElement.addEventListener("change", onSwitch); + } + }); diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index e96cbf70b1239e..3f88fc8e91faad 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -43,96 +43,7 @@ {{ super() }} {%- if not embedded %} - - + + {%- endif %} {% endblock %} From 3ff06ebec4e8b466f76078aa9c97cea2093d52ab Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Mon, 27 May 2024 11:07:16 -0700 Subject: [PATCH 40/43] Withdraw most of my ownership in favor of Mark (#119611) --- .github/CODEOWNERS | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e955567ec0b0f8..e08d6cc5719737 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -29,20 +29,20 @@ Objects/type* @markshannon Objects/codeobject.c @markshannon Objects/frameobject.c @markshannon Objects/call.c @markshannon -Python/ceval*.c @markshannon @gvanrossum -Python/ceval*.h @markshannon @gvanrossum +Python/ceval*.c @markshannon +Python/ceval*.h @markshannon Python/compile.c @markshannon @iritkatriel Python/assemble.c @markshannon @iritkatriel Python/flowgraph.c @markshannon @iritkatriel Python/ast_opt.c @isidentical -Python/bytecodes.c @markshannon @gvanrossum -Python/optimizer*.c @markshannon @gvanrossum +Python/bytecodes.c @markshannon +Python/optimizer*.c @markshannon Python/optimizer_analysis.c @Fidget-Spinner Python/optimizer_bytecodes.c @Fidget-Spinner Lib/_pyrepl/* @pablogsal @lysnikolaou @ambv Lib/test/test_patma.py @brandtbucher Lib/test/test_type_*.py @JelleZijlstra -Lib/test/test_capi/test_misc.py @markshannon @gvanrossum +Lib/test/test_capi/test_misc.py @markshannon Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently @@ -152,7 +152,7 @@ Include/internal/pycore_time.h @pganssle @abalkin /Lib/test/test_tokenize.py @pablogsal @lysnikolaou # Code generator -/Tools/cases_generator/ @gvanrossum +/Tools/cases_generator/ @markshannon # AST Python/ast.c @isidentical From 0bd0d4072a49df49a88e8b02c3258dbd294170f6 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 27 May 2024 13:22:57 -0500 Subject: [PATCH 41/43] Misc cleanups and wording improvements for the itertools docs (gh-119626) --- Doc/library/itertools.rst | 238 +++++++++++++++++++------------------- 1 file changed, 116 insertions(+), 122 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 43432dae1623ce..121bfd3de343c4 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -56,13 +56,13 @@ Iterator Arguments Results :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` -:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) → 6 4 1`` -:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x%2, range(10)) → 0 2 4 6 8`` +:func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` +:func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` -:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) → 1 4`` +:func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n :func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D-`` ============================ ============================ ================================================= ============================================================= @@ -97,31 +97,27 @@ The following module functions all construct and return iterators. Some provide streams of infinite length, so they should only be accessed by functions or loops that truncate the stream. -.. function:: accumulate(iterable[, func, *, initial=None]) - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). +.. function:: accumulate(iterable[, function, *, initial=None]) - If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) + Make an iterator that returns accumulated sums or accumulated + results from other binary functions. - Usually, the number of elements output matches the input iterable. - However, if the keyword argument *initial* is provided, the - accumulation leads off with the *initial* value so that the output - has one more element than the input iterable. + The *function* defaults to addition. The *function* should accept + two arguments, an accumulated total and a value from the *iterable*. + + If an *initial* value is provided, the accumulation will start with + that value and the output will have one more element than the input + iterable. Roughly equivalent to:: - def accumulate(iterable, func=operator.add, *, initial=None): + def accumulate(iterable, function=operator.add, *, initial=None): 'Return running totals' # accumulate([1,2,3,4,5]) → 1 3 6 10 15 # accumulate([1,2,3,4,5], initial=100) → 100 101 103 106 110 115 # accumulate([1,2,3,4,5], operator.mul) → 1 2 6 24 120 + iterator = iter(iterable) total = initial if initial is None: @@ -129,27 +125,29 @@ loops that truncate the stream. total = next(iterator) except StopIteration: return + yield total for element in iterator: - total = func(total, element) + total = function(total, element) yield total - The *func* argument can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments: + The *function* argument can be set to :func:`min` for a running + minimum, :func:`max` for a running maximum, or :func:`operator.mul` + for a running product. `Amortization tables + `_ + can be built by accumulating interest and applying payments: .. doctest:: >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] >>> list(accumulate(data, max)) # running maximum [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] + >>> list(accumulate(data, operator.mul)) # running product + [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] # Amortize a 5% loan of 1000 with 10 annual payments of 90 - >>> account_update = lambda bal, pmt: round(bal * 1.05) + pmt - >>> list(accumulate(repeat(-90, 10), account_update, initial=1_000)) + >>> update = lambda balance, payment: round(balance * 1.05) - payment + >>> list(accumulate(repeat(90, 10), update, initial=1_000)) [1000, 960, 918, 874, 828, 779, 728, 674, 618, 559, 497] See :func:`functools.reduce` for a similar function that returns only the @@ -158,7 +156,7 @@ loops that truncate the stream. .. versionadded:: 3.2 .. versionchanged:: 3.3 - Added the optional *func* parameter. + Added the optional *function* parameter. .. versionchanged:: 3.8 Added the optional *initial* parameter. @@ -190,8 +188,8 @@ loops that truncate the stream. # batched('ABCDEFG', 3) → ABC DEF G if n < 1: raise ValueError('n must be at least one') - iterable = iter(iterable) - while batch := tuple(islice(iterable, n)): + iterator = iter(iterable) + while batch := tuple(islice(iterator, n)): if strict and len(batch) != n: raise ValueError('batched(): incomplete batch') yield batch @@ -230,12 +228,17 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable*. + The output is a subsequence of :func:`product` keeping only entries that + are subsequences of the *iterable*. The length of the output is given + by :func:`math.comb` which computes ``n! / r! / (n - r)!`` when ``0 ≤ r + ≤ n`` or zero when ``r > n``. + The combination tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So, if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within each combination. Roughly equivalent to:: @@ -243,11 +246,13 @@ loops that truncate the stream. def combinations(iterable, r): # combinations('ABCD', 2) → AB AC AD BC BD CD # combinations(range(4), 3) → 012 013 023 123 + pool = tuple(iterable) n = len(pool) if r > n: return indices = list(range(r)) + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -260,42 +265,36 @@ loops that truncate the stream. indices[j] = indices[j-1] + 1 yield tuple(pool[i] for i in indices) - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: combinations_with_replacement(iterable, r) Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. + The output is a subsequence of :func:`product` that keeps only entries + that are subsequences (with possible repeated elements) of the + *iterable*. The number of subsequence returned is ``(n + r - 1)! / r! / + (n - 1)!`` when ``n > 0``. + The combination tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. if the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So, if the input elements are unique, the generated combinations + value. If the input elements are unique, the generated combinations will also be unique. Roughly equivalent to:: def combinations_with_replacement(iterable, r): # combinations_with_replacement('ABC', 2) → AA AB AC BB BC CC + pool = tuple(iterable) n = len(pool) if not n and r: return indices = [0] * r + yield tuple(pool[i] for i in indices) while True: for i in reversed(range(r)): @@ -306,28 +305,15 @@ loops that truncate the stream. indices[i:] = [indices[i] + 1] * (r - i) yield tuple(pool[i] for i in indices) - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - .. versionadded:: 3.1 .. function:: compress(data, selectors) - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* is true. - Stops when either the *data* or *selectors* iterables have been exhausted. - Roughly equivalent to:: + Make an iterator that returns elements from *data* where the + corresponding element in *selectors* is true. Stops when either the + *data* or *selectors* iterables have been exhausted. Roughly + equivalent to:: def compress(data, selectors): # compress('ABCDEF', [1,0,1,0,1,1]) → A C E F @@ -338,9 +324,10 @@ loops that truncate the stream. .. function:: count(start=0, step=1) - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: + Make an iterator that returns evenly spaced values beginning with + *start*. Can be used with :func:`map` to generate consecutive data + points or with :func:`zip` to add sequence numbers. Roughly + equivalent to:: def count(start=0, step=1): # count(10) → 10 11 12 13 14 ... @@ -357,11 +344,12 @@ loops that truncate the stream. .. versionchanged:: 3.1 Added *step* argument and allowed non-integer arguments. + .. function:: cycle(iterable) - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: + Make an iterator returning elements from the *iterable* and saving a + copy of each. When the iterable is exhausted, return elements from + the saved copy. Repeats indefinitely. Roughly equivalent to:: def cycle(iterable): # cycle('ABCD') → A B C D A B C D A B C D ... @@ -373,32 +361,38 @@ loops that truncate the stream. for element in saved: yield element - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). + This itertool may require significant auxiliary storage (depending on + the length of the iterable). .. function:: dropwhile(predicate, iterable) - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: + Make an iterator that drops elements from the *iterable* while the + *predicate* is true and afterwards returns every element. Roughly + equivalent to:: def dropwhile(predicate, iterable): # dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8 - iterable = iter(iterable) - for x in iterable: + + iterator = iter(iterable) + for x in iterator: if not predicate(x): yield x break - for x in iterable: + + for x in iterator: yield x + Note this does not produce *any* output until the predicate first + becomes false, so this itertool may have a lengthy start-up time. + + .. function:: filterfalse(predicate, iterable) - Make an iterator that filters elements from iterable returning only those for - which the predicate is false. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: + Make an iterator that filters elements from the *iterable* returning + only those for which the *predicate* returns a false value. If + *predicate* is ``None``, returns the items that are false. Roughly + equivalent to:: def filterfalse(predicate, iterable): # filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8 @@ -473,20 +467,19 @@ loops that truncate the stream. .. function:: islice(iterable, stop) islice(iterable, start, stop[, step]) - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. + Make an iterator that returns selected elements from the iterable. + Works like sequence slicing but does not support negative values for + *start*, *stop*, or *step*. + + If *start* is zero or ``None``, iteration starts at zero. Otherwise, + elements from the iterable are skipped until *start* is reached. - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. + If *stop* is ``None``, iteration continues until the iterator is + exhausted, if at all. Otherwise, it stops at the specified position. - Unlike regular slicing, :func:`islice` does not support negative values for - *start*, *stop*, or *step*. Can be used to extract related fields from - data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). + If *step* is ``None``, the step defaults to one. Elements are returned + consecutively unless *step* is set higher than one which results in + items being skipped. Roughly equivalent to:: @@ -534,18 +527,24 @@ loops that truncate the stream. .. function:: permutations(iterable, r=None) - Return successive *r* length permutations of elements in the *iterable*. + Return successive *r* length `permutations of elements + `_ from the *iterable*. If *r* is not specified or is ``None``, then *r* defaults to the length of the *iterable* and all possible full-length permutations are generated. + The output is a subsequence of :func:`product` where entries with + repeated elements have been filtered out. The length of the output is + given by :func:`math.perm` which computes ``n! / (n - r)!`` when + ``0 ≤ r ≤ n`` or zero when ``r > n``. + The permutation tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, + the order of the input *iterable*. If the input *iterable* is sorted, the output tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their - value. So, if the input elements are unique, there will be no repeated + value. If the input elements are unique, there will be no repeated values within a permutation. Roughly equivalent to:: @@ -578,20 +577,6 @@ loops that truncate the stream. else: return - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product` filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. .. function:: product(*iterables, repeat=1) @@ -615,10 +600,13 @@ loops that truncate the stream. def product(*iterables, repeat=1): # product('ABCD', 'xy') → Ax Ay Bx By Cx Cy Dx Dy # product(range(2), repeat=3) → 000 001 010 011 100 101 110 111 + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -626,6 +614,7 @@ loops that truncate the stream. keeping pools of values in memory to generate the products. Accordingly, it is only useful with finite inputs. + .. function:: repeat(object[, times]) Make an iterator that returns *object* over and over again. Runs indefinitely @@ -650,12 +639,12 @@ loops that truncate the stream. >>> list(map(pow, range(10), repeat(2))) [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] + .. function:: starmap(function, iterable) - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (when the data has been - "pre-zipped"). + Make an iterator that computes the *function* using arguments obtained + from the *iterable*. Used instead of :func:`map` when argument + parameters have already been "pre-zipped" into tuples. The difference between :func:`map` and :func:`starmap` parallels the distinction between ``function(a,b)`` and ``function(*c)``. Roughly @@ -669,8 +658,8 @@ loops that truncate the stream. .. function:: takewhile(predicate, iterable) - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: + Make an iterator that returns elements from the *iterable* as long as + the *predicate* is true. Roughly equivalent to:: def takewhile(predicate, iterable): # takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4 @@ -726,9 +715,15 @@ loops that truncate the stream. .. function:: zip_longest(*iterables, fillvalue=None) - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: + Make an iterator that aggregates elements from each of the + *iterables*. + + If the iterables are of uneven length, missing values are filled-in + with *fillvalue*. If not specified, *fillvalue* defaults to ``None``. + + Iteration continues until the longest iterable is exhausted. + + Roughly equivalent to:: def zip_longest(*iterables, fillvalue=None): # zip_longest('ABCD', 'xy', fillvalue='-') → Ax By C- D- @@ -754,8 +749,7 @@ loops that truncate the stream. If one of the iterables is potentially infinite, then the :func:`zip_longest` function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. + (for example :func:`islice` or :func:`takewhile`). .. _itertools-recipes: From ae7b17673f29efe17b416cbcfbf43b5b3ff5977c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Mon, 27 May 2024 15:35:30 -0400 Subject: [PATCH 42/43] gh-119584: Fix test_import Failed Assertion (gh-119623) The fix in gh-119561 introduced an assertion that doesn't hold true if any of the three new test extension modules are loaded more than once. This is fine normally but breaks if the new test_check_state_first() is run more than once, which happens for refleak checking and with the regrtest --forever flag. We fix that here by clearing each of the three modules after loading them. We also tweak a check in _modules_by_index_check(). --- Lib/test/test_import/__init__.py | 3 +++ Modules/_testsinglephase.c | 3 +++ Python/import.c | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 11eaae5e47e97a..b09065f812c0e4 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -2887,12 +2887,15 @@ def test_with_reinit_reloaded(self): self.assertIs(reloaded.snapshot.cached, reloaded.module) + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") def test_check_state_first(self): for variant in ['', '_with_reinit', '_with_state']: name = f'{self.NAME}{variant}_check_cache_first' with self.subTest(name): mod = self._load_dynamic(name, self.ORIGIN) self.assertEqual(mod.__name__, name) + sys.modules.pop(name, None) + _testinternalcapi.clear_extension(name, self.ORIGIN) # Currently, for every single-phrase init module loaded # in multiple interpreters, those interpreters share a diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index bcdb5ba31842fd..066e0dbfb63fbf 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -682,6 +682,9 @@ PyInit__testsinglephase_with_state(void) /* the _testsinglephase_*_check_cache_first modules */ /****************************************************/ +/* Each of these modules should only be freshly loaded. That means + clearing the caches and each module def's m_base after each load. */ + static struct PyModuleDef _testsinglephase_check_cache_first = { PyModuleDef_HEAD_INIT, .m_name = "_testsinglephase_check_cache_first", diff --git a/Python/import.c b/Python/import.c index 4f3325aa67bd0a..6fe6df4db4f55e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -494,7 +494,7 @@ _modules_by_index_check(PyInterpreterState *interp, Py_ssize_t index) if (MODULES_BY_INDEX(interp) == NULL) { return "Interpreters module-list not accessible."; } - if (index > PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { + if (index >= PyList_GET_SIZE(MODULES_BY_INDEX(interp))) { return "Module index out of bounds."; } return NULL; @@ -2183,7 +2183,7 @@ clear_singlephase_extension(PyInterpreterState *interp, /* Clear data set when the module was initially loaded. */ def->m_base.m_init = NULL; Py_CLEAR(def->m_base.m_copy); - // We leave m_index alone since there's no reason to reset it. + def->m_base.m_index = 0; /* Clear the PyState_*Module() cache entry. */ Py_ssize_t index = _get_cached_module_index(cached); From 3e8b60905e97a4fe89bb24180063732214368938 Mon Sep 17 00:00:00 2001 From: "Erlend E. Aasland" Date: Tue, 28 May 2024 00:02:46 +0200 Subject: [PATCH 43/43] gh-117398: Add multiphase support to _datetime (gh-119373) This is minimal support. Subinterpreters are not supported yet. That will be addressed in a later change. Co-authored-by: Eric Snow --- Lib/test/datetimetester.py | 21 +++++++++++++++++++++ Modules/_datetimemodule.c | 26 +++++++++++--------------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b3838d5b406e94..ba7f185a092629 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -47,6 +47,26 @@ pass # +# This is copied from test_import/__init__.py. +# XXX Move it to support/__init__.py. +def no_rerun(reason): + """Skip rerunning for a particular test. + + WARNING: Use this decorator with care; skipping rerunning makes it + impossible to find reference leaks. Provide a clear reason for skipping the + test using the 'reason' parameter. + """ + def deco(func): + _has_run = False + def wrapper(self): + nonlocal _has_run + if _has_run: + self.skipTest(reason) + func(self) + _has_run = True + return wrapper + return deco + pickle_loads = {pickle.loads, pickle._loads} pickle_choices = [(pickle, pickle, proto) @@ -6383,6 +6403,7 @@ class IranTest(ZoneInfoTest): @unittest.skipIf(_testcapi is None, 'need _testcapi module') +@no_rerun("the encapsulated datetime C API does not support reloading") class CapiTest(unittest.TestCase): def setUp(self): # Since the C API is not present in the _Pure tests, skip all tests diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 3c6d270b8d1331..271b37dfcded6c 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -7040,30 +7040,26 @@ _datetime_exec(PyObject *module) } #undef DATETIME_ADD_MACRO -static struct PyModuleDef datetimemodule = { +static PyModuleDef_Slot module_slots[] = { + {Py_mod_exec, _datetime_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static PyModuleDef datetimemodule = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "_datetime", .m_doc = "Fast implementation of the datetime type.", - .m_size = -1, + .m_size = 0, .m_methods = module_methods, + .m_slots = module_slots, }; PyMODINIT_FUNC PyInit__datetime(void) { - PyObject *mod = PyModule_Create(&datetimemodule); - if (mod == NULL) - return NULL; -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); -#endif - - if (_datetime_exec(mod) < 0) { - Py_DECREF(mod); - return NULL; - } - - return mod; + return PyModuleDef_Init(&datetimemodule); } /* ---------------------------------------------------------------------------