From 95f4db88d5ab7d900f05d0418b2a2e77bf9ff126 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Sat, 8 Jun 2024 10:51:09 +0300 Subject: [PATCH 01/21] gh-120242: Fix handling of `[setUp,tearDown]Class` in `test_datetime` (#120243) --- Lib/test/test_datetime.py | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 3859733a4fe65b..005187f13e665f 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -1,5 +1,6 @@ import unittest import sys +import functools from test.support.import_helper import import_fresh_module @@ -39,21 +40,26 @@ def load_tests(loader, tests, pattern): for cls in test_classes: cls.__name__ += suffix cls.__qualname__ += suffix - @classmethod - def setUpClass(cls_, module=module): - cls_._save_sys_modules = sys.modules.copy() - sys.modules[TESTS] = module - sys.modules['datetime'] = module.datetime_module - if hasattr(module, '_pydatetime'): - sys.modules['_pydatetime'] = module._pydatetime - sys.modules['_strptime'] = module._strptime - @classmethod - def tearDownClass(cls_): - sys.modules.clear() - sys.modules.update(cls_._save_sys_modules) - cls.setUpClass = setUpClass - cls.tearDownClass = tearDownClass - tests.addTests(loader.loadTestsFromTestCase(cls)) + + @functools.wraps(cls, updated=()) + class Wrapper(cls): + @classmethod + def setUpClass(cls_, module=module): + cls_._save_sys_modules = sys.modules.copy() + sys.modules[TESTS] = module + sys.modules['datetime'] = module.datetime_module + if hasattr(module, '_pydatetime'): + sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_strptime'] = module._strptime + super().setUpClass() + + @classmethod + def tearDownClass(cls_): + super().tearDownClass() + sys.modules.clear() + sys.modules.update(cls_._save_sys_modules) + + tests.addTests(loader.loadTestsFromTestCase(Wrapper)) return tests From 2080425154d235b4b7dcc9a8a2f58e71769125ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrico=20Tr=C3=B6ger?= Date: Sat, 8 Jun 2024 11:19:13 +0200 Subject: [PATCH 02/21] bpo-37755: Use configured output in pydoc instead of pager (GH-15105) If the Helper() class was initialized with an output, the topics, keywords and symbols help still use the pager instead of the output. Change the behavior so the output is used if available while keeping the previous behavior if no output was configured. --- Lib/pydoc.py | 8 +- Lib/test/test_pydoc/test_pydoc.py | 125 +++++++++++++++--- ...4-06-02-13-35-11.gh-issue-81936.ETeW9x.rst | 3 + 3 files changed, 116 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 2ba597d01f245e..d7579c1cc3dcd1 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2034,7 +2034,7 @@ def help(self, request, is_cli=False): elif request in self.symbols: self.showsymbol(request) elif request in ['True', 'False', 'None']: # special case these keywords since they are objects too - doc(eval(request), 'Help on %s:', is_cli=is_cli) + doc(eval(request), 'Help on %s:', output=self._output, is_cli=is_cli) elif request in self.keywords: self.showtopic(request) elif request in self.topics: self.showtopic(request) elif request: doc(request, 'Help on %s:', output=self._output, is_cli=is_cli) @@ -2127,7 +2127,11 @@ def showtopic(self, topic, more_xrefs=''): text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) - pager(doc, f'Help on {topic!s}') + + if self._output is None: + pager(doc, f'Help on {topic!s}') + else: + self.output.write(doc) def _gettopic(self, topic, more_xrefs=''): """Return unbuffered tuple of (topic, xrefs). diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 436fdb38756ddd..57e5b8e8abddfa 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -17,6 +17,7 @@ import types import typing import unittest +import unittest.mock import urllib.parse import xml.etree import xml.etree.ElementTree @@ -658,16 +659,13 @@ def test_fail_help_output_redirect(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __locals__ unexpectedly') + @unittest.mock.patch('pydoc.pager') @requires_docstrings - def test_help_output_redirect(self): + def test_help_output_redirect(self, pager_mock): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected - getpager_old = pydoc.getpager - getpager_new = lambda: (lambda x: x) self.maxDiff = None - buf = StringIO() - helper = pydoc.Helper(output=buf) unused, doc_loc = get_pydoc_text(pydoc_mod) module = "test.test_pydoc.pydoc_mod" help_header = """ @@ -677,21 +675,112 @@ def test_help_output_redirect(self): help_header = textwrap.dedent(help_header) expected_help_pattern = help_header + expected_text_pattern - pydoc.getpager = getpager_new - try: + with captured_output('stdout') as output, \ + captured_output('stderr') as err, \ + StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help(module) + result = buf.getvalue().strip() + expected_text = expected_help_pattern % ( + (doc_loc,) + + expected_text_data_docstrings + + (inspect.getabsfile(pydoc_mod),)) + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertEqual(expected_text, result) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_help_output_redirect_various_requests(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.help should be redirected + + def run_pydoc_for_request(request, expected_text_part): + """Helper function to run pydoc with its output redirected""" with captured_output('stdout') as output, \ - captured_output('stderr') as err: - helper.help(module) + captured_output('stderr') as err, \ + StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.help(request) result = buf.getvalue().strip() - expected_text = expected_help_pattern % ( - (doc_loc,) + - expected_text_data_docstrings + - (inspect.getabsfile(pydoc_mod),)) - self.assertEqual('', output.getvalue()) - self.assertEqual('', err.getvalue()) - self.assertEqual(expected_text, result) - finally: - pydoc.getpager = getpager_old + self.assertEqual('', output.getvalue(), msg=f'failed on request "{request}"') + self.assertEqual('', err.getvalue(), msg=f'failed on request "{request}"') + self.assertIn(expected_text_part, result, msg=f'failed on request "{request}"') + pager_mock.assert_not_called() + + self.maxDiff = None + + # test for "keywords" + run_pydoc_for_request('keywords', 'Here is a list of the Python keywords.') + # test for "symbols" + run_pydoc_for_request('symbols', 'Here is a list of the punctuation symbols') + # test for "topics" + run_pydoc_for_request('topics', 'Here is a list of available topics.') + # test for "modules" skipped, see test_modules() + # test for symbol "%" + run_pydoc_for_request('%', 'The power operator') + # test for special True, False, None keywords + run_pydoc_for_request('True', 'class bool(int)') + run_pydoc_for_request('False', 'class bool(int)') + run_pydoc_for_request('None', 'class NoneType(object)') + # test for keyword "assert" + run_pydoc_for_request('assert', 'The "assert" statement') + # test for topic "TYPES" + run_pydoc_for_request('TYPES', 'The standard type hierarchy') + # test for "pydoc.Helper.help" + run_pydoc_for_request('pydoc.Helper.help', 'Help on function help in pydoc.Helper:') + # test for pydoc.Helper.help + run_pydoc_for_request(pydoc.Helper.help, 'Help on function help in module pydoc:') + # test for pydoc.Helper() instance skipped because it is always meant to be interactive + + def test_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('with') + helptext = showtopic_io.getvalue() + self.assertIn('The "with" statement', helptext) + + def test_fail_showtopic(self): + with captured_stdout() as showtopic_io: + helper = pydoc.Helper() + helper.showtopic('abd') + expected = "no documentation found for 'abd'" + self.assertEqual(expected, showtopic_io.getvalue().strip()) + + @unittest.mock.patch('pydoc.pager') + def test_fail_showtopic_output_redirect(self, pager_mock): + with StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic("abd") + expected = "no documentation found for 'abd'" + self.assertEqual(expected, buf.getvalue().strip()) + + pager_mock.assert_not_called() + + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') + @requires_docstrings + @unittest.mock.patch('pydoc.pager') + def test_showtopic_output_redirect(self, pager_mock): + # issue 940286, if output is set in Helper, then all output from + # Helper.showtopic should be redirected + self.maxDiff = None + + with captured_output('stdout') as output, \ + captured_output('stderr') as err, \ + StringIO() as buf: + helper = pydoc.Helper(output=buf) + helper.showtopic('with') + result = buf.getvalue().strip() + self.assertEqual('', output.getvalue()) + self.assertEqual('', err.getvalue()) + self.assertIn('The "with" statement', result) + + pager_mock.assert_not_called() def test_lambda_with_return_annotation(self): func = lambda a, b, c: 1 diff --git a/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst new file mode 100644 index 00000000000000..d53cc73e728d54 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-02-13-35-11.gh-issue-81936.ETeW9x.rst @@ -0,0 +1,3 @@ +:meth:`!help` and :meth:`!showtopic` methods now respect a +configured *output* argument to :class:`!pydoc.Helper` and not use the +pager in such cases. Patch by Enrico Tröger. From 55402d3232ca400ebafe4fe3bd70f252304ebe07 Mon Sep 17 00:00:00 2001 From: Saul Shanabrook Date: Sat, 8 Jun 2024 05:41:45 -0400 Subject: [PATCH 03/21] gh-119258: Eliminate Type Guards in Tier 2 Optimizer with Watcher (GH-119365) Co-authored-by: parmeggiani Co-authored-by: dpdani Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Brandt Bucher Co-authored-by: Ken Jin --- Include/internal/pycore_optimizer.h | 9 +- Include/internal/pycore_typeobject.h | 11 ++ Lib/test/test_capi/test_opt.py | 147 ++++++++++++++++++ Lib/test/test_capi/test_watchers.py | 6 +- Lib/test/test_type_cache.py | 3 +- ...-05-23-20-17-37.gh-issue-119258.wZFIpt.rst | 3 + Modules/_testcapimodule.c | 17 -- Modules/_testinternalcapi.c | 19 +++ Objects/typeobject.c | 78 ++++++++-- Python/optimizer_analysis.c | 16 +- Python/optimizer_bytecodes.c | 37 ++++- Python/optimizer_cases.c.h | 33 +++- Python/optimizer_symbols.c | 46 +++++- 13 files changed, 366 insertions(+), 59 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 76123987ac99f5..fd7833fd231299 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -33,6 +33,7 @@ struct _Py_UopsSymbol { int flags; // 0 bits: Top; 2 or more bits: Bottom PyTypeObject *typ; // Borrowed reference PyObject *const_val; // Owned reference (!) + unsigned int type_version; // currently stores type version }; #define UOP_FORMAT_TARGET 0 @@ -123,9 +124,11 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *con extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version); extern void _Py_uop_sym_set_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); extern void _Py_uop_sym_set_non_null(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym); extern void _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *typ); +extern bool _Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version); extern void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); @@ -138,9 +141,9 @@ extern void _Py_uop_abstractcontext_fini(_Py_UOpsContext *ctx); extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, - _Py_UopsSymbol **localsplus_start, - int n_locals_already_filled, - int curr_stackentries); + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len); extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 8664ae0e44533f..bc295b1b066bd1 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -63,6 +63,8 @@ typedef struct { PyObject *tp_weaklist; } managed_static_type_state; +#define TYPE_VERSION_CACHE_SIZE (1<<12) /* Must be a power of 2 */ + struct types_state { /* Used to set PyTypeObject.tp_version_tag. It starts at _Py_MAX_GLOBAL_TYPE_VERSION_TAG + 1, @@ -118,6 +120,12 @@ struct types_state { managed_static_type_state initialized[_Py_MAX_MANAGED_STATIC_EXT_TYPES]; } for_extensions; PyMutex mutex; + + // Borrowed references to type objects whose + // tp_version_tag % TYPE_VERSION_CACHE_SIZE + // once was equal to the index in the table. + // They are cleared when the type object is deallocated. + PyTypeObject *type_version_cache[TYPE_VERSION_CACHE_SIZE]; }; @@ -230,6 +238,9 @@ extern void _PyType_SetFlags(PyTypeObject *self, unsigned long mask, extern void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags); +extern unsigned int _PyType_GetVersionForCurrentState(PyTypeObject *tp); +PyAPI_FUNC(void) _PyType_SetVersion(PyTypeObject *tp, unsigned int version); +PyTypeObject *_PyType_LookupByVersion(unsigned int version); #ifdef __cplusplus } diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 0491ff9b84d486..fc6d8b0a3f01d2 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1333,6 +1333,153 @@ def test_modified_local_is_seen_by_optimized_code(self): self.assertIs(type(s), float) self.assertEqual(s, 1024.0) + def test_guard_type_version_removed(self): + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_removed_inlined(self): + """ + Verify that the guard type version if we have an inlined function + """ + + def fn(): + pass + + def thing(a): + x = 0 + for _ in range(100): + x += a.attr + fn() + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 1) + + def test_guard_type_version_not_removed(self): + """ + Verify that the guard type version is not removed if we modify the class + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # for the first 90 iterations we set the attribute on this dummy function which shouldn't + # trigger the type watcher + # then after 90 it should trigger it and stop optimizing + # Note that the code needs to be in this weird form so it's optimized inline without any control flow + setattr((Foo, Bar)[i < 90], "attr", 2) + x += a.attr + return x + + class Foo: + attr = 1 + + class Bar: + pass + + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + + self.assertIsNotNone(ex) + self.assertEqual(res, 219) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + self.assertEqual(guard_type_version_count, 2) + + + @unittest.expectedFailure + def test_guard_type_version_not_removed_escaping(self): + """ + Verify that the guard type version is not removed if have an escaping function + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + # eval should be escaping and so should cause optimization to stop and preserve both type versions + eval("None") + x += a.attr + return x + + class Foo: + attr = 1 + res, ex = self._run_with_optimizer(thing, Foo()) + opnames = list(iter_opnames(ex)) + self.assertIsNotNone(ex) + self.assertEqual(res, 200) + guard_type_version_count = opnames.count("_GUARD_TYPE_VERSION") + # Note: This will actually be 1 for noe + # https://github.com/python/cpython/pull/119365#discussion_r1626220129 + self.assertEqual(guard_type_version_count, 2) + + + def test_guard_type_version_executor_invalidated(self): + """ + Verify that the executor is invalided on a type change. + """ + + def thing(a): + x = 0 + for i in range(100): + x += a.attr + x += a.attr + return x + + class Foo: + attr = 1 + + res, ex = self._run_with_optimizer(thing, Foo()) + self.assertEqual(res, 200) + self.assertIsNotNone(ex) + self.assertEqual(list(iter_opnames(ex)).count("_GUARD_TYPE_VERSION"), 1) + self.assertTrue(ex.is_valid()) + Foo.attr = 0 + self.assertFalse(ex.is_valid()) + + def test_type_version_doesnt_segfault(self): + """ + Tests that setting a type version doesn't cause a segfault when later looking at the stack. + """ + + # Minimized from mdp.py benchmark + + class A: + def __init__(self): + self.attr = {} + + def method(self, arg): + self.attr[arg] = None + + def fn(a): + for _ in range(100): + (_ for _ in []) + (_ for _ in [a.method(None)]) + + fn(A()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_watchers.py b/Lib/test/test_capi/test_watchers.py index 90665a7561b316..709b5e1c4b716a 100644 --- a/Lib/test/test_capi/test_watchers.py +++ b/Lib/test/test_capi/test_watchers.py @@ -282,8 +282,10 @@ class C: pass self.watch(wid, C) with catch_unraisable_exception() as cm: C.foo = "bar" - self.assertEqual(cm.unraisable.err_msg, - f"Exception ignored in type watcher callback #0 for {C!r}") + self.assertEqual( + cm.unraisable.err_msg, + f"Exception ignored in type watcher callback #1 for {C!r}", + ) self.assertIs(cm.unraisable.object, None) self.assertEqual(str(cm.unraisable.exc_value), "boom!") self.assert_events([]) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index e90e315c808361..edaf076707ad8b 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -10,8 +10,9 @@ # Skip this test if the _testcapi module isn't available. _testcapi = import_helper.import_module("_testcapi") +_testinternalcapi = import_helper.import_module("_testinternalcapi") type_get_version = _testcapi.type_get_version -type_assign_specific_version_unsafe = _testcapi.type_assign_specific_version_unsafe +type_assign_specific_version_unsafe = _testinternalcapi.type_assign_specific_version_unsafe type_assign_version = _testcapi.type_assign_version type_modified = _testcapi.type_modified diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst new file mode 100644 index 00000000000000..68f1ec1efa5751 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-23-20-17-37.gh-issue-119258.wZFIpt.rst @@ -0,0 +1,3 @@ +Eliminate type version guards in the tier two interpreter. + +Note that setting the ``tp_version_tag`` manually (which has never been supported) may result in crashes. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index b58c17260626c2..b139b46c826a3f 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2403,21 +2403,6 @@ type_modified(PyObject *self, PyObject *type) Py_RETURN_NONE; } -// Circumvents standard version assignment machinery - use with caution and only on -// short-lived heap types -static PyObject * -type_assign_specific_version_unsafe(PyObject *self, PyObject *args) -{ - PyTypeObject *type; - unsigned int version; - if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { - return NULL; - } - assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); - type->tp_version_tag = version; - type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; - Py_RETURN_NONE; -} static PyObject * type_assign_version(PyObject *self, PyObject *type) @@ -3427,8 +3412,6 @@ static PyMethodDef TestMethods[] = { {"test_py_is_funcs", test_py_is_funcs, METH_NOARGS}, {"type_get_version", type_get_version, METH_O, PyDoc_STR("type->tp_version_tag")}, {"type_modified", type_modified, METH_O, PyDoc_STR("PyType_Modified")}, - {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, - PyDoc_STR("forcefully assign type->tp_version_tag")}, {"type_assign_version", type_assign_version, METH_O, PyDoc_STR("PyUnstable_Type_AssignVersionTag")}, {"type_get_tp_bases", type_get_tp_bases, METH_O}, {"type_get_tp_mro", type_get_tp_mro, METH_O}, diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 6d4a00c06ca9de..139a0509795de9 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2002,6 +2002,22 @@ has_inline_values(PyObject *self, PyObject *obj) } +// Circumvents standard version assignment machinery - use with caution and only on +// short-lived heap types +static PyObject * +type_assign_specific_version_unsafe(PyObject *self, PyObject *args) +{ + PyTypeObject *type; + unsigned int version; + if (!PyArg_ParseTuple(args, "Oi:type_assign_specific_version_unsafe", &type, &version)) { + return NULL; + } + assert(!PyType_HasFeature(type, Py_TPFLAGS_IMMUTABLETYPE)); + _PyType_SetVersion(type, version); + type->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; + Py_RETURN_NONE; +} + /*[clinic input] gh_119213_getargs @@ -2102,6 +2118,9 @@ static PyMethodDef module_functions[] = { {"get_rare_event_counters", get_rare_event_counters, METH_NOARGS}, {"reset_rare_event_counters", reset_rare_event_counters, METH_NOARGS}, {"has_inline_values", has_inline_values, METH_O}, + {"type_assign_specific_version_unsafe", type_assign_specific_version_unsafe, METH_VARARGS, + PyDoc_STR("forcefully assign type->tp_version_tag")}, + #ifdef Py_GIL_DISABLED {"py_thread_id", get_py_thread_id, METH_NOARGS}, #endif diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 880ac6b9c009fe..cd16bebd1e1cb8 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -853,7 +853,8 @@ PyType_AddWatcher(PyType_WatchCallback callback) { PyInterpreterState *interp = _PyInterpreterState_GET(); - for (int i = 0; i < TYPE_MAX_WATCHERS; i++) { + // start at 1, 0 is reserved for cpython optimizer + for (int i = 1; i < TYPE_MAX_WATCHERS; i++) { if (!interp->type_watchers[i]) { interp->type_watchers[i] = callback; return i; @@ -960,7 +961,7 @@ type_modification_starting_unlocked(PyTypeObject *type) } /* 0 is not a valid version tag */ - _Py_atomic_store_uint32_release(&type->tp_version_tag, 0); + _PyType_SetVersion(type, 0); } #endif @@ -1024,7 +1025,7 @@ type_modified_unlocked(PyTypeObject *type) } type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ + _PyType_SetVersion(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1101,7 +1102,7 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { clear: assert(!(type->tp_flags & _Py_TPFLAGS_STATIC_BUILTIN)); type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, 0); /* 0 is not a valid version tag */ + _PyType_SetVersion(type, 0); /* 0 is not a valid version tag */ if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // This field *must* be invalidated if the type is modified (see the // comment on struct _specialization_cache): @@ -1109,6 +1110,64 @@ type_mro_modified(PyTypeObject *type, PyObject *bases) { } } +/* +The Tier 2 interpreter requires looking up the type object by the type version, so it can install +watchers to understand when they change. + +So we add a global cache from type version to borrowed references of type objects. + +This is similar to func_version_cache. +*/ + +void +_PyType_SetVersion(PyTypeObject *tp, unsigned int version) +{ +#ifndef Py_GIL_DISABLED + PyInterpreterState *interp = _PyInterpreterState_GET(); + // lookup the old version and set to null + if (tp->tp_version_tag != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (tp->tp_version_tag % TYPE_VERSION_CACHE_SIZE); + *slot = NULL; + } +#endif + FT_ATOMIC_STORE_UINT32_RELAXED(tp->tp_version_tag, version); +#ifndef Py_GIL_DISABLED + if (version != 0) { + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + *slot = tp; + } +#endif +} + +PyTypeObject * +_PyType_LookupByVersion(unsigned int version) +{ +#ifdef Py_GIL_DISABLED + return NULL; +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + PyTypeObject **slot = + interp->types.type_version_cache + + (version % TYPE_VERSION_CACHE_SIZE); + if (*slot && (*slot)->tp_version_tag == version) { + return *slot; + } + return NULL; +#endif +} + +unsigned int +_PyType_GetVersionForCurrentState(PyTypeObject *tp) +{ + return tp->tp_version_tag; +} + + + #define MAX_VERSIONS_PER_CLASS 1000 static int @@ -1137,8 +1196,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, - NEXT_GLOBAL_VERSION_TAG++); + _PyType_SetVersion(type, NEXT_GLOBAL_VERSION_TAG++); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1147,8 +1205,7 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) /* We have run out of version numbers */ return 0; } - FT_ATOMIC_STORE_UINT32_RELAXED(type->tp_version_tag, - NEXT_VERSION_TAG(interp)++); + _PyType_SetVersion(type, NEXT_VERSION_TAG(interp)++); assert (type->tp_version_tag != 0); } @@ -5768,7 +5825,7 @@ fini_static_type(PyInterpreterState *interp, PyTypeObject *type, if (final) { type->tp_flags &= ~Py_TPFLAGS_READY; type->tp_flags &= ~Py_TPFLAGS_VALID_VERSION_TAG; - type->tp_version_tag = 0; + _PyType_SetVersion(type, 0); } _PyStaticType_ClearWeakRefs(interp, type); @@ -5798,7 +5855,6 @@ type_dealloc(PyObject *self) _PyObject_ASSERT((PyObject *)type, type->tp_flags & Py_TPFLAGS_HEAPTYPE); _PyObject_GC_UNTRACK(type); - type_dealloc_common(type); // PyObject_ClearWeakRefs() raises an exception if Py_REFCNT() != 0 @@ -8367,7 +8423,7 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, self->tp_flags |= Py_TPFLAGS_IMMUTABLETYPE; assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); - self->tp_version_tag = NEXT_GLOBAL_VERSION_TAG++; + _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); self->tp_flags |= Py_TPFLAGS_VALID_VERSION_TAG; } else { diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index e5d3793bd4d204..75d1d9f6b2a794 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -79,6 +79,7 @@ increment_mutations(PyObject* dict) { * so we don't need to check that they haven't been used */ #define BUILTINS_WATCHER_ID 0 #define GLOBALS_WATCHER_ID 1 +#define TYPE_WATCHER_ID 0 static int globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, @@ -92,6 +93,14 @@ globals_watcher_callback(PyDict_WatchEvent event, PyObject* dict, return 0; } +static int +type_watcher_callback(PyTypeObject* type) +{ + _Py_Executors_InvalidateDependency(_PyInterpreterState_GET(), type, 1); + PyType_Unwatch(TYPE_WATCHER_ID, (PyObject *)type); + return 0; +} + static PyObject * convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) { @@ -167,6 +176,9 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; } + if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + } for (int pc = 0; pc < buffer_size; pc++) { _PyUOpInstruction *inst = &buffer[pc]; int opcode = inst->opcode; @@ -310,9 +322,11 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, #define sym_has_type _Py_uop_sym_has_type #define sym_get_type _Py_uop_sym_get_type #define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define sym_truthiness _Py_uop_sym_truthiness @@ -395,7 +409,7 @@ optimize_uops( _PyUOpInstruction *corresponding_check_stack = NULL; _Py_uop_abstractcontext_init(ctx); - _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, ctx->n_consumed, 0, curr_stacklen); + _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); if (frame == NULL) { return -1; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index a2cb4c0b2c5192..e6fb85a90603eb 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -21,11 +21,13 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type +#define sym_matches_type_version _Py_uop_sym_matches_type_version #define sym_get_type _Py_uop_sym_get_type #define sym_has_type _Py_uop_sym_has_type #define sym_set_null(SYM) _Py_uop_sym_set_null(ctx, SYM) #define sym_set_non_null(SYM) _Py_uop_sym_set_non_null(ctx, SYM) #define sym_set_type(SYM, TYPE) _Py_uop_sym_set_type(ctx, SYM, TYPE) +#define sym_set_type_version(SYM, VERSION) _Py_uop_sym_set_type_version(ctx, SYM, VERSION) #define sym_set_const(SYM, CNST) _Py_uop_sym_set_const(ctx, SYM, CNST) #define sym_is_bottom _Py_uop_sym_is_bottom #define frame_new _Py_uop_frame_new @@ -113,6 +115,29 @@ dummy_func(void) { sym_set_type(right, &PyLong_Type); } + op(_GUARD_TYPE_VERSION, (type_version/2, owner -- owner)) { + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + + } + } + op(_GUARD_BOTH_FLOAT, (left, right -- left, right)) { if (sym_matches_type(left, &PyFloat_Type)) { if (sym_matches_type(right, &PyFloat_Type)) { @@ -563,16 +588,12 @@ dummy_func(void) { argcount++; } - _Py_UopsSymbol **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); + } - new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); } op(_PY_FRAME_GENERAL, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b3787345ec6714..18f3ca4cb73e5a 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -930,6 +930,28 @@ } case _GUARD_TYPE_VERSION: { + _Py_UopsSymbol *owner; + owner = stack_pointer[-1]; + uint32_t type_version = (uint32_t)this_instr->operand; + assert(type_version); + if (sym_matches_type_version(owner, type_version)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } else { + // add watcher so that whenever the type changes we invalidate this + PyTypeObject *type = _PyType_LookupByVersion(type_version); + // if the type is null, it was not found in the cache (there was a conflict) + // with the key, in which case we can't trust the version + if (type) { + // if the type version was set properly, then add a watcher + // if it wasn't this means that the type version was previously set to something else + // and we set the owner to bottom, so we don't need to add a watcher because we must have + // already added one earlier. + if (sym_set_type_version(owner, type_version)) { + PyType_Watch(TYPE_WATCHER_ID, (PyObject *)type); + _Py_BloomFilter_Add(dependencies, type); + } + } + } break; } @@ -1583,16 +1605,11 @@ args--; argcount++; } - _Py_UopsSymbol **localsplus_start = ctx->n_consumed; - int n_locals_already_filled = 0; - // Can determine statically, so we interleave the new locals - // and make the current stack the new locals. - // This also sets up for true call inlining. if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { - localsplus_start = args; - n_locals_already_filled = argcount; + new_frame = frame_new(ctx, co, 0, args, argcount); + } else { + new_frame = frame_new(ctx, co, 0, NULL, 0); } - new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0); stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e546eef306eeca..f3d4078bf1a890 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -52,7 +52,8 @@ static inline int get_lltrace(void) { static _Py_UopsSymbol NO_SPACE_SYMBOL = { .flags = IS_NULL | NOT_NULL | NO_SPACE, .typ = NULL, - .const_val = NULL + .const_val = NULL, + .type_version = 0, }; _Py_UopsSymbol * @@ -76,6 +77,7 @@ sym_new(_Py_UOpsContext *ctx) self->flags = 0; self->typ = NULL; self->const_val = NULL; + self->type_version = 0; return self; } @@ -152,6 +154,18 @@ _Py_uop_sym_set_type(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyTypeObject *ty } } +bool +_Py_uop_sym_set_type_version(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, unsigned int version) +{ + // if the type version was already set, then it must be different and we should set it to bottom + if (sym->type_version) { + sym_set_bottom(ctx, sym); + return false; + } + sym->type_version = version; + return true; +} + void _Py_uop_sym_set_const(_Py_UOpsContext *ctx, _Py_UopsSymbol *sym, PyObject *const_val) { @@ -256,6 +270,12 @@ _Py_uop_sym_get_type(_Py_UopsSymbol *sym) return sym->typ; } +unsigned int +_Py_uop_sym_get_type_version(_Py_UopsSymbol *sym) +{ + return sym->type_version; +} + bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym) { @@ -272,6 +292,13 @@ _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) return _Py_uop_sym_get_type(sym) == typ; } +bool +_Py_uop_sym_matches_type_version(_Py_UopsSymbol *sym, unsigned int version) +{ + return _Py_uop_sym_get_type_version(sym) == version; +} + + int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym) { @@ -311,9 +338,9 @@ _Py_UOpsAbstractFrame * _Py_uop_frame_new( _Py_UOpsContext *ctx, PyCodeObject *co, - _Py_UopsSymbol **localsplus_start, - int n_locals_already_filled, - int curr_stackentries) + int curr_stackentries, + _Py_UopsSymbol **args, + int arg_len) { assert(ctx->curr_frame_depth < MAX_ABSTRACT_FRAME_DEPTH); _Py_UOpsAbstractFrame *frame = &ctx->frames[ctx->curr_frame_depth]; @@ -321,19 +348,22 @@ _Py_uop_frame_new( frame->stack_len = co->co_stacksize; frame->locals_len = co->co_nlocalsplus; - frame->locals = localsplus_start; + frame->locals = ctx->n_consumed; frame->stack = frame->locals + co->co_nlocalsplus; frame->stack_pointer = frame->stack + curr_stackentries; - ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); + ctx->n_consumed = ctx->n_consumed + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { ctx->done = true; ctx->out_of_space = true; return NULL; } - // Initialize with the initial state of all local variables - for (int i = n_locals_already_filled; i < co->co_nlocalsplus; i++) { + for (int i = 0; i < arg_len; i++) { + frame->locals[i] = args[i]; + } + + for (int i = arg_len; i < co->co_nlocalsplus; i++) { _Py_UopsSymbol *local = _Py_uop_sym_new_unknown(ctx); frame->locals[i] = local; } From 38a25e9560cf0ff0b80d9e90bce793ff24c6e027 Mon Sep 17 00:00:00 2001 From: neonene <53406459+neonene@users.noreply.github.com> Date: Sat, 8 Jun 2024 19:22:07 +0900 Subject: [PATCH 04/21] gh-120244: Fix re.sub() reference leak (GH-120245) --- .../next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst | 1 + Modules/_sre/sre.c | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst diff --git a/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst new file mode 100644 index 00000000000000..d21532f22a1d38 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-09-45-31.gh-issue-120244.8o9Dzr.rst @@ -0,0 +1 @@ +Fix memory leak in :func:`re.sub()` when the replacement string contains backreferences. diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index c1eff63d921de9..e33034086481c2 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -1622,6 +1622,7 @@ _sre_template_impl(PyObject *module, PyObject *pattern, PyObject *template) } self->items[i].literal = Py_XNewRef(literal); } + PyObject_GC_Track(self); return (PyObject*) self; bad_template: From 5d59b870effa0f576acf7264cfcbfca2b36e34e3 Mon Sep 17 00:00:00 2001 From: AN Long Date: Sun, 9 Jun 2024 00:11:19 +0800 Subject: [PATCH 05/21] gh-120121: Add InvalidStateError to concurrent.futures.__all__ (#120123) Co-authored-by: Nikita Sobolev --- Lib/concurrent/futures/__init__.py | 1 + .../next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 292e886d5a88ac..72de617a5b6f61 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -23,6 +23,7 @@ 'ALL_COMPLETED', 'CancelledError', 'TimeoutError', + 'InvalidStateError', 'BrokenExecutor', 'Future', 'Executor', diff --git a/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst new file mode 100644 index 00000000000000..4f3526477c8cce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-05-16-30-28.gh-issue-120121.9dz8i7.rst @@ -0,0 +1 @@ +Add :exc:`concurrent.futures.InvalidStateError` to module's ``__all__``. From 7c016deae62308dd1b4e2767fc6abf04857c7843 Mon Sep 17 00:00:00 2001 From: Clinton Date: Sat, 8 Jun 2024 13:18:58 -0400 Subject: [PATCH 06/21] gh-120276: Fix incorrect email.header.Header maxlinelen default (GH-120277) --- Doc/library/email.header.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index 6e230d5faf1654..219fad0d2f6745 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -77,7 +77,7 @@ Here is the :class:`Header` class description: The maximum line length can be specified explicitly via *maxlinelen*. For splitting the first line to a shorter value (to account for the field header which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value + field in *header_name*. The default *maxlinelen* is 78, and the default value for *header_name* is ``None``, meaning it is not taken into account for the first line of a long, split header. From 34f5ae69fe9ab0f5b23311d5c396d0cbb5902913 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Sat, 8 Jun 2024 23:45:57 +0300 Subject: [PATCH 07/21] gh-120268: Prohibit passing ``None`` to ``_pydatetime.date.fromtimestamp`` (#120269) This makes the pure Python implementation consistent with the C implementation. --- Lib/_pydatetime.py | 2 ++ Lib/test/datetimetester.py | 5 +++++ .../Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index b7d569cc41740e..34ccb2da13d0f3 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -966,6 +966,8 @@ def __new__(cls, year, month=None, day=None): @classmethod def fromtimestamp(cls, t): "Construct a date from a POSIX timestamp (like time.time())." + if t is None: + raise TypeError("'NoneType' object cannot be interpreted as an integer") y, m, d, hh, mm, ss, weekday, jday, dst = _time.localtime(t) return cls(y, m, d) diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index b80da5697ef865..28f75a803b4e04 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1336,6 +1336,11 @@ def test_insane_fromtimestamp(self): self.assertRaises(OverflowError, self.theclass.fromtimestamp, insane) + def test_fromtimestamp_with_none_arg(self): + # See gh-120268 for more details + with self.assertRaises(TypeError): + self.theclass.fromtimestamp(None) + def test_today(self): import time diff --git a/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst new file mode 100644 index 00000000000000..d48d43cd047f7a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-08-14-36-40.gh-issue-120268.MNpd1q.rst @@ -0,0 +1,2 @@ +Prohibit passing ``None`` to pure-Python :meth:`datetime.date.fromtimestamp` +to achieve consistency with C-extension implementation. From 0ae8579b85f9b0cd3f287082ad6e194bdb025d88 Mon Sep 17 00:00:00 2001 From: Carl Meyer Date: Sun, 9 Jun 2024 22:23:30 -0400 Subject: [PATCH 08/21] gh-119666: fix multiple class-scope comprehensions referencing __class__ (#120295) --- Lib/test/test_listcomps.py | 25 +++++++++++++++++++ ...-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst | 1 + Python/symtable.c | 23 ++++++++--------- 3 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index ec2aac81682db8..58b076e9ea5d8a 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -168,6 +168,31 @@ def test_references___class__(self): """ self._check_in_scopes(code, raises=NameError) + def test_references___class___defined(self): + code = """ + __class__ = 2 + res = [__class__ for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___class___enclosing(self): + code = """ + __class__ = 2 + class C: + res = [__class__ for x in [1]] + res = C.res + """ + self._check_in_scopes(code, raises=NameError) + + def test_super_and_class_cell_in_sibling_comps(self): + code = """ + [super for _ in [1]] + [__class__ for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + def test_inner_cell_shadows_outer(self): code = """ items = [(lambda: i) for i in range(5)] diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst new file mode 100644 index 00000000000000..09c1f553c48702 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-09-19-13-38.gh-issue-119666.S0G4rZ.rst @@ -0,0 +1 @@ +Fix a compiler crash in the case where two comprehensions in class scope both reference ``__class__``. diff --git a/Python/symtable.c b/Python/symtable.c index 0ee8ca36cf8df0..7e452cdb13badf 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -781,22 +781,19 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } + // __class__ is never allowed to be free through a class scope (see + // drop_class_free) + if (scope == FREE && ste->ste_type == ClassBlock && + _PyUnicode_EqualToASCIIString(k, "__class__")) { + scope = GLOBAL_IMPLICIT; + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + remove_dunder_class = 1; + } if (!existing) { // name does not exist in scope, copy from comprehension assert(scope != FREE || PySet_Contains(comp_free, k) == 1); - if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { - // if __class__ is unbound in the enclosing class scope and free - // in the comprehension scope, it needs special handling; just - // letting it be marked as free in class scope will break due to - // drop_class_free - scope = GLOBAL_IMPLICIT; - only_flags &= ~DEF_FREE; - if (PySet_Discard(comp_free, k) < 0) { - return 0; - } - remove_dunder_class = 1; - } PyObject *v_flags = PyLong_FromLong(only_flags); if (v_flags == NULL) { return 0; From e5a7bc6f2eb9a3875063423caa67bb0ffcc3a6b8 Mon Sep 17 00:00:00 2001 From: Clinton Date: Mon, 10 Jun 2024 04:17:50 -0400 Subject: [PATCH 09/21] gh-120296: Fix format string of fcntl.ioctl() audit (#120301) --- Modules/fcntlmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 873bdf2ac0657a..0c06c03a6c403e 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -170,7 +170,7 @@ fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, Py_ssize_t len; char buf[IOCTL_BUFSZ+1]; /* argument plus NUL byte */ - if (PySys_Audit("fcntl.ioctl", "iIO", fd, code, + if (PySys_Audit("fcntl.ioctl", "ikO", fd, code, ob_arg ? ob_arg : Py_None) < 0) { return NULL; } From 4829522b8d3e1a28930f1cccfcc9635e035a0eb4 Mon Sep 17 00:00:00 2001 From: "E. M. Bray" Date: Mon, 10 Jun 2024 10:55:49 +0200 Subject: [PATCH 10/21] bpo-24766: doc= argument to subclasses of property not handled correctly (GH-2487) Co-authored-by: Serhiy Storchaka --- Lib/test/test_property.py | 34 +++++++++++++++++++ .../2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst | 1 + Objects/descrobject.c | 19 +++-------- 3 files changed, 39 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 408e64f53142db..b7a2219b96149a 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -463,6 +463,40 @@ def getter3(self): self.assertEqual(p.__doc__, "user") self.assertEqual(p2.__doc__, "user") + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst new file mode 100644 index 00000000000000..93a8562efe6d6f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-10-09-15-14-53.bpo-24766.c_C1Wc.rst @@ -0,0 +1 @@ +Fix handling of ``doc`` argument to subclasses of ``property``. diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 1b7e2fde3ceccd..4eccd1704eb95a 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -1859,22 +1859,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, /* if no docstring given and the getter has one, use that one */ else if (fget != NULL) { int rc = PyObject_GetOptionalAttr(fget, &_Py_ID(__doc__), &prop_doc); - if (rc <= 0) { + if (rc < 0) { return rc; } - if (!Py_IS_TYPE(self, &PyProperty_Type) && - prop_doc != NULL && prop_doc != Py_None) { - // This oddity preserves the long existing behavior of surfacing - // an AttributeError when using a dict-less (__slots__) property - // subclass as a decorator on a getter method with a docstring. - // See PropertySubclassTest.test_slots_docstring_copy_exception. - int err = PyObject_SetAttr( - (PyObject *)self, &_Py_ID(__doc__), prop_doc); - if (err < 0) { - Py_DECREF(prop_doc); // release our new reference. - return -1; - } - } if (prop_doc == Py_None) { prop_doc = NULL; Py_DECREF(Py_None); @@ -1902,7 +1889,9 @@ property_init_impl(propertyobject *self, PyObject *fget, PyObject *fset, Py_DECREF(prop_doc); if (err < 0) { assert(PyErr_Occurred()); - if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + if (!self->getter_doc && + PyErr_ExceptionMatches(PyExc_AttributeError)) + { PyErr_Clear(); // https://github.com/python/cpython/issues/98963#issuecomment-1574413319 // Python silently dropped this doc assignment through 3.11. From b90bd3e5bbc136f53b24ee791824acd6b17e0d42 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jun 2024 11:54:35 +0200 Subject: [PATCH 11/21] gh-120155: Fix Coverity issue in zoneinfo load_data() (#120232) Declare the 'rv' varaible at the top of the load_data() function to make sure that it's initialized before the first 'goto error' which uses 'rv' (return rv). Fix the Coverity issue: Error: UNINIT (CWE-457): Python-3.12.2/Modules/_zoneinfo.c:1233:5: skipped_decl: Jumping over declaration of ""rv"". Python-3.12.2/Modules/_zoneinfo.c:1284:5: uninit_use: Using uninitialized value ""rv"". 1282| } 1283| 1284|-> return rv; 1285| } 1286| --- Modules/_zoneinfo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index 38c3f0c45d803f..902ece795b575b 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -944,6 +944,7 @@ ttinfo_eq(const _ttinfo *const tti0, const _ttinfo *const tti1) static int load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) { + int rv = 0; PyObject *data_tuple = NULL; long *utcoff = NULL; @@ -1220,7 +1221,6 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } } - int rv = 0; goto cleanup; error: // These resources only need to be freed if we have failed, if we succeed From c3b6dbff2c8886de1edade737febe85dd47ff4d0 Mon Sep 17 00:00:00 2001 From: Pieter Eendebak Date: Mon, 10 Jun 2024 13:06:18 +0200 Subject: [PATCH 12/21] gh-115801: Only allow sequence of strings as input for difflib.unified_diff (GH-118333) --- Lib/difflib.py | 6 ++++ Lib/test/test_difflib.py | 30 +++++++++++++++---- ...-04-27-18-36-46.gh-issue-115801.SVeHSy.rst | 1 + 3 files changed, 31 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst diff --git a/Lib/difflib.py b/Lib/difflib.py index 0443963b4fd697..7f595b6c72e641 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1264,6 +1264,12 @@ def _check_types(a, b, *args): if b and not isinstance(b[0], str): raise TypeError('lines to compare must be str, not %s (%r)' % (type(b[0]).__name__, b[0])) + if isinstance(a, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(a).__name__) + if isinstance(b, str): + raise TypeError('input must be a sequence of strings, not %s' % + type(b).__name__) for arg in args: if not isinstance(arg, str): raise TypeError('all arguments must be str, not: %r' % (arg,)) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index bf6e5b1152b4a2..9e217249be7332 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -295,7 +295,7 @@ def test_close_matches_aligned(self): class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): - args = ['one', 'two', 'Original', 'Current', + args = [['one'], ['two'], 'Original', 'Current', '2005-01-26 23:30:50', '2010-04-02 10:20:52'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], [ @@ -307,7 +307,7 @@ def test_tab_delimiter(self): "--- Current\t2010-04-02 10:20:52"]) def test_no_trailing_tab_on_empty_filedate(self): - args = ['one', 'two', 'Original', 'Current'] + args = [['one'], ['two'], 'Original', 'Current'] ud = difflib.unified_diff(*args, lineterm='') self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"]) @@ -447,6 +447,28 @@ def assertDiff(expect, actual): lineterm=b'') assertDiff(expect, actual) + +class TestInputTypes(unittest.TestCase): + def _assert_type_error(self, msg, generator, *args): + with self.assertRaises(TypeError) as ctx: + list(generator(*args)) + self.assertEqual(msg, str(ctx.exception)) + + def test_input_type_checks(self): + unified = difflib.unified_diff + context = difflib.context_diff + + expect = "input must be a sequence of strings, not str" + self._assert_type_error(expect, unified, 'a', ['b']) + self._assert_type_error(expect, context, 'a', ['b']) + + self._assert_type_error(expect, unified, ['a'], 'b') + self._assert_type_error(expect, context, ['a'], 'b') + + expect = "lines to compare must be str, not NoneType (None)" + self._assert_type_error(expect, unified, ['a'], [None]) + self._assert_type_error(expect, context, ['a'], [None]) + def test_mixed_types_content(self): # type of input content must be consistent: all str or all bytes a = [b'hello'] @@ -495,10 +517,6 @@ def test_mixed_types_dates(self): b = ['bar\n'] list(difflib.unified_diff(a, b, 'a', 'b', datea, dateb)) - def _assert_type_error(self, msg, generator, *args): - with self.assertRaises(TypeError) as ctx: - list(generator(*args)) - self.assertEqual(msg, str(ctx.exception)) class TestJunkAPIs(unittest.TestCase): def test_is_line_junk_true(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst new file mode 100644 index 00000000000000..93b176d5767335 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-27-18-36-46.gh-issue-115801.SVeHSy.rst @@ -0,0 +1 @@ +Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` and :func:`difflib.context_diff`. From 56c3815ba14c790d2e9a227b4ac0ead5e6b1e570 Mon Sep 17 00:00:00 2001 From: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Date: Mon, 10 Jun 2024 16:15:12 +0100 Subject: [PATCH 13/21] gh-119786: copy compiler doc from devguide to InternalDocs and convert to markdown (#120134) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gh-119876: move compiler doc from devguide to InternalDocs Copy of https://github.com/python/devguide/commit/78fc0d7aa9fd0d6733d10c23b178b2a0e2799afc Co-Authored-By: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-Authored-By: Adam Turner <9087854+aa-turner@users.noreply.github.com> Co-Authored-By: Brett Cannon Co-Authored-By: Carol Willing Co-Authored-By: Daniel Porteous Co-Authored-By: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Co-Authored-By: Éric Araujo Co-Authored-By: Erlend Egeberg Aasland Co-Authored-By: Ezio Melotti Co-Authored-By: Georg Brandl Co-Authored-By: Guido van Rossum Co-Authored-By: Hugo van Kemenade Co-Authored-By: Irit Katriel <1055913+iritkatriel@users.noreply.github.com> Co-Authored-By: Jeff Allen Co-Authored-By: Jim Fasarakis-Hilliard Co-Authored-By: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Co-Authored-By: Lincoln <71312724+Lincoln-developer@users.noreply.github.com> Co-Authored-By: Mariatta Co-Authored-By: Muhammad Mahad Co-Authored-By: Ned Deily Co-Authored-By: Pablo Galindo Salgado Co-Authored-By: Serhiy Storchaka Co-Authored-By: Stéphane Wirtel Co-Authored-By: Suriyaa ✌️️ Co-Authored-By: Zachary Ware Co-Authored-By: psyker156 <242220+psyker156@users.noreply.github.com> Co-Authored-By: slateny <46876382+slateny@users.noreply.github.com> Co-Authored-By: svelankar <17737361+svelankar@users.noreply.github.com> Co-Authored-By: zikcheng * convert to markdown * add to index * update more of the out of date stuff --------- Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Brett Cannon Co-authored-by: Carol Willing Co-authored-by: Daniel Porteous Co-authored-by: Dennis Sweeney <36520290+sweeneyde@users.noreply.github.com> Co-authored-by: Éric Araujo Co-authored-by: Erlend Egeberg Aasland Co-authored-by: Ezio Melotti Co-authored-by: Georg Brandl Co-authored-by: Guido van Rossum Co-authored-by: Hugo van Kemenade Co-authored-by: Jeff Allen Co-authored-by: Jim Fasarakis-Hilliard Co-authored-by: Ken Jin <28750310+Fidget-Spinner@users.noreply.github.com> Co-authored-by: Lincoln <71312724+Lincoln-developer@users.noreply.github.com> Co-authored-by: Mariatta Co-authored-by: Muhammad Mahad Co-authored-by: Ned Deily Co-authored-by: Pablo Galindo Salgado Co-authored-by: Serhiy Storchaka Co-authored-by: Stéphane Wirtel Co-authored-by: Suriyaa ✌️️ Co-authored-by: Zachary Ware Co-authored-by: psyker156 <242220+psyker156@users.noreply.github.com> Co-authored-by: slateny <46876382+slateny@users.noreply.github.com> Co-authored-by: svelankar <17737361+svelankar@users.noreply.github.com> Co-authored-by: zikcheng --- InternalDocs/README.md | 2 + InternalDocs/compiler.md | 651 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 653 insertions(+) create mode 100644 InternalDocs/compiler.md diff --git a/InternalDocs/README.md b/InternalDocs/README.md index a2502fbf198735..42f6125794266a 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -12,6 +12,8 @@ it is not, please report that through the [issue tracker](https://github.com/python/cpython/issues). +[Compiler Design](compiler.md) + [Exception Handling](exception_handling.md) [Adaptive Instruction Families](adaptive.md) diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md new file mode 100644 index 00000000000000..0abc10da6e05c6 --- /dev/null +++ b/InternalDocs/compiler.md @@ -0,0 +1,651 @@ + +Compiler design +=============== + +Abstract +-------- + +In CPython, the compilation from source code to bytecode involves several steps: + +1. Tokenize the source code + [Parser/lexer/](https://github.com/python/cpython/blob/main/Parser/lexer/) + and [Parser/tokenizer/](https://github.com/python/cpython/blob/main/Parser/tokenizer/). +2. Parse the stream of tokens into an Abstract Syntax Tree + [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c). +3. Transform AST into an instruction sequence + [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c). +4. Construct a Control Flow Graph and apply optimizations to it + [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c). +5. Emit bytecode based on the Control Flow Graph + [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + +This document outlines how these steps of the process work. + +This document only describes parsing in enough depth to explain what is needed +for understanding compilation. This document provides a detailed, though not +exhaustive, view of the how the entire system works. You will most likely need +to read some source code to have an exact understanding of all details. + + +Parsing +======= + +As of Python 3.9, Python's parser is a PEG parser of a somewhat +unusual design. It is unusual in the sense that the parser's input is a stream +of tokens rather than a stream of characters which is more common with PEG +parsers. + +The grammar file for Python can be found in +[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram). +The definitions for literal tokens (such as ``:``, numbers, etc.) can be found in +[Grammar/Tokens](https://github.com/python/cpython/blob/main/Grammar/Tokens). +Various C files, including +[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) +are generated from these. + +See Also: + +* [Guide to the parser](https://devguide.python.org/internals/parser/index.html) + for a detailed description of the parser. + +* [Changing CPython’s grammar](https://devguide.python.org/developer-workflow/grammar/#grammar) + for a detailed description of the grammar. + + +Abstract syntax trees (AST) +=========================== + + +The abstract syntax tree (AST) is a high-level representation of the +program structure without the necessity of containing the source code; +it can be thought of as an abstract representation of the source code. The +specification of the AST nodes is specified using the Zephyr Abstract +Syntax Definition Language (ASDL) [^1], [^2]. + +The definition of the AST nodes for Python is found in the file +[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl). + +Each AST node (representing statements, expressions, and several +specialized types, like list comprehensions and exception handlers) is +defined by the ASDL. Most definitions in the AST correspond to a +particular source construct, such as an 'if' statement or an attribute +lookup. The definition is independent of its realization in any +particular programming language. + +The following fragment of the Python ASDL construct demonstrates the +approach and syntax: + +``` + module Python + { + stmt = FunctionDef(identifier name, arguments args, stmt* body, + expr* decorators) + | Return(expr? value) | Yield(expr? value) + attributes (int lineno) + } +``` + +The preceding example describes two different kinds of statements and an +expression: function definitions, return statements, and yield expressions. +All three kinds are considered of type ``stmt`` as shown by ``|`` separating +the various kinds. They all take arguments of various kinds and amounts. + +Modifiers on the argument type specify the number of values needed; ``?`` +means it is optional, ``*`` means 0 or more, while no modifier means only one +value for the argument and it is required. ``FunctionDef``, for instance, +takes an ``identifier`` for the *name*, ``arguments`` for *args*, zero or more +``stmt`` arguments for *body*, and zero or more ``expr`` arguments for +*decorators*. + +Do notice that something like 'arguments', which is a node type, is +represented as a single AST node and not as a sequence of nodes as with +stmt as one might expect. + +All three kinds also have an 'attributes' argument; this is shown by the +fact that 'attributes' lacks a '|' before it. + +The statement definitions above generate the following C structure type: + + +``` + typedef struct _stmt *stmt_ty; + + struct _stmt { + enum { FunctionDef_kind=1, Return_kind=2, Yield_kind=3 } kind; + union { + struct { + identifier name; + arguments_ty args; + asdl_seq *body; + } FunctionDef; + + struct { + expr_ty value; + } Return; + + struct { + expr_ty value; + } Yield; + } v; + int lineno; + } +``` + +Also generated are a series of constructor functions that allocate (in +this case) a ``stmt_ty`` struct with the appropriate initialization. The +``kind`` field specifies which component of the union is initialized. The +``FunctionDef()`` constructor function sets 'kind' to ``FunctionDef_kind`` and +initializes the *name*, *args*, *body*, and *attributes* fields. + +See also +[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest) + by Thomas Kluyver. + +Memory management +================= + +Before discussing the actual implementation of the compiler, a discussion of +how memory is handled is in order. To make memory management simple, an **arena** +is used that pools memory in a single location for easy +allocation and removal. This enables the removal of explicit memory +deallocation. Because memory allocation for all needed memory in the compiler +registers that memory with the arena, a single call to free the arena is all +that is needed to completely free all memory used by the compiler. + +In general, unless you are working on the critical core of the compiler, memory +management can be completely ignored. But if you are working at either the +very beginning of the compiler or the end, you need to care about how the arena +works. All code relating to the arena is in either +[Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h) +or [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c). + +``PyArena_New()`` will create a new arena. The returned ``PyArena`` structure +will store pointers to all memory given to it. This does the bookkeeping of +what memory needs to be freed when the compiler is finished with the memory it +used. That freeing is done with ``PyArena_Free()``. This only needs to be +called in strategic areas where the compiler exits. + +As stated above, in general you should not have to worry about memory +management when working on the compiler. The technical details of memory +management have been designed to be hidden from you for most cases. + +The only exception comes about when managing a PyObject. Since the rest +of Python uses reference counting, there is extra support added +to the arena to cleanup each PyObject that was allocated. These cases +are very rare. However, if you've allocated a PyObject, you must tell +the arena about it by calling ``PyArena_AddPyObject()``. + + +Source code to AST +================== + +The AST is generated from source code using the function +``_PyParser_ASTFromString()`` or ``_PyParser_ASTFromFile()`` +[Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c). + +After some checks, a helper function in +[Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) +begins applying production rules on the source code it receives; converting source +code to tokens and matching these tokens recursively to their corresponding rule. The +production rule's corresponding rule function is called on every match. These rule +functions follow the format `xx_rule`. Where *xx* is the grammar rule +that the function handles and is automatically derived from +[Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram) by +[Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py). + +Each rule function in turn creates an AST node as it goes along. It does this +by allocating all the new nodes it needs, calling the proper AST node creation +functions for any required supporting functions and connecting them as needed. +This continues until all nonterminal symbols are replaced with terminals. If an +error occurs, the rule functions backtrack and try another rule function. If +there are no more rules, an error is set and the parsing ends. + +The AST node creation helper functions have the name `_PyAST_{xx}` +where *xx* is the AST node that the function creates. These are defined by the +ASDL grammar and contained in +[Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) +(which is generated by +[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py) +from +[Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl)). +This all leads to a sequence of AST nodes stored in ``asdl_seq`` structs. + +To demonstrate everything explained so far, here's the +rule function responsible for a simple named import statement such as +``import sys``. Note that error-checking and debugging code has been +omitted. Removed parts are represented by ``...``. +Furthermore, some comments have been added for explanation. These comments +may not be present in the actual code. + + +``` + // This is the production rule (from python.gram) the rule function + // corresponds to: + // import_name: 'import' dotted_as_names + static stmt_ty + import_name_rule(Parser *p) + { + ... + stmt_ty _res = NULL; + { // 'import' dotted_as_names + ... + Token * _keyword; + asdl_alias_seq* a; + // The tokenizing steps. + if ( + (_keyword = _PyPegen_expect_token(p, 513)) // token='import' + && + (a = dotted_as_names_rule(p)) // dotted_as_names + ) + { + ... + // Generate an AST for the import statement. + _res = _PyAST_Import ( a , ...); + ... + goto done; + } + ... + } + _res = NULL; + done: + ... + return _res; + } +``` + + +To improve backtracking performance, some rules (chosen by applying a +``(memo)`` flag in the grammar file) are memoized. Each rule function checks if +a memoized version exists and returns that if so, else it continues in the +manner stated in the previous paragraphs. + +There are macros for creating and using ``asdl_xx_seq *`` types, where *xx* is +a type of the ASDL sequence. Three main types are defined +manually -- ``generic``, ``identifier`` and ``int``. These types are found in +[Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c) +and its corresponding header file +[Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h). +Functions and macros for creating ``asdl_xx_seq *`` types are as follows: + +``_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_generic_seq`` of the specified length +``_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_identifier_seq`` of the specified length +``_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)`` + Allocate memory for an ``asdl_int_seq`` of the specified length + +In addition to the three types mentioned above, some ASDL sequence types are +automatically generated by +[Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py) +and found in +[Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h). +Macros for using both manually defined and automatically generated ASDL +sequence types are as follows: + +``asdl_seq_GET(asdl_xx_seq *, int)`` + Get item held at a specific position in an ``asdl_xx_seq`` +``asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)`` + Set a specific index in an ``asdl_xx_seq`` to the specified value + +Untyped counterparts exist for some of the typed macros. These are useful +when a function needs to manipulate a generic ASDL sequence: + +``asdl_seq_GET_UNTYPED(asdl_seq *, int)`` + Get item held at a specific position in an ``asdl_seq`` +``asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)`` + Set a specific index in an ``asdl_seq`` to the specified value +``asdl_seq_LEN(asdl_seq *)`` + Return the length of an ``asdl_seq`` or ``asdl_xx_seq`` + +Note that typed macros and functions are recommended over their untyped +counterparts. Typed macros carry out checks in debug mode and aid +debugging errors caused by incorrectly casting from ``void *``. + +If you are working with statements, you must also worry about keeping +track of what line number generated the statement. Currently the line +number is passed as the last parameter to each ``stmt_ty`` function. + +See also [PEP 617: New PEG parser for CPython](https://peps.python.org/pep-0617/). + + +Control flow graphs +=================== + +A **control flow graph** (often referenced by its acronym, **CFG**) is a +directed graph that models the flow of a program. A node of a CFG is +not an individual bytecode instruction, but instead represents a +sequence of bytecode instructions that always execute sequentially. +Each node is called a *basic block* and must always execute from +start to finish, with a single entry point at the beginning and a +single exit point at the end. If some bytecode instruction *a* needs +to jump to some other bytecode instruction *b*, then *a* must occur at +the end of its basic block, and *b* must occur at the start of its +basic block. + +As an example, consider the following code snippet: + +.. code-block:: Python + + if x < 10: + f1() + f2() + else: + g() + end() + +The ``x < 10`` guard is represented by its own basic block that +compares ``x`` with ``10`` and then ends in a conditional jump based on +the result of the comparison. This conditional jump allows the block +to point to both the body of the ``if`` and the body of the ``else``. The +``if`` basic block contains the ``f1()`` and ``f2()`` calls and points to +the ``end()`` basic block. The ``else`` basic block contains the ``g()`` +call and similarly points to the ``end()`` block. + +Note that more complex code in the guard, the ``if`` body, or the ``else`` +body may be represented by multiple basic blocks. For instance, +short-circuiting boolean logic in a guard like ``if x or y:`` +will produce one basic block that tests the truth value of ``x`` +and then points both (1) to the start of the ``if`` body and (2) to +a different basic block that tests the truth value of y. + +CFGs are useful as an intermediate representation of the code because +they are a convenient data structure for optimizations. + +AST to CFG to bytecode +====================== + +The conversion of an ``AST`` to bytecode is initiated by a call to the function +``_PyAST_Compile()`` in +[Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c). + +The first step is to construct the symbol table. This is implemented by +``_PySymtable_Build()`` in +[Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c). +This function begins by entering the starting code block for the AST (passed-in) +and then calling the proper `symtable_visit_{xx}` function (with *xx* being the +AST node type). Next, the AST tree is walked with the various code blocks that +delineate the reach of a local variable as blocks are entered and exited using +``symtable_enter_block()`` and ``symtable_exit_block()``, respectively. + +Once the symbol table is created, the ``AST`` is transformed by ``compiler_codegen()`` +in [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c) +into a sequence of pseudo instructions. These are similar to bytecode, but +in some cases they are more abstract, and are resolved later into actual +bytecode. The construction of this instruction sequence is handled by several +functions that break the task down by various AST node types. The functions are +all named `compiler_visit_{xx}` where *xx* is the name of the node type (such +as ``stmt``, ``expr``, etc.). Each function receives a ``struct compiler *`` +and `{xx}_ty` where *xx* is the AST node type. Typically these functions +consist of a large 'switch' statement, branching based on the kind of +node type passed to it. Simple things are handled inline in the +'switch' statement with more complex transformations farmed out to other +functions named `compiler_{xx}` with *xx* being a descriptive name of what is +being handled. + +When transforming an arbitrary AST node, use the ``VISIT()`` macro. +The appropriate `compiler_visit_{xx}` function is called, based on the value +passed in for (so `VISIT({c}, expr, {node})` calls +`compiler_visit_expr({c}, {node})`). The ``VISIT_SEQ()`` macro is very similar, +but is called on AST node sequences (those values that were created as +arguments to a node that used the '*' modifier). + +Emission of bytecode is handled by the following macros: + +* ``ADDOP(struct compiler *, location, int)`` + add a specified opcode +* ``ADDOP_IN_SCOPE(struct compiler *, location, int)`` + like ``ADDOP``, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* ``ADDOP_I(struct compiler *, location, int, Py_ssize_t)`` + add an opcode that takes an integer argument +* ``ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)`` + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (``names`` or ``varnames``) +* ``ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)`` + just like ``ADDOP_O``, but steals a reference to PyObject +* ``ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)`` + just like ``ADDOP_O``, but name mangling is also handled; used for + attribute loading or importing based on name +* ``ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)`` + add the ``LOAD_CONST`` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* ``ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)`` + just like ``ADDOP_LOAD_CONST_NEW``, but steals a reference to PyObject +* ``ADDOP_JUMP(struct compiler *, location, int, basicblock *)`` + create a jump to a basic block + +The ``location`` argument is a struct with the source location to be +associated with this instruction. It is typically extracted from an +``AST`` node with the ``LOC`` macro. The ``NO_LOCATION`` can be used +for *synthetic* instructions, which we do not associate with a line +number at this stage. For example, the implicit ``return None`` +which is added at the end of a function is not associated with any +line in the source code. + +There are several helper functions that will emit pseudo-instructions +and are named `compiler_{xx}()` where *xx* is what the function helps +with (``list``, ``boolop``, etc.). A rather useful one is ``compiler_nameop()``. +This function looks up the scope of a variable and, based on the +expression context, emits the proper opcode to load, store, or delete +the variable. + +Once the instruction sequence is created, it is transformed into a CFG +by ``_PyCfg_FromInstructionSequence()``. Then ``_PyCfg_OptimizeCodeUnit()`` +applies various peephole optimizations, and +``_PyCfg_OptimizedCfgToInstructionSequence()`` converts the optimized ``CFG`` +back into an instruction sequence. These conversions and optimizations are +implemented in +[Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c). + +Finally, the sequence of pseudo-instructions is converted into actual +bytecode. This includes transforming pseudo instructions into actual instructions, +converting jump targets from logical labels to relative offsets, and +construction of the +[exception table](exception_handling.md) and +[locations table](https://github.com/python/cpython/blob/main/Objects/locations.md). +The bytecode and tables are then wrapped into a ``PyCodeObject`` along with additional +metadata, including the ``consts`` and ``names`` arrays, information about function +reference to the source code (filename, etc). All of this is implemented by +``_PyAssemble_MakeCodeObject()`` in +[Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c). + + +Code objects +============ + +The result of ``PyAST_CompileObject()`` is a ``PyCodeObject`` which is defined in +[Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h). +And with that you now have executable Python bytecode! + +The code objects (byte code) are executed in +[Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c). +This file will also need a new case statement for the new opcode in the big switch +statement in ``_PyEval_EvalFrameDefault()``. + + +Important files +=============== + +* [Parser/](https://github.com/python/cpython/blob/main/Parser/) + + * [Parser/Python.asdl](https://github.com/python/cpython/blob/main/Parser/Python.asdl): + ASDL syntax file. + + * [Parser/asdl.py](https://github.com/python/cpython/blob/main/Parser/asdl.py): + Parser for ASDL definition files. + Reads in an ASDL description and parses it into an AST that describes it. + + * [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py): + Generate C code from an ASDL description. Generates + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + and + [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h). + + * [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c): + The new PEG parser introduced in Python 3.9. + Generated by + [Tools/peg_generator/pegen/c_generator.py](https://github.com/python/cpython/blob/main/Tools/peg_generator/pegen/c_generator.py) + from the grammar [Grammar/python.gram](https://github.com/python/cpython/blob/main/Grammar/python.gram). + Creates the AST from source code. Rule functions for their corresponding production + rules are found here. + + * [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c): + Contains high-level functions which are + used by the interpreter to create an AST from source code. + + * [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c): + Contains helper functions which are used by functions in + [Parser/parser.c](https://github.com/python/cpython/blob/main/Parser/parser.c) + to construct the AST. Also contains helper functions which help raise better error messages + when parsing source code. + + * [Parser/pegen.h](https://github.com/python/cpython/blob/main/Parser/pegen.h): + Header file for the corresponding + [Parser/pegen.c](https://github.com/python/cpython/blob/main/Parser/pegen.c). + Also contains definitions of the ``Parser`` and ``Token`` structs. + +* [Python/](https://github.com/python/cpython/blob/main/Python) + + * [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c): + Creates C structs corresponding to the ASDL types. Also contains code for + marshalling AST nodes (core ASDL types have marshalling code in + [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c)). + "File automatically generated by + [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py). + This file must be committed separately after every grammar change + is committed since the ``__version__`` value is set to the latest + grammar change revision number. + + * [Python/asdl.c](https://github.com/python/cpython/blob/main/Python/asdl.c): + Contains code to handle the ASDL sequence type. + Also has code to handle marshalling the core ASDL types, such as number + and identifier. Used by + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + for marshalling AST nodes. + + * [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c): + Used for validating the AST. + + * [Python/ast_opt.c](https://github.com/python/cpython/blob/main/Python/ast_opt.c): + Optimizes the AST. + + * [Python/ast_unparse.c](https://github.com/python/cpython/blob/main/Python/ast_unparse.c): + Converts the AST expression node back into a string (for string annotations). + + * [Python/ceval.c](https://github.com/python/cpython/blob/main/Python/ceval.c): + Executes byte code (aka, eval loop). + + * [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c): + Generates a symbol table from AST. + + * [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c): + Implementation of the arena memory manager. + + * [Python/compile.c](https://github.com/python/cpython/blob/main/Python/compile.c): + Emits pseudo bytecode based on the AST. + + * [Python/flowgraph.c](https://github.com/python/cpython/blob/main/Python/flowgraph.c): + Implements peephole optimizations. + + * [Python/assemble.c](https://github.com/python/cpython/blob/main/Python/assemble.c): + Constructs a code object from a sequence of pseudo instructions. + + * [Python/instruction_sequence.c.c](https://github.com/python/cpython/blob/main/Python/instruction_sequence.c.c): + A data structure representing a sequence of bytecode-like pseudo-instructions. + +* [Include/](https://github.com/python/cpython/blob/main/Include/) + + * [Include/cpython/code.h](https://github.com/python/cpython/blob/main/Include/cpython/code.h) + : Header file for + [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c); + contains definition of ``PyCodeObject``. + + * [Include/opcode.h](https://github.com/python/cpython/blob/main/Include/opcode.h) + : One of the files that must be modified if + [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) is. + + * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h) + : Contains the actual definitions of the C structs as generated by + [Python/Python-ast.c](https://github.com/python/cpython/blob/main/Python/Python-ast.c) + "Automatically generated by + [Parser/asdl_c.py](https://github.com/python/cpython/blob/main/Parser/asdl_c.py). + + * [Include/internal/pycore_asdl.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_asdl.h) + : Header for the corresponding + [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c). + + * [Include/internal/pycore_ast.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_ast.h) + : Declares ``_PyAST_Validate()`` external (from + [Python/ast.c](https://github.com/python/cpython/blob/main/Python/ast.c)). + + * [Include/internal/pycore_symtable.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_symtable.h) + : Header for + [Python/symtable.c](https://github.com/python/cpython/blob/main/Python/symtable.c). + ``struct symtable`` and ``PySTEntryObject`` are defined here. + + * [Include/internal/pycore_parser.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_parser.h) + : Header for the corresponding + [Parser/peg_api.c](https://github.com/python/cpython/blob/main/Parser/peg_api.c). + + * [Include/internal/pycore_pyarena.h](https://github.com/python/cpython/blob/main/Include/internal/pycore_pyarena.h) + : Header file for the corresponding + [Python/pyarena.c](https://github.com/python/cpython/blob/main/Python/pyarena.c). + + * [Include/opcode_ids.h](https://github.com/python/cpython/blob/main/Include/opcode_ids.h) + : List of opcodes. Generated from + [Python/bytecodes.c](https://github.com/python/cpython/blob/main/Python/bytecodes.c) + by + [Tools/cases_generator/opcode_id_generator.py](https://github.com/python/cpython/blob/main/Tools/cases_generator/opcode_id_generator.py). + +* [Objects/](https://github.com/python/cpython/blob/main/Objects/) + + * [Objects/codeobject.c](https://github.com/python/cpython/blob/main/Objects/codeobject.c) + : Contains PyCodeObject-related code. + + * [Objects/frameobject.c](https://github.com/python/cpython/blob/main/Objects/frameobject.c) + : Contains the ``frame_setlineno()`` function which should determine whether it is allowed + to make a jump between two points in a bytecode. + +* [Lib/](https://github.com/python/cpython/blob/main/Lib/) + + * [Lib/opcode.py](https://github.com/python/cpython/blob/main/Lib/opcode.py) + : opcode utilities exposed to Python. + + * [Lib/importlib/_bootstrap_external.py](https://github.com/python/cpython/blob/main/Lib/importlib/_bootstrap_external.py) + : Home of the magic number (named ``MAGIC_NUMBER``) for bytecode versioning. + + +Objects +======= + +* [Objects/locations.md](https://github.com/python/cpython/blob/main/Objects/locations.md): Describes the location table +* [Objects/frame_layout.md](https://github.com/python/cpython/blob/main/Objects/frame_layout.md): Describes the frame stack +* [Objects/object_layout.md](https://github.com/python/cpython/blob/main/Objects/object_layout.md): Descibes object layout for 3.11 and later +* [Exception Handling](exception_handling.md): Describes the exception table + + +Specializing Adaptive Interpreter +================================= + +Adding a specializing, adaptive interpreter to CPython will bring significant +performance improvements. These documents provide more information: + +* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/). +* [Adding or extending a family of adaptive instructions](adaptive.md) + + +References +========== + +[^1]: Daniel C. Wang, Andrew W. Appel, Jeff L. Korn, and Chris + S. Serra. `The Zephyr Abstract Syntax Description Language.`_ + In Proceedings of the Conference on Domain-Specific Languages, + pp. 213--227, 1997. + +[^2]: The Zephyr Abstract Syntax Description Language.: + https://www.cs.princeton.edu/research/techreps/TR-554-97 From 7aff2de62bc28eb23888270b698c6b6915f69b21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 10 Jun 2024 18:34:17 +0200 Subject: [PATCH 14/21] gh-120057: Add os.environ.refresh() method (#120059) --- Doc/library/os.rst | 11 +++++ Doc/whatsnew/3.14.rst | 7 +++ Lib/os.py | 25 ++++++++-- Lib/test/test_os.py | 46 +++++++++++++++++++ ...-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst | 4 ++ Modules/clinic/posixmodule.c.h | 20 +++++++- Modules/posixmodule.c | 15 ++++++ 7 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index b93b06d4e72afc..360d71e70960c7 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -193,6 +193,10 @@ process and user. to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. + The :meth:`!os.environ.refresh()` method updates :data:`os.environ` with + changes to the environment made by :func:`os.putenv`, by + :func:`os.unsetenv`, or made outside Python in the same process. + This mapping may be used to modify the environment as well as query the environment. :func:`putenv` will be called automatically when the mapping is modified. @@ -225,6 +229,9 @@ process and user. .. versionchanged:: 3.9 Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. + .. versionchanged:: 3.14 + Added the :meth:`!os.environ.refresh()` method. + .. data:: environb @@ -561,6 +568,8 @@ process and user. of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which respectively use :data:`os.environ` and :data:`os.environb` in their implementations. + See also the :data:`os.environ.refresh() ` method. + .. note:: On some platforms, including FreeBSD and macOS, setting ``environ`` may @@ -809,6 +818,8 @@ process and user. don't update :data:`os.environ`, so it is actually preferable to delete items of :data:`os.environ`. + See also the :data:`os.environ.refresh() ` method. + .. audit-event:: os.unsetenv key os.unsetenv .. versionchanged:: 3.9 diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b2dd80b64a691a..b77ff30a8fbbee 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -92,6 +92,13 @@ ast Added :func:`ast.compare` for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :issue:`15987`.) +os +-- + +* Added the :data:`os.environ.refresh() ` method to update + :data:`os.environ` with changes to the environment made by :func:`os.putenv`, + by :func:`os.unsetenv`, or made outside Python in the same process. + (Contributed by Victor Stinner in :gh:`120057`.) Optimizations diff --git a/Lib/os.py b/Lib/os.py index 0408e2db79e66e..4b48afb040e565 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -64,6 +64,10 @@ def _get_exports_list(module): from posix import _have_functions except ImportError: pass + try: + from posix import _create_environ + except ImportError: + pass import posix __all__.extend(_get_exports_list(posix)) @@ -88,6 +92,10 @@ def _get_exports_list(module): from nt import _have_functions except ImportError: pass + try: + from nt import _create_environ + except ImportError: + pass else: raise ImportError('no os specific module found') @@ -773,7 +781,18 @@ def __ror__(self, other): new.update(self) return new -def _createenviron(): + if _exists("_create_environ"): + def refresh(self): + data = _create_environ() + if name == 'nt': + data = {self.encodekey(key): value + for key, value in data.items()} + + # modify in-place to keep os.environb in sync + self._data.clear() + self._data.update(data) + +def _create_environ_mapping(): if name == 'nt': # Where Env Var Names Must Be UPPERCASE def check_str(value): @@ -803,8 +822,8 @@ def decode(value): encode, decode) # unicode environ -environ = _createenviron() -del _createenviron +environ = _create_environ_mapping() +del _create_environ_mapping def getenv(key, default=None): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2beb9ca8aa6ccb..f93937fb587386 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1298,6 +1298,52 @@ def test_ror_operator(self): self._test_underlying_process_env('_A_', '') self._test_underlying_process_env(overridden_key, original_value) + def test_refresh(self): + # Test os.environ.refresh() + has_environb = hasattr(os, 'environb') + + # Test with putenv() which doesn't update os.environ + os.environ['test_env'] = 'python_value' + os.putenv("test_env", "new_value") + self.assertEqual(os.environ['test_env'], 'python_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'python_value') + + os.environ.refresh() + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + # Test with unsetenv() which doesn't update os.environ + os.unsetenv('test_env') + self.assertEqual(os.environ['test_env'], 'new_value') + if has_environb: + self.assertEqual(os.environb[b'test_env'], b'new_value') + + os.environ.refresh() + self.assertNotIn('test_env', os.environ) + if has_environb: + self.assertNotIn(b'test_env', os.environb) + + if has_environb: + # test os.environb.refresh() with putenv() + os.environb[b'test_env'] = b'python_value2' + os.putenv("test_env", "new_value2") + self.assertEqual(os.environb[b'test_env'], b'python_value2') + self.assertEqual(os.environ['test_env'], 'python_value2') + + os.environb.refresh() + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + # test os.environb.refresh() with unsetenv() + os.unsetenv('test_env') + self.assertEqual(os.environb[b'test_env'], b'new_value2') + self.assertEqual(os.environ['test_env'], 'new_value2') + + os.environb.refresh() + self.assertNotIn(b'test_env', os.environb) + self.assertNotIn('test_env', os.environ) class WalkTests(unittest.TestCase): """Tests for os.walk().""" diff --git a/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst new file mode 100644 index 00000000000000..955be59821ee0c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-18-53-10.gh-issue-120057.RSD9_Z.rst @@ -0,0 +1,4 @@ +Added the :data:`os.environ.refresh() ` method to update +:data:`os.environ` with changes to the environment made by :func:`os.putenv`, +by :func:`os.unsetenv`, or made outside Python in the same process. +Patch by Victor Stinner. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 69fc178331c09c..07b28fef3a57ea 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -12152,6 +12152,24 @@ os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) return os__is_inputhook_installed_impl(module); } +PyDoc_STRVAR(os__create_environ__doc__, +"_create_environ($module, /)\n" +"--\n" +"\n" +"Create the environment dictionary."); + +#define OS__CREATE_ENVIRON_METHODDEF \ + {"_create_environ", (PyCFunction)os__create_environ, METH_NOARGS, os__create_environ__doc__}, + +static PyObject * +os__create_environ_impl(PyObject *module); + +static PyObject * +os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__create_environ_impl(module); +} + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -12819,4 +12837,4 @@ os__is_inputhook_installed(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=faaa5e5ffb7b165d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5ae2e5ffcd9c8a84 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5f943d4b1c8085..a8fd5c494769b5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -16809,6 +16809,20 @@ os__is_inputhook_installed_impl(PyObject *module) return PyBool_FromLong(PyOS_InputHook != NULL); } +/*[clinic input] +os._create_environ + +Create the environment dictionary. +[clinic start generated code]*/ + +static PyObject * +os__create_environ_impl(PyObject *module) +/*[clinic end generated code: output=19d9039ab14f8ad4 input=a4c05686b34635e8]*/ +{ + return convertenviron(); +} + + static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF @@ -17023,6 +17037,7 @@ static PyMethodDef posix_methods[] = { OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF OS__INPUTHOOK_METHODDEF OS__IS_INPUTHOOK_INSTALLED_METHODDEF + OS__CREATE_ENVIRON_METHODDEF {NULL, NULL} /* Sentinel */ }; From 6efe3460693c4f39de198a64cebeeee8b1d4e8b6 Mon Sep 17 00:00:00 2001 From: AN Long Date: Tue, 11 Jun 2024 00:45:16 +0800 Subject: [PATCH 15/21] Fix the CODEOWNERS for _interpretersmodule.c (gh-120288) --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 811b8cfdab17dc..8bc40fcb9e8999 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -245,7 +245,7 @@ Doc/howto/clinic.rst @erlend-aasland **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently Lib/test/support/interpreters/ @ericsnowcurrently -Modules/_xx*interp*module.c @ericsnowcurrently +Modules/_interp*module.c @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently # Android From 422c4fc855afd18bcc6415902ea1d85a50cb7ce1 Mon Sep 17 00:00:00 2001 From: Robert Collins Date: Tue, 11 Jun 2024 07:41:12 +0200 Subject: [PATCH 16/21] gh-119600: mock: do not access attributes of original when new_callable is set (#119601) In order to patch flask.g e.g. as in #84982, that proxies getattr must not be invoked. For that, mock must not try to read from the original object. In some cases that is unavoidable, e.g. when doing autospec. However, patch("flask.g", new_callable=MagicMock) should be entirely safe. --- Lib/test/test_unittest/testmock/support.py | 11 +++++++++++ Lib/test/test_unittest/testmock/testpatch.py | 7 +++++++ Lib/unittest/mock.py | 14 +++++++++----- .../2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst | 2 ++ 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst diff --git a/Lib/test/test_unittest/testmock/support.py b/Lib/test/test_unittest/testmock/support.py index 49986d65dc47af..6c535b7944f261 100644 --- a/Lib/test/test_unittest/testmock/support.py +++ b/Lib/test/test_unittest/testmock/support.py @@ -14,3 +14,14 @@ def wibble(self): pass class X(object): pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/test/test_unittest/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py index be75fda7826af1..f26e74ce0bc1ba 100644 --- a/Lib/test/test_unittest/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -2045,6 +2045,13 @@ def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 3ef83e263f53b7..edabb4520c13cd 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -1508,13 +1508,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1527,7 +1526,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: diff --git a/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst new file mode 100644 index 00000000000000..04c9ca9c3fd737 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-10-14-00-40.gh-issue-119600.jJMf4C.rst @@ -0,0 +1,2 @@ +Fix :func:`unittest.mock.patch` to not read attributes of the target when +``new_callable`` is set. Patch by Robert Collins. From 9e9ee50421c857b443e2060274f17fb884d54473 Mon Sep 17 00:00:00 2001 From: blhsing Date: Tue, 11 Jun 2024 13:42:49 +0800 Subject: [PATCH 17/21] gh-65454: avoid triggering call to a PropertyMock in NonCallableMock.__setattr__ (#120019) --- Lib/test/test_unittest/testmock/testhelpers.py | 8 ++++++++ Lib/unittest/mock.py | 3 +++ .../Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst | 1 + 3 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index 74785a83757a92..c9c20f008ca5a2 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1127,6 +1127,14 @@ def test_propertymock_side_effect(self): p.assert_called_once_with() + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index edabb4520c13cd..08975e0e1bd132 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -830,6 +830,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) diff --git a/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst new file mode 100644 index 00000000000000..0b232cf8ca1baf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-04-08-57-02.gh-issue-65454.o9j4wF.rst @@ -0,0 +1 @@ +:func:`unittest.mock.Mock.attach_mock` no longer triggers a call to a ``PropertyMock`` being attached. From 141babad9b4eceb83371bf19ba3a36b50dd05250 Mon Sep 17 00:00:00 2001 From: Nikita Sobolev Date: Tue, 11 Jun 2024 10:04:27 +0300 Subject: [PATCH 18/21] gh-120298: Fix use-after-free in `list_richcompare_impl` (#120303) Co-authored-by: Serhiy Storchaka --- Lib/test/test_list.py | 11 +++++++++++ .../2024-06-10-10-42-48.gh-issue-120298.napREA.rst | 2 ++ Objects/listobject.c | 9 ++++++++- 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 0601b33e79ebb6..d21429fae09b37 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -234,6 +234,17 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) + def test_lt_operator_modifying_operand(self): + # See gh-120298 + class evil: + def __lt__(self, other): + other.clear() + return NotImplemented + + a = [[evil()]] + with self.assertRaises(TypeError): + a[0] < a + @cpython_only def test_preallocation(self): iterable = [0] * 10 diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst new file mode 100644 index 00000000000000..531d39517ac423 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-06-10-10-42-48.gh-issue-120298.napREA.rst @@ -0,0 +1,2 @@ +Fix use-after free in ``list_richcompare_impl`` which can be invoked via +some specificly tailored evil input. diff --git a/Objects/listobject.c b/Objects/listobject.c index d09bb6391034d1..6829d5d28656cf 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3382,7 +3382,14 @@ list_richcompare_impl(PyObject *v, PyObject *w, int op) } /* Compare the final item again using the proper operator */ - return PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + PyObject *vitem = vl->ob_item[i]; + PyObject *witem = wl->ob_item[i]; + Py_INCREF(vitem); + Py_INCREF(witem); + PyObject *result = PyObject_RichCompare(vl->ob_item[i], wl->ob_item[i], op); + Py_DECREF(vitem); + Py_DECREF(witem); + return result; } static PyObject * From 7d2447137e117ea9a6ee1493bce0b071c76b1bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20G=C3=B3rny?= Date: Tue, 11 Jun 2024 09:11:13 +0200 Subject: [PATCH 19/21] gh-120291: Fix a bashism in python-config.sh.in (#120292) gh-120291: Fix bashisms in python-config.sh.in Replace the use of bash-specific `[[ ... ]]` with POSIX-compliant `[ ... ]` to make the `python-config` shell script work with non-bash shells again. While at it, use `local` in a safer way, since it is not in POSIX either (though universally supported). Fixes #120291 --- .../Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst | 1 + Misc/python-config.sh.in | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst diff --git a/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst new file mode 100644 index 00000000000000..d0bb297b51dc6e --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-06-09-15-54-22.gh-issue-120291.IpfHzE.rst @@ -0,0 +1 @@ +Make the ``python-config`` shell script compatible with non-bash shells. diff --git a/Misc/python-config.sh.in b/Misc/python-config.sh.in index c3c0b34fc1451d..9929f5b2653dca 100644 --- a/Misc/python-config.sh.in +++ b/Misc/python-config.sh.in @@ -4,11 +4,12 @@ exit_with_usage () { - local USAGE="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" - if [[ "$1" -eq 0 ]]; then - echo "$USAGE" + local usage + usage="Usage: $0 --prefix|--exec-prefix|--includes|--libs|--cflags|--ldflags|--extension-suffix|--help|--abiflags|--configdir|--embed" + if [ "$1" -eq 0 ]; then + echo "$usage" else - echo "$USAGE" >&2 + echo "$usage" >&2 fi exit $1 } From 02c1dfff073a3dd6ce34a11b038defde291c2203 Mon Sep 17 00:00:00 2001 From: Kirill Podoprigora Date: Tue, 11 Jun 2024 10:56:38 +0300 Subject: [PATCH 20/21] gh-120080: Mark test_round_with_none_arg_direct_call as cpython_only (#120328) --- Lib/test/test_float.py | 1 + Lib/test/test_int.py | 1 + 2 files changed, 2 insertions(+) diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 53695cefb8fded..756cf9bd7719c0 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -949,6 +949,7 @@ def test_None_ndigits(self): self.assertEqual(x, 2) self.assertIsInstance(x, int) + @support.cpython_only def test_round_with_none_arg_direct_call(self): for val in [(1.0).__round__(None), round(1.0), diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 77221dfb6d5aa2..2747d9219255ac 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -517,6 +517,7 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7_8_9', 16), 0x123456789) self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) + @support.cpython_only def test_round_with_none_arg_direct_call(self): for val in [(1).__round__(None), round(1), From 9b8611eeea172cd4aa626ccd1ca333dc4093cd8c Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 11 Jun 2024 07:06:49 -0600 Subject: [PATCH 21/21] gh-119180: PEP 649 compiler changes (#119361) --- .../pycore_global_objects_fini_generated.h | 2 +- Include/internal/pycore_global_strings.h | 2 +- Include/internal/pycore_opcode_utils.h | 1 + .../internal/pycore_runtime_init_generated.h | 2 +- Include/internal/pycore_symtable.h | 8 +- .../internal/pycore_unicodeobject_generated.h | 3 - Lib/inspect.py | 8 +- Lib/symtable.py | 2 + Lib/test/test_dis.py | 29 +- Lib/test/test_grammar.py | 69 +--- Lib/test/test_module/__init__.py | 2 + Lib/test/test_opcodes.py | 13 +- Lib/test/test_positional_only_arg.py | 5 +- Lib/test/test_pyclbr.py | 2 + Lib/test/test_pydoc/test_pydoc.py | 11 +- Lib/test/test_pyrepl/test_interact.py | 2 +- Lib/test/test_symtable.py | 6 +- Lib/test/test_traceback.py | 5 +- Lib/test/test_type_annotations.py | 163 +++++++- Lib/test/test_typing.py | 4 +- Lib/test/typinganndata/ann_module.py | 4 - Lib/typing.py | 23 +- ...-05-22-06-22-47.gh-issue-119180.vZMiXm.rst | 1 + Python/bytecodes.c | 5 + Python/compile.c | 357 ++++++++++-------- Python/executor_cases.c.h | 5 + Python/generated_cases.c.h | 5 + Python/symtable.c | 198 +++++++--- 28 files changed, 609 insertions(+), 328 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index b186408931c92e..30851dc2dbec44 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -559,6 +559,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(dot_locals)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(empty)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(format)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(generic_base)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(json_decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_STR(kwdefaults)); @@ -745,7 +746,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_abstract_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_active)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_align_)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_annotation)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_anonymous_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_argtypes_)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(_as_parameter_)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index e1808c85acfb2d..009802c441685c 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -45,6 +45,7 @@ struct _Py_global_strings { STRUCT_FOR_STR(dot, ".") STRUCT_FOR_STR(dot_locals, ".") STRUCT_FOR_STR(empty, "") + STRUCT_FOR_STR(format, ".format") STRUCT_FOR_STR(generic_base, ".generic_base") STRUCT_FOR_STR(json_decoder, "json.decoder") STRUCT_FOR_STR(kwdefaults, ".kwdefaults") @@ -234,7 +235,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(_abstract_) STRUCT_FOR_ID(_active) STRUCT_FOR_ID(_align_) - STRUCT_FOR_ID(_annotation) STRUCT_FOR_ID(_anonymous_) STRUCT_FOR_ID(_argtypes_) STRUCT_FOR_ID(_as_parameter_) diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index b06e469dd5bd91..e76f4840a66891 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -57,6 +57,7 @@ extern "C" { #define MAKE_FUNCTION_KWDEFAULTS 0x02 #define MAKE_FUNCTION_ANNOTATIONS 0x04 #define MAKE_FUNCTION_CLOSURE 0x08 +#define MAKE_FUNCTION_ANNOTATE 0x10 /* Values used as the oparg for LOAD_COMMON_CONSTANT */ #define CONSTANT_ASSERTIONERROR 0 diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 2dde6febc2cae4..ff5b6ee8e0f006 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -554,6 +554,7 @@ extern "C" { INIT_STR(dot, "."), \ INIT_STR(dot_locals, "."), \ INIT_STR(empty, ""), \ + INIT_STR(format, ".format"), \ INIT_STR(generic_base, ".generic_base"), \ INIT_STR(json_decoder, "json.decoder"), \ INIT_STR(kwdefaults, ".kwdefaults"), \ @@ -743,7 +744,6 @@ extern "C" { INIT_ID(_abstract_), \ INIT_ID(_active), \ INIT_ID(_align_), \ - INIT_ID(_annotation), \ INIT_ID(_anonymous_), \ INIT_ID(_argtypes_), \ INIT_ID(_as_parameter_), \ diff --git a/Include/internal/pycore_symtable.h b/Include/internal/pycore_symtable.h index ac6c499c08264e..5d544765237df5 100644 --- a/Include/internal/pycore_symtable.h +++ b/Include/internal/pycore_symtable.h @@ -12,8 +12,9 @@ struct _mod; // Type defined in pycore_ast.h typedef enum _block_type { FunctionBlock, ClassBlock, ModuleBlock, - // Used for annotations if 'from __future__ import annotations' is active. - // Annotation blocks cannot bind names and are not evaluated. + // Used for annotations. If 'from __future__ import annotations' is active, + // annotation blocks cannot bind names and are not evaluated. Otherwise, they + // are lazily evaluated (see PEP 649). AnnotationBlock, // Used for generics and type aliases. These work mostly like functions // (see PEP 695 for details). The three different blocks function identically; @@ -89,6 +90,7 @@ typedef struct _symtable_entry { including free refs to globals */ unsigned ste_generator : 1; /* true if namespace is a generator */ unsigned ste_coroutine : 1; /* true if namespace is a coroutine */ + unsigned ste_annotations_used : 1; /* true if there are any annotations in this scope */ _Py_comprehension_ty ste_comprehension; /* Kind of comprehension (if any) */ unsigned ste_varargs : 1; /* true if block has varargs */ unsigned ste_varkeywords : 1; /* true if block has varkeywords */ @@ -110,6 +112,7 @@ typedef struct _symtable_entry { int ste_end_col_offset; /* end offset of first line of block */ int ste_opt_lineno; /* lineno of last exec or import * */ int ste_opt_col_offset; /* offset of last exec or import * */ + struct _symtable_entry *ste_annotation_block; /* symbol table entry for this entry's annotations */ struct symtable *ste_table; } PySTEntryObject; @@ -126,6 +129,7 @@ extern struct symtable* _PySymtable_Build( PyObject *filename, _PyFutureFeatures *future); extern PySTEntryObject* _PySymtable_Lookup(struct symtable *, void *); +extern int _PySymtable_LookupOptional(struct symtable *, void *, PySTEntryObject **); extern void _PySymtable_Free(struct symtable *); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index b00119a1bad7ff..69d93a9610a2e5 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -543,9 +543,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { string = &_Py_ID(_align_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); - string = &_Py_ID(_annotation); - assert(_PyUnicode_CheckConsistency(string, 1)); - _PyUnicode_InternInPlace(interp, &string); string = &_Py_ID(_anonymous_); assert(_PyUnicode_CheckConsistency(string, 1)); _PyUnicode_InternInPlace(interp, &string); diff --git a/Lib/inspect.py b/Lib/inspect.py index 2b7f8bec482f8e..5570a43ebfea19 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -220,13 +220,7 @@ def get_annotations(obj, *, globals=None, locals=None, eval_str=False): """ if isinstance(obj, type): # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, types.GetSetDescriptorType): - ann = None - else: - ann = None + ann = obj.__annotations__ obj_globals = None module_name = getattr(obj, '__module__', None) diff --git a/Lib/symtable.py b/Lib/symtable.py index ba2f0dafcd0063..af65e93e68eda4 100644 --- a/Lib/symtable.py +++ b/Lib/symtable.py @@ -222,6 +222,8 @@ def get_methods(self): if self.__methods is None: d = {} for st in self._table.children: + if st.type == _symtable.TYPE_ANNOTATION: + continue d[st.name] = 1 self.__methods = tuple(d) return self.__methods diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index b1a1b77c53e8cb..b0ae1289224070 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -352,32 +352,21 @@ def wrap_func_w_kwargs(): dis_annot_stmt_str = """\ 0 RESUME 0 - 2 SETUP_ANNOTATIONS - LOAD_CONST 0 (1) + 2 LOAD_CONST 0 (1) STORE_NAME 0 (x) - LOAD_NAME 1 (int) - LOAD_NAME 2 (__annotations__) - LOAD_CONST 1 ('x') - STORE_SUBSCR - - 3 LOAD_NAME 3 (fun) - PUSH_NULL - LOAD_CONST 0 (1) - CALL 1 - LOAD_NAME 2 (__annotations__) - LOAD_CONST 2 ('y') - STORE_SUBSCR 4 LOAD_CONST 0 (1) - LOAD_NAME 4 (lst) - LOAD_NAME 3 (fun) + LOAD_NAME 1 (lst) + LOAD_NAME 2 (fun) PUSH_NULL - LOAD_CONST 3 (0) + LOAD_CONST 1 (0) CALL 1 STORE_SUBSCR - LOAD_NAME 1 (int) - POP_TOP - RETURN_CONST 4 (None) + + 2 LOAD_CONST 2 (", line 2>) + MAKE_FUNCTION + STORE_NAME 3 (__annotate__) + RETURN_CONST 3 (None) """ compound_stmt_str = """\ diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c72f4387108ca8..5b7a639c025a0f 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -306,16 +306,6 @@ def test_eof_error(self): var_annot_global: int # a global annotated is necessary for test_var_annot -# custom namespace for testing __annotations__ - -class CNS: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - self._dct[item.lower()] = value - def __getitem__(self, item): - return self._dct[item] - class GrammarTests(unittest.TestCase): @@ -446,22 +436,12 @@ class F(C, A): self.assertEqual(E.__annotations__, {}) self.assertEqual(F.__annotations__, {}) - - def test_var_annot_metaclass_semantics(self): - class CMeta(type): - @classmethod - def __prepare__(metacls, name, bases, **kwds): - return {'__annotations__': CNS()} - class CC(metaclass=CMeta): - XX: 'ANNOT' - self.assertEqual(CC.__annotations__['xx'], 'ANNOT') - def test_var_annot_module_semantics(self): self.assertEqual(test.__annotations__, {}) self.assertEqual(ann_module.__annotations__, - {1: 2, 'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) + {'x': int, 'y': str, 'f': typing.Tuple[int, int], 'u': int | float}) self.assertEqual(ann_module.M.__annotations__, - {'123': 123, 'o': type}) + {'o': type}) self.assertEqual(ann_module2.__annotations__, {}) def test_var_annot_in_module(self): @@ -476,51 +456,12 @@ def test_var_annot_in_module(self): ann_module3.D_bad_ann(5) def test_var_annot_simple_exec(self): - gns = {}; lns= {} + gns = {}; lns = {} exec("'docstring'\n" - "__annotations__[1] = 2\n" "x: int = 5\n", gns, lns) - self.assertEqual(lns["__annotations__"], {1: 2, 'x': int}) - with self.assertRaises(KeyError): - gns['__annotations__'] - - def test_var_annot_custom_maps(self): - # tests with custom locals() and __annotations__ - ns = {'__annotations__': CNS()} - exec('X: int; Z: str = "Z"; (w): complex = 1j', ns) - self.assertEqual(ns['__annotations__']['x'], int) - self.assertEqual(ns['__annotations__']['z'], str) + self.assertEqual(lns["__annotate__"](1), {'x': int}) with self.assertRaises(KeyError): - ns['__annotations__']['w'] - nonloc_ns = {} - class CNS2: - def __init__(self): - self._dct = {} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('x: int = 1', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], int) - - def test_var_annot_refleak(self): - # complex case: custom locals plus custom __annotations__ - # this was causing refleak - cns = CNS() - nonloc_ns = {'__annotations__': cns} - class CNS2: - def __init__(self): - self._dct = {'__annotations__': cns} - def __setitem__(self, item, value): - nonlocal nonloc_ns - self._dct[item] = value - nonloc_ns[item] = value - def __getitem__(self, item): - return self._dct[item] - exec('X: str', {}, CNS2()) - self.assertEqual(nonloc_ns['__annotations__']['x'], str) + gns['__annotate__'] def test_var_annot_rhs(self): ns = {} diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 952ba43f72504d..56edd0c637f376 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -360,6 +360,8 @@ def test_annotations_are_created_correctly(self): ann_module4 = import_helper.import_fresh_module( 'test.typinganndata.ann_module4', ) + self.assertFalse("__annotations__" in ann_module4.__dict__) + self.assertEqual(ann_module4.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in ann_module4.__dict__) del ann_module4.__annotations__ self.assertFalse("__annotations__" in ann_module4.__dict__) diff --git a/Lib/test/test_opcodes.py b/Lib/test/test_opcodes.py index 72488b2bb6b4ff..f7cc8331b8d844 100644 --- a/Lib/test/test_opcodes.py +++ b/Lib/test/test_opcodes.py @@ -39,16 +39,19 @@ class C: pass def test_use_existing_annotations(self): ns = {'__annotations__': {1: 2}} exec('x: int', ns) - self.assertEqual(ns['__annotations__'], {'x': int, 1: 2}) + self.assertEqual(ns['__annotations__'], {1: 2}) def test_do_not_recreate_annotations(self): # Don't rely on the existence of the '__annotations__' global. with support.swap_item(globals(), '__annotations__', {}): - del globals()['__annotations__'] + globals().pop('__annotations__', None) class C: - del __annotations__ - with self.assertRaises(NameError): - x: int + try: + del __annotations__ + except NameError: + pass + x: int + self.assertEqual(C.__annotations__, {"x": int}) def test_raise_class_exceptions(self): diff --git a/Lib/test/test_positional_only_arg.py b/Lib/test/test_positional_only_arg.py index 1a193814d7535d..eea0625012da6d 100644 --- a/Lib/test/test_positional_only_arg.py +++ b/Lib/test/test_positional_only_arg.py @@ -2,6 +2,7 @@ import dis import pickle +import types import unittest from test.support import check_syntax_error @@ -440,7 +441,9 @@ def f(x: not (int is int), /): ... # without constant folding we end up with # COMPARE_OP(is), IS_OP (0) # with constant folding we should expect a IS_OP (1) - codes = [(i.opname, i.argval) for i in dis.get_instructions(g)] + code_obj = next(const for const in g.__code__.co_consts + if isinstance(const, types.CodeType) and const.co_name == "__annotate__") + codes = [(i.opname, i.argval) for i in dis.get_instructions(code_obj)] self.assertNotIn(('UNARY_NOT', None), codes) self.assertIn(('IS_OP', 1), codes) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index 46206accbafc36..0c12a3085b12af 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -109,6 +109,8 @@ def ismethod(oclass, obj, name): actualMethods = [] for m in py_item.__dict__.keys(): + if m == "__annotate__": + continue if ismethod(py_item, getattr(py_item, m), m): actualMethods.append(m) foundMethods = [] diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 57e5b8e8abddfa..a17c16cc73cf0e 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -77,6 +77,11 @@ class A(builtins.object) | __weakref__%s class B(builtins.object) + | Methods defined here: + | + | __annotate__(...) + | + | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__%s @@ -87,8 +92,6 @@ class B(builtins.object) | Data and other attributes defined here: | | NO_MEANING = 'eggs' - | - | __annotations__ = {'NO_MEANING': } class C(builtins.object) | Methods defined here: @@ -176,6 +179,9 @@ class A(builtins.object) list of weak references to the object class B(builtins.object) + Methods defined here: + __annotate__(...) + ---------------------------------------------------------------------- Data descriptors defined here: __dict__ dictionary for instance variables @@ -184,7 +190,6 @@ class B(builtins.object) ---------------------------------------------------------------------- Data and other attributes defined here: NO_MEANING = 'eggs' - __annotations__ = {'NO_MEANING': } class C(builtins.object) diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index df97b1354a168e..31f08cdb25e078 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -105,7 +105,7 @@ def test_runsource_shows_syntax_error_for_failed_compilation(self): def test_no_active_future(self): console = InteractiveColoredConsole() - source = "x: int = 1; print(__annotations__)" + source = "x: int = 1; print(__annotate__(1))" f = io.StringIO() with contextlib.redirect_stdout(f): result = console.runsource(source) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ef2a228b15ed4e..a4b111e865c86e 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -205,12 +205,14 @@ def test_assigned(self): def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') - st2 = st1.get_children()[0] + st2 = st1.get_children()[1] + self.assertEqual(st2.get_type(), "function") self.assertTrue(st2.lookup('x').is_local()) self.assertTrue(st2.lookup('x').is_annotated()) self.assertFalse(st2.lookup('x').is_global()) st3 = symtable.symtable('def f():\n x = 1\n', 'test', 'exec') - st4 = st3.get_children()[0] + st4 = st3.get_children()[1] + self.assertEqual(st4.get_type(), "function") self.assertTrue(st4.lookup('x').is_local()) self.assertFalse(st4.lookup('x').is_annotated()) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 5035de114b5e9d..1895c88d23b70d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -622,6 +622,7 @@ def test_caret_in_type_annotation(self): def f_with_type(): def foo(a: THIS_DOES_NOT_EXIST ) -> int: return 0 + foo.__annotations__ lineno_f = f_with_type.__code__.co_firstlineno expected_f = ( @@ -629,7 +630,9 @@ def foo(a: THIS_DOES_NOT_EXIST ) -> int: f' File "{__file__}", line {self.callable_line}, in get_exception\n' ' callable()\n' ' ~~~~~~~~^^\n' - f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_type\n' + ' foo.__annotations__\n' + f' File "{__file__}", line {lineno_f+1}, in __annotate__\n' ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' ' ^^^^^^^^^^^^^^^^^^^\n' ) diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 5e3c3347a41571..a9be1f5aa84681 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -1,7 +1,12 @@ import textwrap import types import unittest -from test.support import run_code +from test.support import run_code, check_syntax_error + +VALUE = 1 +FORWARDREF = 2 +SOURCE = 3 + class TypeAnnotationTests(unittest.TestCase): @@ -49,6 +54,7 @@ def test_annotations_are_created_correctly(self): class C: a:int=3 b:str=4 + self.assertEqual(C.__annotations__, {"a": int, "b": str}) self.assertTrue("__annotations__" in C.__dict__) del C.__annotations__ self.assertFalse("__annotations__" in C.__dict__) @@ -106,6 +112,13 @@ class D(metaclass=C): self.assertEqual(D.__annotations__, {}) +def build_module(code: str, name: str = "top") -> types.ModuleType: + ns = run_code(code) + mod = types.ModuleType(name) + mod.__dict__.update(ns) + return mod + + class TestSetupAnnotations(unittest.TestCase): def check(self, code: str): code = textwrap.dedent(code) @@ -113,11 +126,10 @@ def check(self, code: str): with self.subTest(scope=scope): if scope == "class": code = f"class C:\n{textwrap.indent(code, ' ')}" - ns = run_code(code) - if scope == "class": + ns = run_code(code) annotations = ns["C"].__annotations__ else: - annotations = ns["__annotations__"] + annotations = build_module(code).__annotations__ self.assertEqual(annotations, {"x": int}) def test_top_level(self): @@ -256,3 +268,146 @@ def check_annotations(self, f): # Setting f.__annotations__ also clears __annotate__ f.__annotations__ = {"z": 43} self.assertIs(f.__annotate__, None) + + +class DeferredEvaluationTests(unittest.TestCase): + def test_function(self): + def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_async_function(self): + async def func(x: undefined, /, y: undefined, *args: undefined, z: undefined, **kwargs: undefined) -> undefined: + pass + + with self.assertRaises(NameError): + func.__annotations__ + + undefined = 1 + self.assertEqual(func.__annotations__, { + "x": 1, + "y": 1, + "args": 1, + "z": 1, + "kwargs": 1, + "return": 1, + }) + + def test_class(self): + class X: + a: undefined + + with self.assertRaises(NameError): + X.__annotations__ + + undefined = 1 + self.assertEqual(X.__annotations__, {"a": 1}) + + def test_module(self): + ns = run_code("x: undefined = 1") + anno = ns["__annotate__"] + with self.assertRaises(NotImplementedError): + anno(2) + + with self.assertRaises(NameError): + anno(1) + + ns["undefined"] = 1 + self.assertEqual(anno(1), {"x": 1}) + + def test_class_scoping(self): + class Outer: + def meth(self, x: Nested): ... + x: Nested + class Nested: ... + + self.assertEqual(Outer.meth.__annotations__, {"x": Outer.Nested}) + self.assertEqual(Outer.__annotations__, {"x": Outer.Nested}) + + def test_no_exotic_expressions(self): + check_syntax_error(self, "def func(x: (yield)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (yield from x)): ...", "yield expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (y := 3)): ...", "named expression cannot be used within an annotation") + check_syntax_error(self, "def func(x: (await 42)): ...", "await expression cannot be used within an annotation") + + def test_no_exotic_expressions_in_unevaluated_annotations(self): + preludes = [ + "", + "class X: ", + "def f(): ", + "async def f(): ", + ] + for prelude in preludes: + with self.subTest(prelude=prelude): + check_syntax_error(self, prelude + "(x): (yield)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (yield from x)", "yield expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (y := 3)", "named expression cannot be used within an annotation") + check_syntax_error(self, prelude + "(x): (await 42)", "await expression cannot be used within an annotation") + + def test_ignore_non_simple_annotations(self): + ns = run_code("class X: (y): int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int.b: int") + self.assertEqual(ns["X"].__annotations__, {}) + ns = run_code("class X: int[str]: int") + self.assertEqual(ns["X"].__annotations__, {}) + + def test_generated_annotate(self): + def func(x: int): + pass + class X: + x: int + mod = build_module("x: int") + for obj in (func, X, mod): + with self.subTest(obj=obj): + annotate = obj.__annotate__ + self.assertIsInstance(annotate, types.FunctionType) + self.assertEqual(annotate.__name__, "__annotate__") + with self.assertRaises(NotImplementedError): + annotate(FORWARDREF) + with self.assertRaises(NotImplementedError): + annotate(SOURCE) + with self.assertRaises(NotImplementedError): + annotate(None) + self.assertEqual(annotate(VALUE), {"x": int}) + + def test_comprehension_in_annotation(self): + # This crashed in an earlier version of the code + ns = run_code("x: [y for y in range(10)]") + self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + + def test_future_annotations(self): + code = """ + from __future__ import annotations + + def f(x: int) -> int: pass + """ + ns = run_code(code) + f = ns["f"] + self.assertIsInstance(f.__annotate__, types.FunctionType) + annos = {"x": "int", "return": "int"} + self.assertEqual(f.__annotate__(VALUE), annos) + self.assertEqual(f.__annotations__, annos) + + def test_name_clash_with_format(self): + # this test would fail if __annotate__'s parameter was called "format" + code = """ + class format: pass + + def f(x: format): pass + """ + ns = run_code(code) + f = ns["f"] + self.assertEqual(f.__annotations__, {"x": ns["format"]}) diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dac55ceb9e99e0..9800b3b6a7da29 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6634,7 +6634,7 @@ def test_get_type_hints_from_various_objects(self): gth(None) def test_get_type_hints_modules(self): - ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} + ann_module_type_hints = {'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) @@ -6652,7 +6652,7 @@ def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) self.assertIsInstance(gth(ann_module.j_class), dict) - self.assertEqual(gth(ann_module.M), {'123': 123, 'o': type}) + self.assertEqual(gth(ann_module.M), {'o': type}) self.assertEqual(gth(ann_module.D), {'j': str, 'k': str, 'y': Optional[ann_module.C]}) self.assertEqual(gth(ann_module.Y), {'z': int}) diff --git a/Lib/test/typinganndata/ann_module.py b/Lib/test/typinganndata/ann_module.py index 5081e6b58345a9..e1a1792cb4a867 100644 --- a/Lib/test/typinganndata/ann_module.py +++ b/Lib/test/typinganndata/ann_module.py @@ -8,8 +8,6 @@ from typing import Optional from functools import wraps -__annotations__[1] = 2 - class C: x = 5; y: Optional['C'] = None @@ -18,8 +16,6 @@ class C: x: int = 5; y: str = x; f: Tuple[int, int] class M(type): - - __annotations__['123'] = 123 o: type = object (pars): bool = True diff --git a/Lib/typing.py b/Lib/typing.py index be49aa63464f05..7a9149d3f3c2c1 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -2412,7 +2412,7 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False): base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) else: base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) + ann = getattr(base, '__annotations__', {}) if isinstance(ann, types.GetSetDescriptorType): ann = {} base_locals = dict(vars(base)) if localns is None else localns @@ -2970,7 +2970,12 @@ def __new__(cls, typename, bases, ns): raise TypeError( 'can only inherit from a NamedTuple type and Generic') bases = tuple(tuple if base is _NamedTuple else base for base in bases) - types = ns.get('__annotations__', {}) + if "__annotations__" in ns: + types = ns["__annotations__"] + elif "__annotate__" in ns: + types = ns["__annotate__"](1) # VALUE + else: + types = {} default_names = [] for field_name in types: if field_name in ns: @@ -3131,7 +3136,12 @@ def __new__(cls, name, bases, ns, total=True): tp_dict.__orig_bases__ = bases annotations = {} - own_annotations = ns.get('__annotations__', {}) + if "__annotations__" in ns: + own_annotations = ns["__annotations__"] + elif "__annotate__" in ns: + own_annotations = ns["__annotate__"](1) # VALUE + else: + own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_annotations = { n: _type_check(tp, msg, module=tp_dict.__module__) @@ -3143,7 +3153,12 @@ def __new__(cls, name, bases, ns, total=True): mutable_keys = set() for base in bases: - annotations.update(base.__dict__.get('__annotations__', {})) + # TODO: Avoid eagerly evaluating annotations in VALUE format. + # Instead, evaluate in FORWARDREF format to figure out which + # keys have Required/NotRequired/ReadOnly qualifiers, and create + # a new __annotate__ function for the resulting TypedDict that + # combines the annotations from this class and its parents. + annotations.update(base.__annotations__) base_required = base.__dict__.get('__required_keys__', set()) required_keys |= base_required diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst new file mode 100644 index 00000000000000..265ffb32e6a1f9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-05-22-06-22-47.gh-issue-119180.vZMiXm.rst @@ -0,0 +1 @@ +Evaluation of annotations is now deferred. See :pep:`649` for details. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 413ad1105f9428..05c17ac334b69f 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3975,6 +3975,11 @@ dummy_func( assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/compile.c b/Python/compile.c index cb724154206b7e..c3372766d0bd50 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -132,7 +132,7 @@ enum { COMPILER_SCOPE_ASYNC_FUNCTION, COMPILER_SCOPE_LAMBDA, COMPILER_SCOPE_COMPREHENSION, - COMPILER_SCOPE_TYPEPARAMS, + COMPILER_SCOPE_ANNOTATIONS, }; @@ -142,6 +142,15 @@ typedef _PyInstructionSequence instr_sequence; #define INITIAL_INSTR_SEQUENCE_SIZE 100 #define INITIAL_INSTR_SEQUENCE_LABELS_MAP_SIZE 10 +static const int compare_masks[] = { + [Py_LT] = COMPARISON_LESS_THAN, + [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, + [Py_EQ] = COMPARISON_EQUALS, + [Py_NE] = COMPARISON_NOT_EQUALS, + [Py_GT] = COMPARISON_GREATER_THAN, + [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, +}; + /* * Resize the array if index is out of range. * @@ -208,6 +217,7 @@ struct compiler_unit { PyObject *u_private; /* for private name mangling */ PyObject *u_static_attributes; /* for class: attributes accessed via self.X */ + PyObject *u_deferred_annotations; /* AnnAssign nodes deferred to the end of compilation */ instr_sequence *u_instr_sequence; /* codegen output */ @@ -330,6 +340,8 @@ static int compiler_pattern(struct compiler *, pattern_ty, pattern_context *); static int compiler_match(struct compiler *, stmt_ty); static int compiler_pattern_subpattern(struct compiler *, pattern_ty, pattern_context *); +static int compiler_make_closure(struct compiler *c, location loc, + PyCodeObject *co, Py_ssize_t flags); static PyCodeObject *optimize_and_assemble(struct compiler *, int addNone); @@ -545,6 +557,7 @@ compiler_unit_free(struct compiler_unit *u) Py_CLEAR(u->u_metadata.u_fasthidden); Py_CLEAR(u->u_private); Py_CLEAR(u->u_static_attributes); + Py_CLEAR(u->u_deferred_annotations); PyMem_Free(u); } @@ -582,8 +595,8 @@ compiler_set_qualname(struct compiler *c) capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1); parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, CAPSULE_NAME); assert(parent); - if (parent->u_scope_type == COMPILER_SCOPE_TYPEPARAMS) { - /* The parent is a type parameter scope, so we need to + if (parent->u_scope_type == COMPILER_SCOPE_ANNOTATIONS) { + /* The parent is an annotation scope, so we need to look at the grandparent. */ if (stack_size == 2) { // If we're immediately within the module, we can skip @@ -1128,6 +1141,7 @@ compiler_enter_scope(struct compiler *c, identifier name, } u->u_private = NULL; + u->u_deferred_annotations = NULL; if (scope_type == COMPILER_SCOPE_CLASS) { u->u_static_attributes = PySet_New(0); if (!u->u_static_attributes) { @@ -1209,85 +1223,6 @@ compiler_exit_scope(struct compiler *c) PyErr_SetRaisedException(exc); } -/* Search if variable annotations are present statically in a block. */ - -static bool -find_ann(asdl_stmt_seq *stmts) -{ - int i, j, res = 0; - stmt_ty st; - - for (i = 0; i < asdl_seq_LEN(stmts); i++) { - st = (stmt_ty)asdl_seq_GET(stmts, i); - switch (st->kind) { - case AnnAssign_kind: - return true; - case For_kind: - res = find_ann(st->v.For.body) || - find_ann(st->v.For.orelse); - break; - case AsyncFor_kind: - res = find_ann(st->v.AsyncFor.body) || - find_ann(st->v.AsyncFor.orelse); - break; - case While_kind: - res = find_ann(st->v.While.body) || - find_ann(st->v.While.orelse); - break; - case If_kind: - res = find_ann(st->v.If.body) || - find_ann(st->v.If.orelse); - break; - case With_kind: - res = find_ann(st->v.With.body); - break; - case AsyncWith_kind: - res = find_ann(st->v.AsyncWith.body); - break; - case Try_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Try.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.Try.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.Try.body) || - find_ann(st->v.Try.finalbody) || - find_ann(st->v.Try.orelse); - break; - case TryStar_kind: - for (j = 0; j < asdl_seq_LEN(st->v.TryStar.handlers); j++) { - excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET( - st->v.TryStar.handlers, j); - if (find_ann(handler->v.ExceptHandler.body)) { - return true; - } - } - res = find_ann(st->v.TryStar.body) || - find_ann(st->v.TryStar.finalbody) || - find_ann(st->v.TryStar.orelse); - break; - case Match_kind: - for (j = 0; j < asdl_seq_LEN(st->v.Match.cases); j++) { - match_case_ty match_case = (match_case_ty)asdl_seq_GET( - st->v.Match.cases, j); - if (find_ann(match_case->body)) { - return true; - } - } - break; - default: - res = false; - break; - } - if (res) { - break; - } - } - return res; -} - /* * Frame block handling functions */ @@ -1502,6 +1437,47 @@ compiler_unwind_fblock_stack(struct compiler *c, location *ploc, return SUCCESS; } +static int +compiler_setup_annotations_scope(struct compiler *c, location loc, + void *key, PyObject *name) +{ + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, + key, loc.lineno) == -1) { + return ERROR; + } + c->u->u_metadata.u_posonlyargcount = 1; + // if .format != 1: raise NotImplementedError + _Py_DECLARE_STR(format, ".format"); + ADDOP_I(c, loc, LOAD_FAST, 0); + ADDOP_LOAD_CONST(c, loc, _PyLong_GetOne()); + ADDOP_I(c, loc, COMPARE_OP, (Py_NE << 5) | compare_masks[Py_NE]); + NEW_JUMP_TARGET_LABEL(c, body); + ADDOP_JUMP(c, loc, POP_JUMP_IF_FALSE, body); + ADDOP_I(c, loc, LOAD_COMMON_CONSTANT, CONSTANT_NOTIMPLEMENTEDERROR); + ADDOP_I(c, loc, RAISE_VARARGS, 1); + USE_LABEL(c, body); + return 0; +} + +static int +compiler_leave_annotations_scope(struct compiler *c, location loc, + Py_ssize_t annotations_len) +{ + ADDOP_I(c, loc, BUILD_MAP, annotations_len); + ADDOP_IN_SCOPE(c, loc, RETURN_VALUE); + PyCodeObject *co = optimize_and_assemble(c, 1); + compiler_exit_scope(c); + if (co == NULL) { + return ERROR; + } + if (compiler_make_closure(c, loc, co, 0) < 0) { + Py_DECREF(co); + return ERROR; + } + Py_DECREF(co); + return 0; +} + /* Compile a sequence of statements, checking for a docstring and for annotations. */ @@ -1517,34 +1493,79 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); loc = LOC(st); } - /* Every annotated class and module should have __annotations__. */ - if (find_ann(stmts)) { + /* If from __future__ import annotations is active, + * every annotated class and module should have __annotations__. + * Else __annotate__ is created when necessary. */ + if ((c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && c->u->u_ste->ste_annotations_used) { ADDOP(c, loc, SETUP_ANNOTATIONS); } if (!asdl_seq_LEN(stmts)) { return SUCCESS; } Py_ssize_t first_instr = 0; - PyObject *docstring = _PyAST_GetDocString(stmts); - if (docstring) { - first_instr = 1; - /* if not -OO mode, set docstring */ - if (c->c_optimize < 2) { - PyObject *cleandoc = _PyCompile_CleanDoc(docstring); - if (cleandoc == NULL) { - return ERROR; + if (!c->c_interactive) { + PyObject *docstring = _PyAST_GetDocString(stmts); + if (docstring) { + first_instr = 1; + /* if not -OO mode, set docstring */ + if (c->c_optimize < 2) { + PyObject *cleandoc = _PyCompile_CleanDoc(docstring); + if (cleandoc == NULL) { + return ERROR; + } + stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); + assert(st->kind == Expr_kind); + location loc = LOC(st->v.Expr.value); + ADDOP_LOAD_CONST(c, loc, cleandoc); + Py_DECREF(cleandoc); + RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } - stmt_ty st = (stmt_ty)asdl_seq_GET(stmts, 0); - assert(st->kind == Expr_kind); - location loc = LOC(st->v.Expr.value); - ADDOP_LOAD_CONST(c, loc, cleandoc); - Py_DECREF(cleandoc); - RETURN_IF_ERROR(compiler_nameop(c, NO_LOCATION, &_Py_ID(__doc__), Store)); } } for (Py_ssize_t i = first_instr; i < asdl_seq_LEN(stmts); i++) { VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i)); } + // If there are annotations and the future import is not on, we + // collect the annotations in a separate pass and generate an + // __annotate__ function. See PEP 649. + if (!(c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) && + c->u->u_deferred_annotations != NULL) { + + // It's possible that ste_annotations_block is set but + // u_deferred_annotations is not, because the former is still + // set if there are only non-simple annotations (i.e., annotations + // for attributes, subscripts, or parenthesized names). However, the + // reverse should not be possible. + assert(c->u->u_ste->ste_annotation_block != NULL); + PyObject *deferred_anno = Py_NewRef(c->u->u_deferred_annotations); + void *key = (void *)((uintptr_t)c->u->u_ste->ste_id + 1); + if (compiler_setup_annotations_scope(c, loc, key, + c->u->u_ste->ste_annotation_block->ste_name) == -1) { + Py_DECREF(deferred_anno); + return ERROR; + } + Py_ssize_t annotations_len = PyList_Size(deferred_anno); + for (Py_ssize_t i = 0; i < annotations_len; i++) { + PyObject *ptr = PyList_GET_ITEM(deferred_anno, i); + stmt_ty st = (stmt_ty)PyLong_AsVoidPtr(ptr); + if (st == NULL) { + compiler_exit_scope(c); + Py_DECREF(deferred_anno); + return ERROR; + } + PyObject *mangled = _Py_Mangle(c->u->u_private, st->v.AnnAssign.target->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, LOC(st), mangled); + VISIT(c, expr, st->v.AnnAssign.annotation); + } + Py_DECREF(deferred_anno); + + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + RETURN_IF_ERROR( + compiler_nameop(c, loc, &_Py_ID(__annotate__), Store) + ); + } return SUCCESS; } @@ -1559,11 +1580,10 @@ compiler_codegen(struct compiler *c, mod_ty mod) } break; case Interactive_kind: - if (find_ann(mod->v.Interactive.body)) { - ADDOP(c, loc, SETUP_ANNOTATIONS); - } c->c_interactive = 1; - VISIT_SEQ(c, stmt, mod->v.Interactive.body); + if (compiler_body(c, loc, mod->v.Interactive.body) < 0) { + return ERROR; + } break; case Expression_kind: VISIT(c, expr, mod->v.Expression.body); @@ -1702,6 +1722,9 @@ compiler_make_closure(struct compiler *c, location loc, if (flags & MAKE_FUNCTION_ANNOTATIONS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATIONS); } + if (flags & MAKE_FUNCTION_ANNOTATE) { + ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_ANNOTATE); + } if (flags & MAKE_FUNCTION_KWDEFAULTS) { ADDOP_I(c, loc, SET_FUNCTION_ATTRIBUTE, MAKE_FUNCTION_KWDEFAULTS); } @@ -1833,7 +1856,7 @@ compiler_visit_argannotation(struct compiler *c, identifier id, VISIT(c, expr, annotation); } } - *annotations_len += 2; + *annotations_len += 1; return SUCCESS; } @@ -1856,43 +1879,76 @@ compiler_visit_argannotations(struct compiler *c, asdl_arg_seq* args, } static int -compiler_visit_annotations(struct compiler *c, location loc, - arguments_ty args, expr_ty returns) +compiler_visit_annotations_in_scope(struct compiler *c, location loc, + arguments_ty args, expr_ty returns, + Py_ssize_t *annotations_len) { - /* Push arg annotation names and values. - The expressions are evaluated out-of-order wrt the source code. - - Return -1 on error, 0 if no annotations pushed, 1 if a annotations is pushed. - */ - Py_ssize_t annotations_len = 0; - RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->args, &annotations_len, loc)); + compiler_visit_argannotations(c, args->args, annotations_len, loc)); RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->posonlyargs, &annotations_len, loc)); + compiler_visit_argannotations(c, args->posonlyargs, annotations_len, loc)); if (args->vararg && args->vararg->annotation) { RETURN_IF_ERROR( compiler_visit_argannotation(c, args->vararg->arg, - args->vararg->annotation, &annotations_len, loc)); + args->vararg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotations(c, args->kwonlyargs, &annotations_len, loc)); + compiler_visit_argannotations(c, args->kwonlyargs, annotations_len, loc)); if (args->kwarg && args->kwarg->annotation) { RETURN_IF_ERROR( compiler_visit_argannotation(c, args->kwarg->arg, - args->kwarg->annotation, &annotations_len, loc)); + args->kwarg->annotation, annotations_len, loc)); } RETURN_IF_ERROR( - compiler_visit_argannotation(c, &_Py_ID(return), returns, &annotations_len, loc)); + compiler_visit_argannotation(c, &_Py_ID(return), returns, annotations_len, loc)); - if (annotations_len) { - ADDOP_I(c, loc, BUILD_TUPLE, annotations_len); - return 1; + return 0; +} + +static int +compiler_visit_annotations(struct compiler *c, location loc, + arguments_ty args, expr_ty returns) +{ + /* Push arg annotation names and values. + The expressions are evaluated separately from the rest of the source code. + + Return -1 on error, or a combination of flags to add to the function. + */ + Py_ssize_t annotations_len = 0; + + PySTEntryObject *ste; + if (_PySymtable_LookupOptional(c->c_st, args, &ste) < 0) { + return ERROR; + } + assert(ste != NULL); + bool annotations_used = ste->ste_annotations_used; + + if (annotations_used) { + if (compiler_setup_annotations_scope(c, loc, (void *)args, + ste->ste_name) < 0) { + Py_DECREF(ste); + return ERROR; + } + } + Py_DECREF(ste); + + if (compiler_visit_annotations_in_scope(c, loc, args, returns, &annotations_len) < 0) { + if (annotations_used) { + compiler_exit_scope(c); + } + return ERROR; + } + + if (annotations_used) { + RETURN_IF_ERROR( + compiler_leave_annotations_scope(c, loc, annotations_len) + ); + return MAKE_FUNCTION_ANNOTATE; } return 0; @@ -2001,7 +2057,7 @@ compiler_type_param_bound_or_default(struct compiler *c, expr_ty e, identifier name, void *key, bool allow_starred) { - if (compiler_enter_scope(c, name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, name, COMPILER_SCOPE_ANNOTATIONS, key, e->lineno) == -1) { return ERROR; } @@ -2220,7 +2276,6 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) asdl_expr_seq *decos; asdl_type_param_seq *type_params; Py_ssize_t funcflags; - int annotations; int firstlineno; if (is_async) { @@ -2274,7 +2329,7 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, firstlineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2286,16 +2341,14 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } } - annotations = compiler_visit_annotations(c, loc, args, returns); - if (annotations < 0) { + int annotations_flag = compiler_visit_annotations(c, loc, args, returns); + if (annotations_flag < 0) { if (is_generic) { compiler_exit_scope(c); } return ERROR; } - if (annotations > 0) { - funcflags |= MAKE_FUNCTION_ANNOTATIONS; - } + funcflags |= annotations_flag; if (compiler_function_body(c, s, is_async, funcflags, firstlineno) < 0) { if (is_generic) { @@ -2510,7 +2563,7 @@ compiler_class(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, firstlineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2630,7 +2683,7 @@ compiler_typealias(struct compiler *c, stmt_ty s) if (!type_params_name) { return ERROR; } - if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_TYPEPARAMS, + if (compiler_enter_scope(c, type_params_name, COMPILER_SCOPE_ANNOTATIONS, (void *)type_params, loc.lineno) == -1) { Py_DECREF(type_params_name); return ERROR; @@ -2719,15 +2772,6 @@ check_compare(struct compiler *c, expr_ty e) return SUCCESS; } -static const int compare_masks[] = { - [Py_LT] = COMPARISON_LESS_THAN, - [Py_LE] = COMPARISON_LESS_THAN | COMPARISON_EQUALS, - [Py_EQ] = COMPARISON_EQUALS, - [Py_NE] = COMPARISON_NOT_EQUALS, - [Py_GT] = COMPARISON_GREATER_THAN, - [Py_GE] = COMPARISON_GREATER_THAN | COMPARISON_EQUALS, -}; - static int compiler_addcompare(struct compiler *c, location loc, cmpop_ty op) { @@ -6366,7 +6410,8 @@ compiler_annassign(struct compiler *c, stmt_ty s) { location loc = LOC(s); expr_ty targ = s->v.AnnAssign.target; - PyObject* mangled; + bool future_annotations = c->c_future.ff_features & CO_FUTURE_ANNOTATIONS; + PyObject *mangled; assert(s->kind == AnnAssign_kind); @@ -6384,16 +6429,30 @@ compiler_annassign(struct compiler *c, stmt_ty s) if (s->v.AnnAssign.simple && (c->u->u_scope_type == COMPILER_SCOPE_MODULE || c->u->u_scope_type == COMPILER_SCOPE_CLASS)) { - if (c->c_future.ff_features & CO_FUTURE_ANNOTATIONS) { - VISIT(c, annexpr, s->v.AnnAssign.annotation) + if (future_annotations) { + VISIT(c, annexpr, s->v.AnnAssign.annotation); + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); + mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); + ADDOP_LOAD_CONST_NEW(c, loc, mangled); + ADDOP(c, loc, STORE_SUBSCR); } else { - VISIT(c, expr, s->v.AnnAssign.annotation); + if (c->u->u_deferred_annotations == NULL) { + c->u->u_deferred_annotations = PyList_New(0); + if (c->u->u_deferred_annotations == NULL) { + return ERROR; + } + } + PyObject *ptr = PyLong_FromVoidPtr((void *)s); + if (ptr == NULL) { + return ERROR; + } + if (PyList_Append(c->u->u_deferred_annotations, ptr) < 0) { + Py_DECREF(ptr); + return ERROR; + } + Py_DECREF(ptr); } - ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__annotations__), names); - mangled = _Py_MaybeMangle(c->u->u_private, c->u->u_ste, targ->v.Name.id); - ADDOP_LOAD_CONST_NEW(c, loc, mangled); - ADDOP(c, loc, STORE_SUBSCR); } break; case Attribute_kind: @@ -6419,7 +6478,7 @@ compiler_annassign(struct compiler *c, stmt_ty s) return ERROR; } /* Annotation is evaluated last. */ - if (!s->v.AnnAssign.simple && check_annotation(c, s) < 0) { + if (future_annotations && !s->v.AnnAssign.simple && check_annotation(c, s) < 0) { return ERROR; } return SUCCESS; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index bab629684c53f6..470c82d938ab7c 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -4061,6 +4061,11 @@ assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 355be966cbb84a..0274f8b7a48c3c 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5450,6 +5450,11 @@ assert(func_obj->func_defaults == NULL); func_obj->func_defaults = attr; break; + case MAKE_FUNCTION_ANNOTATE: + assert(PyCallable_Check(attr)); + assert(func_obj->func_annotate == NULL); + func_obj->func_annotate = attr; + break; default: Py_UNREACHABLE(); } diff --git a/Python/symtable.c b/Python/symtable.c index 7e452cdb13badf..287bc2bd58107d 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -112,6 +112,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_varkeywords = 0; ste->ste_opt_lineno = 0; ste->ste_opt_col_offset = 0; + ste->ste_annotations_used = 0; ste->ste_lineno = lineno; ste->ste_col_offset = col_offset; ste->ste_end_lineno = end_lineno; @@ -132,6 +133,7 @@ ste_new(struct symtable *st, identifier name, _Py_block_ty block, ste->ste_can_see_class_scope = 0; ste->ste_comp_iter_expr = 0; ste->ste_needs_classdict = 0; + ste->ste_annotation_block = NULL; ste->ste_symbols = PyDict_New(); ste->ste_varnames = PyList_New(0); @@ -167,6 +169,7 @@ ste_dealloc(PySTEntryObject *ste) Py_XDECREF(ste->ste_varnames); Py_XDECREF(ste->ste_children); Py_XDECREF(ste->ste_directives); + Py_XDECREF(ste->ste_annotation_block); Py_XDECREF(ste->ste_mangled_names); PyObject_Free(ste); } @@ -245,10 +248,11 @@ static int symtable_visit_alias(struct symtable *st, alias_ty); static int symtable_visit_comprehension(struct symtable *st, comprehension_ty); static int symtable_visit_keyword(struct symtable *st, keyword_ty); static int symtable_visit_params(struct symtable *st, asdl_arg_seq *args); -static int symtable_visit_annotation(struct symtable *st, expr_ty annotation); +static int symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key); static int symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args); static int symtable_implicit_arg(struct symtable *st, int pos); -static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty); +static int symtable_visit_annotations(struct symtable *st, stmt_ty, arguments_ty, expr_ty, + struct _symtable_entry *parent_ste); static int symtable_visit_withitem(struct symtable *st, withitem_ty item); static int symtable_visit_match_case(struct symtable *st, match_case_ty m); static int symtable_visit_pattern(struct symtable *st, pattern_ty s); @@ -504,6 +508,21 @@ _PySymtable_Lookup(struct symtable *st, void *key) return (PySTEntryObject *)v; } +int +_PySymtable_LookupOptional(struct symtable *st, void *key, + PySTEntryObject **out) +{ + PyObject *k = PyLong_FromVoidPtr(key); + if (k == NULL) { + *out = NULL; + return -1; + } + int result = PyDict_GetItemRef(st->st_blocks, k, (PyObject **)out); + Py_DECREF(k); + assert(*out == NULL || PySTEntry_Check(*out)); + return result; +} + long _PyST_GetSymbol(PySTEntryObject *ste, PyObject *name) { @@ -525,6 +544,7 @@ int _PyST_IsFunctionLike(PySTEntryObject *ste) { return ste->ste_type == FunctionBlock + || ste->ste_type == AnnotationBlock || ste->ste_type == TypeVarBoundBlock || ste->ste_type == TypeAliasBlock || ste->ste_type == TypeParamBlock; @@ -1317,20 +1337,12 @@ symtable_exit_block(struct symtable *st) } static int -symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, - void *ast, int lineno, int col_offset, - int end_lineno, int end_col_offset) +symtable_enter_existing_block(struct symtable *st, PySTEntryObject* ste) { - PySTEntryObject *prev = NULL, *ste; - - ste = ste_new(st, name, block, ast, lineno, col_offset, end_lineno, end_col_offset); - if (ste == NULL) - return 0; if (PyList_Append(st->st_stack, (PyObject *)ste) < 0) { - Py_DECREF(ste); return 0; } - prev = st->st_cur; + PySTEntryObject *prev = st->st_cur; /* bpo-37757: For now, disallow *all* assignment expressions in the * outermost iterator expression of a comprehension, even those inside * a nested comprehension or a lambda expression. @@ -1340,21 +1352,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, } /* No need to inherit ste_mangled_names in classes, where all names * are mangled. */ - if (prev && prev->ste_mangled_names != NULL && block != ClassBlock) { + if (prev && prev->ste_mangled_names != NULL && ste->ste_type != ClassBlock) { ste->ste_mangled_names = Py_NewRef(prev->ste_mangled_names); } /* The entry is owned by the stack. Borrow it for st_cur. */ - Py_DECREF(ste); st->st_cur = ste; - /* Annotation blocks shouldn't have any affect on the symbol table since in - * the compilation stage, they will all be transformed to strings. They are - * only created if future 'annotations' feature is activated. */ - if (block == AnnotationBlock) { + /* If "from __future__ import annotations" is active, + * annotation blocks shouldn't have any affect on the symbol table since in + * the compilation stage, they will all be transformed to strings. */ + if (st->st_future->ff_features & CO_FUTURE_ANNOTATIONS && ste->ste_type == AnnotationBlock) { return 1; } - if (block == ModuleBlock) + if (ste->ste_type == ModuleBlock) st->st_global = st->st_cur->ste_symbols; if (prev) { @@ -1365,6 +1376,20 @@ symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, return 1; } +static int +symtable_enter_block(struct symtable *st, identifier name, _Py_block_ty block, + void *ast, int lineno, int col_offset, + int end_lineno, int end_col_offset) +{ + PySTEntryObject *ste = ste_new(st, name, block, ast, + lineno, col_offset, end_lineno, end_col_offset); + if (ste == NULL) + return 0; + int result = symtable_enter_existing_block(st, ste); + Py_DECREF(ste); + return result; +} + static long symtable_lookup_entry(struct symtable *st, PySTEntryObject *ste, PyObject *name) { @@ -1643,7 +1668,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } switch (s->kind) { - case FunctionDef_kind: + case FunctionDef_kind: { if (!symtable_add_def(st, s->v.FunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.FunctionDef.args->defaults) @@ -1665,13 +1690,22 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.FunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.FunctionDef.args, - s->v.FunctionDef.returns)) + s->v.FunctionDef.returns, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.FunctionDef.name, - FunctionBlock, (void *)s, - LOCATION(s))) + } + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); VISIT(st, arguments, s->v.FunctionDef.args); VISIT_SEQ(st, stmt, s->v.FunctionDef.body); if (!symtable_exit_block(st)) @@ -1681,6 +1715,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case ClassDef_kind: { PyObject *tmp; if (!symtable_add_def(st, s->v.ClassDef.name, DEF_LOCAL, LOCATION(s))) @@ -1776,6 +1811,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT(st, expr, s->v.Assign.value); break; case AnnAssign_kind: + st->st_cur->ste_annotations_used = 1; if (s->v.AnnAssign.target->kind == Name_kind) { expr_ty e_name = s->v.AnnAssign.target; long cur = symtable_lookup(st, e_name->v.Name.id); @@ -1810,7 +1846,8 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) else { VISIT(st, expr, s->v.AnnAssign.target); } - if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation)) { + if (!symtable_visit_annotation(st, s->v.AnnAssign.annotation, + (void *)((uintptr_t)st->st_cur->ste_id + 1))) { VISIT_QUIT(st, 0); } @@ -1960,7 +1997,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_SEQ(st, withitem, s->v.With.items); VISIT_SEQ(st, stmt, s->v.With.body); break; - case AsyncFunctionDef_kind: + case AsyncFunctionDef_kind: { if (!symtable_add_def(st, s->v.AsyncFunctionDef.name, DEF_LOCAL, LOCATION(s))) VISIT_QUIT(st, 0); if (s->v.AsyncFunctionDef.args->defaults) @@ -1983,14 +2020,21 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) } VISIT_SEQ(st, type_param, s->v.AsyncFunctionDef.type_params); } + PySTEntryObject *new_ste = ste_new(st, s->v.FunctionDef.name, FunctionBlock, (void *)s, + LOCATION(s)); + if (!new_ste) { + VISIT_QUIT(st, 0); + } + if (!symtable_visit_annotations(st, s, s->v.AsyncFunctionDef.args, - s->v.AsyncFunctionDef.returns)) + s->v.AsyncFunctionDef.returns, new_ste)) VISIT_QUIT(st, 0); - if (!symtable_enter_block(st, s->v.AsyncFunctionDef.name, - FunctionBlock, (void *)s, - s->lineno, s->col_offset, - s->end_lineno, s->end_col_offset)) + if (!symtable_enter_existing_block(st, new_ste)) { + Py_DECREF(new_ste); VISIT_QUIT(st, 0); + } + Py_DECREF(new_ste); + st->st_cur->ste_coroutine = 1; VISIT(st, arguments, s->v.AsyncFunctionDef.args); VISIT_SEQ(st, stmt, s->v.AsyncFunctionDef.body); @@ -2001,6 +2045,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s) VISIT_QUIT(st, 0); } break; + } case AsyncWith_kind: VISIT_SEQ(st, withitem, s->v.AsyncWith.items); VISIT_SEQ(st, stmt, s->v.AsyncWith.body); @@ -2444,18 +2489,44 @@ symtable_visit_params(struct symtable *st, asdl_arg_seq *args) } static int -symtable_visit_annotation(struct symtable *st, expr_ty annotation) +symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)annotation, annotation->lineno, - annotation->col_offset, annotation->end_lineno, - annotation->end_col_offset)) { - VISIT_QUIT(st, 0); + struct _symtable_entry *parent_ste = st->st_cur; + if (parent_ste->ste_annotation_block == NULL) { + _Py_block_ty current_type = parent_ste->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + key, LOCATION(annotation))) { + VISIT_QUIT(st, 0); + } + parent_ste->ste_annotation_block = + (struct _symtable_entry *)Py_NewRef(st->st_cur); + int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; + if (current_type == ClassBlock && !future_annotations) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { + return 0; + } + } + + _Py_DECLARE_STR(format, ".format"); + // The generated __annotate__ function takes a single parameter with the + // internal name ".format". + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, + LOCATION(annotation))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, + LOCATION(annotation))) { + return 0; + } + } + else { + if (!symtable_enter_existing_block(st, parent_ste->ste_annotation_block)) { + VISIT_QUIT(st, 0); + } } VISIT(st, expr, annotation); - if (future_annotations && !symtable_exit_block(st)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2471,37 +2542,58 @@ symtable_visit_argannotations(struct symtable *st, asdl_arg_seq *args) for (i = 0; i < asdl_seq_LEN(args); i++) { arg_ty arg = (arg_ty)asdl_seq_GET(args, i); - if (arg->annotation) + if (arg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, arg->annotation); + } } return 1; } static int -symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns) +symtable_visit_annotations(struct symtable *st, stmt_ty o, arguments_ty a, expr_ty returns, + struct _symtable_entry *function_ste) { - int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; - if (future_annotations && - !symtable_enter_block(st, &_Py_ID(_annotation), AnnotationBlock, - (void *)o, o->lineno, o->col_offset, o->end_lineno, - o->end_col_offset)) { + int is_in_class = st->st_cur->ste_can_see_class_scope; + _Py_block_ty current_type = st->st_cur->ste_type; + if (!symtable_enter_block(st, &_Py_ID(__annotate__), AnnotationBlock, + (void *)a, LOCATION(o))) { VISIT_QUIT(st, 0); } + if (is_in_class || current_type == ClassBlock) { + st->st_cur->ste_can_see_class_scope = 1; + if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(o))) { + return 0; + } + } + _Py_DECLARE_STR(format, ".format"); + // We need to insert code that reads this "parameter" to the function. + if (!symtable_add_def(st, &_Py_STR(format), DEF_PARAM, LOCATION(o))) { + return 0; + } + if (!symtable_add_def(st, &_Py_STR(format), USE, LOCATION(o))) { + return 0; + } if (a->posonlyargs && !symtable_visit_argannotations(st, a->posonlyargs)) return 0; if (a->args && !symtable_visit_argannotations(st, a->args)) return 0; - if (a->vararg && a->vararg->annotation) + if (a->vararg && a->vararg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->vararg->annotation); - if (a->kwarg && a->kwarg->annotation) + } + if (a->kwarg && a->kwarg->annotation) { + st->st_cur->ste_annotations_used = 1; VISIT(st, expr, a->kwarg->annotation); + } if (a->kwonlyargs && !symtable_visit_argannotations(st, a->kwonlyargs)) return 0; - if (future_annotations && !symtable_exit_block(st)) { - VISIT_QUIT(st, 0); + if (returns) { + st->st_cur->ste_annotations_used = 1; + VISIT(st, expr, returns); } - if (returns && !symtable_visit_annotation(st, returns)) { + if (!symtable_exit_block(st)) { VISIT_QUIT(st, 0); } return 1; @@ -2733,7 +2825,7 @@ symtable_visit_dictcomp(struct symtable *st, expr_ty e) static int symtable_raise_if_annotation_block(struct symtable *st, const char *name, expr_ty e) { - enum _block_type type = st->st_cur->ste_type; + _Py_block_ty type = st->st_cur->ste_type; if (type == AnnotationBlock) PyErr_Format(PyExc_SyntaxError, ANNOTATION_NOT_ALLOWED, name); else if (type == TypeVarBoundBlock)