From 1cd04c96c7dc05055196f5fa4d6d2d7986ace9af Mon Sep 17 00:00:00 2001 From: Taus Date: Fri, 18 Oct 2024 12:22:17 +0000 Subject: [PATCH 1/2] Python: Fix bug in handling of `**kwargs` in class bases This caused a dataset check error on the `python/cpython` database, as we had a `DictUnpacking` node whose parent was not a `dict_item_list`, but rather an `expr_list`. Investigating a bit further revealed that this was because in a construction like ```python class C[T](base, foo=bar, **kwargs): ... ``` we were mistakenly adding `**kwargs` to the same list as `base` (which is just a list of expressions), rather than the same list as `foo=bar` (which is a list of dictionary items) The ultimate cause of this was the use of `! name` in `python.tsg` to distinguish between bases and keyword arguments (only the latter of which have the `name` field). Because `dictionary_splat` doesn't have a `name` field either, these were mistakenly put in the wrong list, leading to the error. Also, because our previous test of `class` statements did not include a `**kwargs` construction, we were not checking that the new parser behaved correctly in this case. For the most part this was not a problem, but on files that use syntax not supported by the old parser (like type parameters on classes), this became an issue. This is also why we did not see this error previously. To fix this, we added `! value` (which is a field present on `dictionary_splat` nodes) as a secondary filter, and added a third stanza to handle `dictionary_splat` nodes. --- python/extractor/tsg-python/python.tsg | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/python/extractor/tsg-python/python.tsg b/python/extractor/tsg-python/python.tsg index 8a9ab01e9915..4c5a9e6b0d0e 100644 --- a/python/extractor/tsg-python/python.tsg +++ b/python/extractor/tsg-python/python.tsg @@ -1277,16 +1277,18 @@ attr (@class.inner_scope -> @stmt.node) body = (named-child-index @stmt) } -; Class.bases - using `(_ !name)` as a proxy for all non-keyword arguments. +; Class.bases - using `(_ !value !name)` as a proxy for all non-keyword arguments. +; In particular, `keyword_argument` nodes have a `name` field, and `dictionary_splat` +; nodes have a `value` field. (class_definition - superclasses: (argument_list element: (_ !name) @arg) + superclasses: (argument_list element: (_ !value !name) @arg) ) @class { edge @class.class_expr -> @arg.node attr (@class.class_expr -> @arg.node) bases = (named-child-index @arg) } -; Class.keywords +; Class.keywords of the form `foo=bar` (class_definition superclasses: (argument_list element: (keyword_argument) @arg) ) @class @@ -1295,6 +1297,15 @@ attr (@class.class_expr -> @arg.node) keywords = (named-child-index @arg) } +; Class.keywords of the form `**kwargs` +(class_definition + superclasses: (argument_list element: (dictionary_splat) @arg) +) @class +{ + edge @class.class_expr -> @arg.node + attr (@class.class_expr -> @arg.node) keywords = (named-child-index @arg) +} + ;;;;;; End of Class ;;;;;; Assign statements From 9803bbdc4b1060b63ab1f85cc944b07e7e18d27e Mon Sep 17 00:00:00 2001 From: Taus Date: Fri, 18 Oct 2024 12:24:35 +0000 Subject: [PATCH 2/2] Python: Update class parser test --- python/extractor/tests/parser/class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/extractor/tests/parser/class.py b/python/extractor/tests/parser/class.py index d1d4694fb3f2..ffaefbcb5dac 100644 --- a/python/extractor/tests/parser/class.py +++ b/python/extractor/tests/parser/class.py @@ -1,2 +1,2 @@ -class Foo(int, object, metaclass=type): +class Foo(int, object, metaclass=type, **kwargs): x = 5