Skip to content

Commit

Permalink
Python: Fix generator expression locations
Browse files Browse the repository at this point in the history
Our logic for detecting the first and last item in a generator
expression was faulty, sometimes matching comments as well. Because
attributes (like `_location_start`) can only be written once, this
caused `tree-sitter-graph` to get unhappy.

To fix this, we now require the first item to be an `expression`, and
the last one to be either a `for_in_clause` or an `if_clause`.
Crucially, `comment` is neither of these, and this prevents the
unfortunate overlap.
  • Loading branch information
tausbn committed Oct 28, 2024
1 parent ef60b73 commit 5d6600e
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 4 deletions.
7 changes: 7 additions & 0 deletions python/extractor/tests/parser/comprehensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,10 @@
d for e in f if g # comment
# comment
] # comment

# Generator expression with comments
(# comment
alpha # comment
for beta in gamma # comment
# comment
)
8 changes: 4 additions & 4 deletions python/extractor/tsg-python/python.tsg
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@

;;; GeneratorExp

(generator_expression . "(" . (comment)* . (_) @start (_) @end . (comment)* . ")" .) @generatorexp
(generator_expression . "(" . (comment)* . (expression) @start [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @generatorexp
{
attr (@generatorexp.node) _location_start = (location-start @start)
attr (@generatorexp.node) _location_end = (location-end @end)
Expand All @@ -416,13 +416,13 @@
attr (@if.node) _location_end = (location-end @expr)
}

(generator_expression . "(" . (comment)* . (_) @start (for_in_clause) @child (_) @end . (comment)* . ")" .) @genexpr
(generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @child [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @genexpr
{
attr (@child.node) _location_start = (location-start @start)
attr (@child.node) _location_end = (location-end @end)
}

(generator_expression . "(" . (comment)* . (_) @start (for_in_clause) @end . (comment)* . ")" .) @genexpr
(generator_expression . "(" . (comment)* . (expression) @start (for_in_clause) @end . (comment)* . ")" .) @genexpr
{
attr (@end.node) _location_start = (location-start @start)
attr (@end.node) _location_end = (location-end @end)
Expand Down Expand Up @@ -863,7 +863,7 @@
; information for the entire generator expression (yes, it is a wide parameter!) and so we must recreate the logic for
; setting this location information correctly.

(generator_expression . "(" . (comment)* . (_) @start (_) @end . (comment)* . ")" .) @genexpr
(generator_expression . "(" . (comment)* . (expression) @start [(for_in_clause) (if_clause)] @end . (comment)* . ")" .) @genexpr
{
; Synthesize the `genexpr` function
let @genexpr.fun = (ast-node @genexpr "Function")
Expand Down

0 comments on commit 5d6600e

Please sign in to comment.