diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index 692eb9ed2417d7..a987d30a9d6438 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -207,7 +207,7 @@ def string(self) -> Optional[tokenize.TokenInfo]: @memoize def fstring_start(self) -> Optional[tokenize.TokenInfo]: - FSTRING_START = getattr(token, "FSTRING_START") + FSTRING_START = getattr(token, "FSTRING_START", None) if not FSTRING_START: return None tok = self._tokenizer.peek() @@ -217,7 +217,7 @@ def fstring_start(self) -> Optional[tokenize.TokenInfo]: @memoize def fstring_middle(self) -> Optional[tokenize.TokenInfo]: - FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None) if not FSTRING_MIDDLE: return None tok = self._tokenizer.peek() @@ -227,7 +227,7 @@ def fstring_middle(self) -> Optional[tokenize.TokenInfo]: @memoize def fstring_end(self) -> Optional[tokenize.TokenInfo]: - FSTRING_END = getattr(token, "FSTRING_END") + FSTRING_END = getattr(token, "FSTRING_END", None) if not FSTRING_END: return None tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index b42b12c8aa0dee..6ce0649aefe7ff 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -1,3 +1,4 @@ +import sys import ast import contextlib import re @@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor): def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.rules = rules self.tokens = tokens + # If python < 3.12 add the virtual fstring tokens + if sys.version_info < (3, 12): + self.tokens.add("FSTRING_START") + self.tokens.add("FSTRING_END") + self.tokens.add("FSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: