pythongh-125588: Allow to regenerate the parser with Python < 3.12 (p…

…ython#127969)

Signed-off-by: Pablo Galindo <[email protected]>

Tools/peg_generator/pegen/parser.py

@@ -207,7 +207,7 @@ def string(self) -> Optional[tokenize.TokenInfo]:
 
     @memoize
     def fstring_start(self) -> Optional[tokenize.TokenInfo]:
-        FSTRING_START = getattr(token, "FSTRING_START")
+        FSTRING_START = getattr(token, "FSTRING_START", None)
         if not FSTRING_START:
             return None
         tok = self._tokenizer.peek()
@@ -217,7 +217,7 @@ def fstring_start(self) -> Optional[tokenize.TokenInfo]:
 
     @memoize
     def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
-        FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE")
+        FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None)
         if not FSTRING_MIDDLE:
             return None
         tok = self._tokenizer.peek()
@@ -227,7 +227,7 @@ def fstring_middle(self) -> Optional[tokenize.TokenInfo]:
 
     @memoize
     def fstring_end(self) -> Optional[tokenize.TokenInfo]:
-        FSTRING_END = getattr(token, "FSTRING_END")
+        FSTRING_END = getattr(token, "FSTRING_END", None)
         if not FSTRING_END:
             return None
         tok = self._tokenizer.peek()

Tools/peg_generator/pegen/parser_generator.py

@@ -1,3 +1,4 @@
+import sys
 import ast
 import contextlib
 import re
@@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor):
     def __init__(self, rules: Dict[str, Rule], tokens: Set[str]):
         self.rules = rules
         self.tokens = tokens
+        # If python < 3.12 add the virtual fstring tokens
+        if sys.version_info < (3, 12):
+            self.tokens.add("FSTRING_START")
+            self.tokens.add("FSTRING_END")
+            self.tokens.add("FSTRING_MIDDLE")
 
     def visit_NameLeaf(self, node: NameLeaf) -> None:
         if node.value not in self.rules and node.value not in self.tokens: