Add CommentNode to the AST

These nodes have no effect on compiled templates.

Adding these to the TokenStream and AST enables extensions to add logic in
node visitors, such as parsing PHPDoc-style `@var string name` comments.

This in turn is invaluable for static code analysis of Twig templates as
briefly discussed in [twigphp#4003](twigphp#4003) and in more detail in [TwigStan](https://github.com/alisqi/twig-stan/)'s README.

src/Lexer.php

@@ -412,7 +412,9 @@ private function lexComment(): void
             throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source);
         }
 
-        $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
+        $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor) . $match[0][0];
+        $this->pushToken(/* Token::COMMENT_TYPE */ 14, trim(substr($text, 0, strrpos($text, '#}'))));
+        $this->moveCursor($text);
     }
 
     private function lexString(): void

src/Node/CommentNode.php

@@ -0,0 +1,35 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ * (c) Armin Ronacher
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Twig\Node;
+
+use Twig\Attribute\YieldReady;
+use Twig\Compiler;
+
+/**
+ * Represents a comment node.
+ *
+ * @author Jeroen Versteeg <[email protected]>
+ */
+#[YieldReady]
+class CommentNode extends Node
+{
+    public function __construct(string $data, int $lineno)
+    {
+        parent::__construct([], ['text' => $data], $lineno);
+    }
+
+    public function compile(Compiler $compiler): void
+    {
+        // skip comments in compilation
+    }
+}

src/Parser.php

@@ -16,6 +16,7 @@
 use Twig\Node\BlockNode;
 use Twig\Node\BlockReferenceNode;
 use Twig\Node\BodyNode;
+use Twig\Node\CommentNode;
 use Twig\Node\Expression\AbstractExpression;
 use Twig\Node\MacroNode;
 use Twig\Node\ModuleNode;
@@ -125,6 +126,11 @@ public function subparse($test, bool $dropNeedle = false): Node
                     $rv[] = new TextNode($token->getValue(), $token->getLine());
                     break;
 
+                case /* Token::COMMENT_TYPE */ 14:
+                    $token = $this->stream->next();
+                    $rv[] = new CommentNode($token->getValue(), $token->getLine());
+                    break;
+
                 case /* Token::VAR_START_TYPE */ 2:
                     $token = $this->stream->next();
                     $expr = $this->expressionParser->parseExpression();

src/Token.php

@@ -36,6 +36,7 @@ final class Token
     public const INTERPOLATION_END_TYPE = 11;
     public const ARROW_TYPE = 12;
     public const SPREAD_TYPE = 13;
+    public const COMMENT_TYPE = 14;
 
     public function __construct(int $type, $value, int $lineno)
     {
@@ -137,6 +138,9 @@ public static function typeToString(int $type, bool $short = false): string
             case self::SPREAD_TYPE:
                 $name = 'SPREAD_TYPE';
                 break;
+            case self::COMMENT_TYPE:
+                $name = 'COMMENT_TYPE';
+                break;
             default:
                 throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type));
         }
@@ -177,6 +181,8 @@ public static function typeToEnglish(int $type): string
                 return 'arrow function';
             case self::SPREAD_TYPE:
                 return 'spread operator';
+            case self::COMMENT_TYPE:
+                return 'comment';
             default:
                 throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type));
         }

tests/LexerTest.php

@@ -353,6 +353,26 @@ public function testUnterminatedBlock()
         $lexer->tokenize(new Source($template, 'index'));
     }
 
+    public function testCommentValues()
+    {
+        $template = '{# comment #}some text{#another one#}';
+        $lexer = new Lexer(new Environment($this->createMock(LoaderInterface::class)));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+
+        self::assertEquals(
+            'comment', // assert that whitespace is stripped
+            $stream->expect(Token::COMMENT_TYPE)->getValue() // implicit assertion is that expect() doesn't throw
+        );
+        self::assertEquals(
+            'some text',
+            $stream->expect(Token::TEXT_TYPE)->getValue()
+        );
+        self::assertEquals(
+            'another one', // assert that comment is parsed
+            $stream->expect(Token::COMMENT_TYPE)->getValue()
+        );
+    }
+
     public function testOverridingSyntax()
     {
         $template = '[# comment #]{# variable #}/# if true #/true/# endif #/';
@@ -362,6 +382,7 @@ public function testOverridingSyntax()
             'tag_variable' => ['{#', '#}'],
         ]);
         $stream = $lexer->tokenize(new Source($template, 'index'));
+        $stream->expect(Token::COMMENT_TYPE);
         $stream->expect(Token::VAR_START_TYPE);
         $stream->expect(Token::NAME_TYPE, 'variable');
         $stream->expect(Token::VAR_END_TYPE);

tests/Node/CommentTest.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace Twig\Tests\Node;
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Twig\Node\CommentNode;
+use Twig\Test\NodeTestCase;
+
+class CommentTest extends NodeTestCase
+{
+    public function testConstructor()
+    {
+        $node = new CommentNode('foo', 1);
+
+        $this->assertEquals('foo', $node->getAttribute('text'));
+    }
+
+    public function getTests()
+    {
+        return [
+            [new CommentNode('foo', 1), ""],
+        ];
+    }
+}