Add CommentNode to the AST

src/Lexer.php

@@ -412,7 +412,9 @@ private function lexComment(): void
             throw new SyntaxError('Unclosed comment.', $this->lineno, $this->source);
         }
 
-        $this->moveCursor(substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0]);
+        $text = substr($this->code, $this->cursor, $match[0][1] - $this->cursor).$match[0][0];
+        $this->pushToken(/* Token::COMMENT_TYPE */ 14, trim(substr($text, 0, strrpos($text, '#}'))));
+        $this->moveCursor($text);
     }
 
     private function lexString(): void

src/Node/CommentNode.php

@@ -0,0 +1,35 @@
+<?php
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ * (c) Armin Ronacher
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Twig\Node;
+
+use Twig\Attribute\YieldReady;
+use Twig\Compiler;
+
+/**
+ * Represents a comment node.
+ *
+ * @author Jeroen Versteeg <[email protected]>
+ */
+#[YieldReady]
+class CommentNode extends Node
+{
+    public function __construct(string $data, int $lineno)
+    {
+        parent::__construct([], ['text' => $data], $lineno);
+    }
+
+    public function compile(Compiler $compiler): void
+    {
+        // skip comments in compilation
+    }
+}

src/Parser.php

@@ -16,6 +16,7 @@
 use Twig\Node\BlockNode;
 use Twig\Node\BlockReferenceNode;
 use Twig\Node\BodyNode;
+use Twig\Node\CommentNode;
 use Twig\Node\Expression\AbstractExpression;
 use Twig\Node\MacroNode;
 use Twig\Node\ModuleNode;
@@ -125,6 +126,11 @@ public function subparse($test, bool $dropNeedle = false): Node
                     $rv[] = new TextNode($token->getValue(), $token->getLine());
                     break;
 
+                case /* Token::COMMENT_TYPE */ 14:
+                    $token = $this->stream->next();
+                    $rv[] = new CommentNode($token->getValue(), $token->getLine());
+                    break;
+
                 case /* Token::VAR_START_TYPE */ 2:
                     $token = $this->stream->next();
                     $expr = $this->expressionParser->parseExpression();

src/Token.php

@@ -36,6 +36,7 @@ final class Token
     public const INTERPOLATION_END_TYPE = 11;
     public const ARROW_TYPE = 12;
     public const SPREAD_TYPE = 13;
+    public const COMMENT_TYPE = 14;
 
     public function __construct(int $type, $value, int $lineno)
     {
@@ -137,6 +138,9 @@ public static function typeToString(int $type, bool $short = false): string
             case self::SPREAD_TYPE:
                 $name = 'SPREAD_TYPE';
                 break;
+            case self::COMMENT_TYPE:
+                $name = 'COMMENT_TYPE';
+                break;
             default:
                 throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type));
         }
@@ -177,6 +181,8 @@ public static function typeToEnglish(int $type): string
                 return 'arrow function';
             case self::SPREAD_TYPE:
                 return 'spread operator';
+            case self::COMMENT_TYPE:
+                return 'comment';
             default:
                 throw new \LogicException(sprintf('Token of type "%s" does not exist.', $type));
         }

tests/LexerTest.php

@@ -353,6 +353,26 @@ public function testUnterminatedBlock()
         $lexer->tokenize(new Source($template, 'index'));
     }
 
+    public function testCommentValues()
+    {
+        $template = '{# comment #}some text{#another one#}';
+        $lexer = new Lexer(new Environment($this->createMock(LoaderInterface::class)));
+        $stream = $lexer->tokenize(new Source($template, 'index'));
+
+        self::assertEquals(
+            'comment', // assert that whitespace is stripped
+            $stream->expect(Token::COMMENT_TYPE)->getValue() // implicit assertion is that expect() doesn't throw
+        );
+        self::assertEquals(
+            'some text',
+            $stream->expect(Token::TEXT_TYPE)->getValue()
+        );
+        self::assertEquals(
+            'another one', // assert that comment is parsed
+            $stream->expect(Token::COMMENT_TYPE)->getValue()
+        );
+    }
+
     public function testOverridingSyntax()
     {
         $template = '[# comment #]{# variable #}/# if true #/true/# endif #/';
@@ -362,6 +382,7 @@ public function testOverridingSyntax()
             'tag_variable' => ['{#', '#}'],
         ]);
         $stream = $lexer->tokenize(new Source($template, 'index'));
+        $stream->expect(Token::COMMENT_TYPE);
         $stream->expect(Token::VAR_START_TYPE);
         $stream->expect(Token::NAME_TYPE, 'variable');
         $stream->expect(Token::VAR_END_TYPE);

tests/Node/CommentTest.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace Twig\Tests\Node;
+
+/*
+ * This file is part of Twig.
+ *
+ * (c) Fabien Potencier
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+use Twig\Node\CommentNode;
+use Twig\Test\NodeTestCase;
+
+class CommentTest extends NodeTestCase
+{
+    public function testConstructor()
+    {
+        $node = new CommentNode('foo', 1);
+
+        $this->assertEquals('foo', $node->getAttribute('text'));
+    }
+
+    public function getTests()
+    {
+        return [
+            [new CommentNode('foo', 1), ''],
+        ];
+    }
+}