TASK: Remove Tokenizer and all related obsolete concepts

PackageFactory · Aug 11, 2023 · 66e0e7c · 66e0e7c
1 parent dd1625d
commit 66e0e7c
Show file tree

Hide file tree

Showing 20 changed files with 4 additions and 1,759 deletions.
diff --git a/src/Language/AST/Node/IntegerLiteral/IntegerFormat.php b/src/Language/AST/Node/IntegerLiteral/IntegerFormat.php
@@ -22,8 +22,6 @@
 
 namespace PackageFactory\ComponentEngine\Language\AST\Node\IntegerLiteral;
 
-use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType;
-
 enum IntegerFormat: string
 {
     case BINARY = 'BINARY';

diff --git a/src/Language/Parser/Module/ModuleCouldNotBeParsed.php b/src/Language/Parser/Module/ModuleCouldNotBeParsed.php
@@ -23,7 +23,6 @@
 namespace PackageFactory\ComponentEngine\Language\Parser\Module;
 
 use PackageFactory\ComponentEngine\Language\Parser\ParserException;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Token;
 
 final class ModuleCouldNotBeParsed extends ParserException
 {

diff --git a/src/Language/Parser/StructDeclaration/StructDeclarationParser.php b/src/Language/Parser/StructDeclaration/StructDeclarationParser.php
@@ -29,10 +29,8 @@
 use PackageFactory\ComponentEngine\Language\AST\Node\StructDeclaration\StructNameNode;
 use PackageFactory\ComponentEngine\Language\Lexer\Lexer;
 use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenType;
-use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenTypes;
 use PackageFactory\ComponentEngine\Language\Parser\PropertyDeclaration\PropertyDeclarationParser;
 use PackageFactory\ComponentEngine\Parser\Source\Range;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Token as TokenizerToken;
 
 final class StructDeclarationParser
 {

diff --git a/src/Language/Parser/Tag/TagCouldNotBeParsed.php b/src/Language/Parser/Tag/TagCouldNotBeParsed.php
@@ -25,11 +25,11 @@
 use PackageFactory\ComponentEngine\Domain\TagName\TagName;
 use PackageFactory\ComponentEngine\Language\Parser\ParserException;
 use PackageFactory\ComponentEngine\Parser\Source\Range;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\Token;
-use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenTypes;
 
 final class TagCouldNotBeParsed extends ParserException
 {
+    protected const TITLE = 'Tag could not be parsed';
+
     public static function becauseOfClosingTagNameMismatch(
         TagName $expectedTagName,
         string $actualTagName,
@@ -38,27 +38,11 @@ public static function becauseOfClosingTagNameMismatch(
         return new self(
             code: 1690976372,
             message: sprintf(
-                'Tag could not be parsed, because the closing tag name "%s" did not match the opening tag name "%s".',
+                'Closing tag name "%s" did not match the opening tag name "%s".',
                 $actualTagName,
                 $expectedTagName->value
             ),
             affectedRangeInSource: $affectedRangeInSource
         );
     }
-
-    public static function becauseOfUnexpectedToken(
-        TokenTypes $expectedTokenTypes,
-        Token $actualToken
-    ): self {
-        return new self(
-            code: 1691156112,
-            message: sprintf(
-                'Tag could not be parsed because of unexpected token %s. '
-                . 'Expected %s instead.',
-                $actualToken->toDebugString(),
-                $expectedTokenTypes->toDebugString()
-            ),
-            affectedRangeInSource: $actualToken->boundaries
-        );
-    }
 }
diff --git a/src/Parser/Source/Fragment.php b/src/Parser/Source/Fragment.php
diff --git a/src/Parser/Source/Source.php b/src/Parser/Source/Source.php
@@ -22,10 +22,7 @@
 
 namespace PackageFactory\ComponentEngine\Parser\Source;
 
-/**
- * @implements \IteratorAggregate<mixed, Fragment>
- */
-final class Source implements \IteratorAggregate
+final class Source
 {
     public function __construct(
         public readonly Path $path,
@@ -51,32 +48,4 @@ public function equals(Source $other): bool
     {
         return $this->contents === $other->contents;
     }
-
-    /**
-     * @return \Iterator<Fragment>
-     */
-    public function getIterator(): \Iterator
-    {
-        $lineNumber = 0;
-        $columnNumber = 0;
-        $length = strlen($this->contents);
-
-        for ($index = 0; $index < $length; $index++) {
-            $character = $this->contents[$index];
-
-            yield Fragment::create(
-                $character,
-                new Position($lineNumber, $columnNumber),
-                new Position($lineNumber, $columnNumber),
-                $this
-            );
-
-            if ($character === "\n") {
-                $lineNumber++;
-                $columnNumber = 0;
-            } else {
-                $columnNumber++;
-            }
-        }
-    }
 }
diff --git a/src/Parser/Tokenizer/Buffer.php b/src/Parser/Tokenizer/Buffer.php
diff --git a/src/Parser/Tokenizer/CharacterType.php b/src/Parser/Tokenizer/CharacterType.php
diff --git a/src/Parser/Tokenizer/LookAhead.php b/src/Parser/Tokenizer/LookAhead.php