From 66e0e7c75502d8fdbbe97c3d631550faea772845 Mon Sep 17 00:00:00 2001 From: Wilhelm Behncke Date: Fri, 11 Aug 2023 13:24:07 +0200 Subject: [PATCH] TASK: Remove Tokenizer and all related obsolete concepts --- .../AST/Node/IntegerLiteral/IntegerFormat.php | 2 - .../Parser/Module/ModuleCouldNotBeParsed.php | 1 - .../StructDeclarationParser.php | 2 - .../Parser/Tag/TagCouldNotBeParsed.php | 22 +- src/Parser/Source/Fragment.php | 63 --- src/Parser/Source/Source.php | 33 +- src/Parser/Tokenizer/Buffer.php | 62 --- src/Parser/Tokenizer/CharacterType.php | 68 --- src/Parser/Tokenizer/LookAhead.php | 80 --- src/Parser/Tokenizer/Scanner.php | 190 ------- src/Parser/Tokenizer/Token.php | 73 --- src/Parser/Tokenizer/TokenType.php | 226 -------- src/Parser/Tokenizer/TokenTypes.php | 66 --- src/Parser/Tokenizer/Tokenizer.php | 484 ------------------ .../PhpTranspilerIntegrationTest.php | 1 - test/Unit/Language/Parser/ParserTestCase.php | 15 - test/Unit/Parser/Tokenizer/Fixtures.php | 42 -- test/Unit/Parser/Tokenizer/TokenTest.php | 55 -- test/Unit/Parser/Tokenizer/TokenTypesTest.php | 110 ---- test/Unit/Parser/Tokenizer/TokenizerTest.php | 168 ------ 20 files changed, 4 insertions(+), 1759 deletions(-) delete mode 100644 src/Parser/Source/Fragment.php delete mode 100644 src/Parser/Tokenizer/Buffer.php delete mode 100644 src/Parser/Tokenizer/CharacterType.php delete mode 100644 src/Parser/Tokenizer/LookAhead.php delete mode 100644 src/Parser/Tokenizer/Scanner.php delete mode 100644 src/Parser/Tokenizer/Token.php delete mode 100644 src/Parser/Tokenizer/TokenType.php delete mode 100644 src/Parser/Tokenizer/TokenTypes.php delete mode 100644 src/Parser/Tokenizer/Tokenizer.php delete mode 100644 test/Unit/Parser/Tokenizer/Fixtures.php delete mode 100644 test/Unit/Parser/Tokenizer/TokenTest.php delete mode 100644 test/Unit/Parser/Tokenizer/TokenTypesTest.php delete mode 100644 test/Unit/Parser/Tokenizer/TokenizerTest.php diff --git a/src/Language/AST/Node/IntegerLiteral/IntegerFormat.php b/src/Language/AST/Node/IntegerLiteral/IntegerFormat.php index 445ffd10..a4ce9314 100644 --- a/src/Language/AST/Node/IntegerLiteral/IntegerFormat.php +++ b/src/Language/AST/Node/IntegerLiteral/IntegerFormat.php @@ -22,8 +22,6 @@ namespace PackageFactory\ComponentEngine\Language\AST\Node\IntegerLiteral; -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType; - enum IntegerFormat: string { case BINARY = 'BINARY'; diff --git a/src/Language/Parser/Module/ModuleCouldNotBeParsed.php b/src/Language/Parser/Module/ModuleCouldNotBeParsed.php index 4817cb5b..88a60588 100644 --- a/src/Language/Parser/Module/ModuleCouldNotBeParsed.php +++ b/src/Language/Parser/Module/ModuleCouldNotBeParsed.php @@ -23,7 +23,6 @@ namespace PackageFactory\ComponentEngine\Language\Parser\Module; use PackageFactory\ComponentEngine\Language\Parser\ParserException; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Token; final class ModuleCouldNotBeParsed extends ParserException { diff --git a/src/Language/Parser/StructDeclaration/StructDeclarationParser.php b/src/Language/Parser/StructDeclaration/StructDeclarationParser.php index 57f88f81..b6a6a47e 100644 --- a/src/Language/Parser/StructDeclaration/StructDeclarationParser.php +++ b/src/Language/Parser/StructDeclaration/StructDeclarationParser.php @@ -29,10 +29,8 @@ use PackageFactory\ComponentEngine\Language\AST\Node\StructDeclaration\StructNameNode; use PackageFactory\ComponentEngine\Language\Lexer\Lexer; use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenType; -use PackageFactory\ComponentEngine\Language\Lexer\Token\TokenTypes; use PackageFactory\ComponentEngine\Language\Parser\PropertyDeclaration\PropertyDeclarationParser; use PackageFactory\ComponentEngine\Parser\Source\Range; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Token as TokenizerToken; final class StructDeclarationParser { diff --git a/src/Language/Parser/Tag/TagCouldNotBeParsed.php b/src/Language/Parser/Tag/TagCouldNotBeParsed.php index 91408753..46f39b4d 100644 --- a/src/Language/Parser/Tag/TagCouldNotBeParsed.php +++ b/src/Language/Parser/Tag/TagCouldNotBeParsed.php @@ -25,11 +25,11 @@ use PackageFactory\ComponentEngine\Domain\TagName\TagName; use PackageFactory\ComponentEngine\Language\Parser\ParserException; use PackageFactory\ComponentEngine\Parser\Source\Range; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Token; -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenTypes; final class TagCouldNotBeParsed extends ParserException { + protected const TITLE = 'Tag could not be parsed'; + public static function becauseOfClosingTagNameMismatch( TagName $expectedTagName, string $actualTagName, @@ -38,27 +38,11 @@ public static function becauseOfClosingTagNameMismatch( return new self( code: 1690976372, message: sprintf( - 'Tag could not be parsed, because the closing tag name "%s" did not match the opening tag name "%s".', + 'Closing tag name "%s" did not match the opening tag name "%s".', $actualTagName, $expectedTagName->value ), affectedRangeInSource: $affectedRangeInSource ); } - - public static function becauseOfUnexpectedToken( - TokenTypes $expectedTokenTypes, - Token $actualToken - ): self { - return new self( - code: 1691156112, - message: sprintf( - 'Tag could not be parsed because of unexpected token %s. ' - . 'Expected %s instead.', - $actualToken->toDebugString(), - $expectedTokenTypes->toDebugString() - ), - affectedRangeInSource: $actualToken->boundaries - ); - } } diff --git a/src/Parser/Source/Fragment.php b/src/Parser/Source/Fragment.php deleted file mode 100644 index 8f4bc736..00000000 --- a/src/Parser/Source/Fragment.php +++ /dev/null @@ -1,63 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Source; - -final class Fragment -{ - private function __construct( - public readonly string $value, - public readonly Position $start, - public readonly Position $end, - public readonly Source $source - ) { - } - - public static function create( - string $value, - Position $start, - Position $end, - Source $source - ): Fragment { - return new Fragment( - $value, - $start, - $end, - $source - ); - } - - public function append(Fragment $other): Fragment - { - return new Fragment( - $this->value . $other->value, - $this->start, - $other->end, - $this->source - ); - } - - public function __toString(): string - { - return $this->value; - } -} diff --git a/src/Parser/Source/Source.php b/src/Parser/Source/Source.php index ba143ca7..dfc7bbe8 100644 --- a/src/Parser/Source/Source.php +++ b/src/Parser/Source/Source.php @@ -22,10 +22,7 @@ namespace PackageFactory\ComponentEngine\Parser\Source; -/** - * @implements \IteratorAggregate - */ -final class Source implements \IteratorAggregate +final class Source { public function __construct( public readonly Path $path, @@ -51,32 +48,4 @@ public function equals(Source $other): bool { return $this->contents === $other->contents; } - - /** - * @return \Iterator - */ - public function getIterator(): \Iterator - { - $lineNumber = 0; - $columnNumber = 0; - $length = strlen($this->contents); - - for ($index = 0; $index < $length; $index++) { - $character = $this->contents[$index]; - - yield Fragment::create( - $character, - new Position($lineNumber, $columnNumber), - new Position($lineNumber, $columnNumber), - $this - ); - - if ($character === "\n") { - $lineNumber++; - $columnNumber = 0; - } else { - $columnNumber++; - } - } - } } diff --git a/src/Parser/Tokenizer/Buffer.php b/src/Parser/Tokenizer/Buffer.php deleted file mode 100644 index 145ff089..00000000 --- a/src/Parser/Tokenizer/Buffer.php +++ /dev/null @@ -1,62 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Fragment; - -final class Buffer -{ - private function __construct( - private ?Fragment $fragment - ) { - } - - public static function empty(): self - { - return new self(null); - } - - public function append(Fragment $fragment): self - { - $this->fragment = $this->fragment?->append($fragment) ?? $fragment; - return $this; - } - - public function value(): string - { - return $this->fragment?->value ?? ''; - } - - public function isEmpty(): bool - { - return $this->fragment === null; - } - - public function flush(TokenType $tokenType): \Iterator - { - if ($this->fragment !== null) { - yield Token::fromFragment($tokenType, $this->fragment); - $this->fragment = null; - } - } -} diff --git a/src/Parser/Tokenizer/CharacterType.php b/src/Parser/Tokenizer/CharacterType.php deleted file mode 100644 index b74703b8..00000000 --- a/src/Parser/Tokenizer/CharacterType.php +++ /dev/null @@ -1,68 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -enum CharacterType -{ - case BRACKET_OPEN; - case BRACKET_CLOSE; - case ANGLE_OPEN; - case ANGLE_CLOSE; - case STRING_DELIMITER; - case TEMPLATE_LITERAL_DELIMITER; - case COMMENT_DELIMITER; - case ESCAPE; - case FORWARD_SLASH; - case PERIOD; - case SYMBOL; - case DIGIT; - case SPACE; - case OTHER; - - public static function get(string $character): self - { - return match ($character) { - '(', '[', '{' => self::BRACKET_OPEN, - ')', ']', '}' => self::BRACKET_CLOSE, - '<' => self::ANGLE_OPEN, - '>' => self::ANGLE_CLOSE, - '\'', '"' => self::STRING_DELIMITER, - '`' => self::TEMPLATE_LITERAL_DELIMITER, - '#' => self::COMMENT_DELIMITER, - '\\' => self::ESCAPE, - '/' => self::FORWARD_SLASH, - '.' => self::PERIOD, - '!', '%', '&', '|', '=', '?', ':', '-', ',', '+', '*', '$' => self::SYMBOL, - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' => self::DIGIT, - default => match (true) { - ctype_space($character) => self::SPACE, - default => self::OTHER - } - }; - } - - public function is(string $character): bool - { - return self::get($character) === $this; - } -} diff --git a/src/Parser/Tokenizer/LookAhead.php b/src/Parser/Tokenizer/LookAhead.php deleted file mode 100644 index 4f7c3251..00000000 --- a/src/Parser/Tokenizer/LookAhead.php +++ /dev/null @@ -1,80 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -/** - * @implements \IteratorAggregate - */ -final class LookAhead implements \IteratorAggregate -{ - /** - * @var Token[] - */ - private array $buffer = []; - - /** - * @param \Iterator $tokens - */ - private function __construct( - public \Iterator $tokens - ) { - } - - /** - * @param \Iterator $tokens - * @return self - */ - public static function fromTokens(\Iterator $tokens): self - { - return new self(tokens: $tokens); - } - - /** - * @return \Iterator - */ - public function getIterator(): \Iterator - { - foreach ($this->buffer as $token) { - yield $token; - } - - if (!Scanner::isEnd($this->tokens)) { - yield from $this->tokens; - } - } - - public function shift(): void - { - Scanner::assertValid($this->tokens); - $this->buffer[] = $this->tokens->current(); - Scanner::skipOne($this->tokens); - } - - public function type(): ?TokenType - { - if (Scanner::isEnd($this->tokens)) { - return null; - } - return Scanner::type($this->tokens); - } -} diff --git a/src/Parser/Tokenizer/Scanner.php b/src/Parser/Tokenizer/Scanner.php deleted file mode 100644 index 3e11f3e3..00000000 --- a/src/Parser/Tokenizer/Scanner.php +++ /dev/null @@ -1,190 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Path; - -final class Scanner -{ - /** - * @param \Iterator $tokens - * @return void - */ - public static function assertValid(\Iterator $tokens): void - { - if (!$tokens->valid()) { - throw new \Exception("@TODO: Unexpected end of file."); - } - } - - /** - * @param \Iterator $tokens - * @param TokenType ...$types - * @return void - */ - public static function assertType(\Iterator $tokens, TokenType ...$types): void - { - self::assertValid($tokens); - - $actualType = $tokens->current()->type; - foreach ($types as $expectedType) { - if ($actualType === $expectedType) { - return; - } - } - - throw new \Exception( - "@TODO: Unexpected token: " - . $actualType->value - . " at " - . ($tokens->current()->boundaries->start->lineNumber + 1) - . ":" - . ($tokens->current()->boundaries->start->columnNumber + 1) - ); - } - - /** - * @param \Iterator $tokens - * @param string ...$values - * @return void - */ - public static function assertValue(\Iterator $tokens, string ...$values): void - { - self::assertValid($tokens); - - $actualValue = $tokens->current()->value; - foreach ($values as $expectedValue) { - if ($actualValue === $expectedValue) { - return; - } - } - - throw new \Exception("@TODO: Unexpected value: " . $actualValue); - } - - /** - * @param \Iterator $tokens - * @return \Iterator - */ - public static function skipOne(\Iterator &$tokens): \Iterator - { - $tokens->next(); - return $tokens; - } - - /** - * @param \Iterator $tokens - * @return void - */ - public static function skipSpace(\Iterator $tokens): void - { - while ( - $tokens->valid() && match ($tokens->current()->type) { - TokenType::SPACE, - TokenType::END_OF_LINE => true, - default => false - } - ) { - $tokens->next(); - } - } - - /** - * @param \Iterator $tokens - * @return void - */ - public static function skipSpaceAndComments(\Iterator $tokens): void - { - while ( - $tokens->valid() && match ($tokens->current()->type) { - TokenType::SPACE, - TokenType::END_OF_LINE, - TokenType::COMMENT => true, - default => false - } - ) { - $tokens->next(); - } - } - - /** - * @param \Iterator $tokens - * @return string - */ - public static function value(\Iterator $tokens): string - { - self::assertValid($tokens); - return $tokens->current()->value; - } - - /** - * @param \Iterator $tokens - * @return TokenType - */ - public static function type(\Iterator $tokens): TokenType - { - self::assertValid($tokens); - return $tokens->current()->type; - } - - /** - * @param \Iterator $tokens - * @return Path - */ - public static function sourcePath(\Iterator $tokens): Path - { - self::assertValid($tokens); - return $tokens->current()->sourcePath; - } - - /** - * @param \Iterator $tokens - * @return bool - */ - public static function isEnd(\Iterator $tokens): bool - { - return !$tokens->valid(); - } - - /** - * @param \Iterator $tokens - */ - public static function debugPrint(\Iterator &$tokens): string - { - $tokens = (function(): \Generator { - throw new \Exception('Once debugged, $tokens is empty.'); - // @phpstan-ignore-next-line - yield; - })(); - - $tokensAsArray = []; - while ($tokens->valid()) { - $tokensAsArray[] = [ - "type" => $tokens->current()->type, - "value" => $tokens->current()->value - ]; - $tokens->next(); - } - return json_encode($tokensAsArray, JSON_PRETTY_PRINT | JSON_THROW_ON_ERROR); - } -} diff --git a/src/Parser/Tokenizer/Token.php b/src/Parser/Tokenizer/Token.php deleted file mode 100644 index 66b5eb08..00000000 --- a/src/Parser/Tokenizer/Token.php +++ /dev/null @@ -1,73 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Range; -use PackageFactory\ComponentEngine\Parser\Source\Fragment; -use PackageFactory\ComponentEngine\Parser\Source\Path; - -final class Token -{ - public function __construct( - public readonly TokenType $type, - public readonly string $value, - public readonly Range $boundaries, - public readonly Path $sourcePath - ) { - } - - public static function fromFragment( - TokenType $type, - Fragment $fragment - ): Token { - return new Token( - $type, - $fragment->value, - Range::from($fragment->start, $fragment->end), - $fragment->source->path - ); - } - - public static function emptyFromDelimitingFragments( - TokenType $type, - Fragment $startFragment, - Fragment $endFragment - ): Token { - return new Token( - $type, - '', - Range::from($startFragment->start, $endFragment->end), - $startFragment->source->path - ); - } - - public function __toString(): string - { - return $this->value; - } - - public function toDebugString(): string - { - return sprintf('%s ("%s")', $this->type->value, $this->value); - } -} diff --git a/src/Parser/Tokenizer/TokenType.php b/src/Parser/Tokenizer/TokenType.php deleted file mode 100644 index 0c2b8fae..00000000 --- a/src/Parser/Tokenizer/TokenType.php +++ /dev/null @@ -1,226 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Fragment; - -enum TokenType: string -{ - case COMMENT = 'COMMENT'; - - case KEYWORD_FROM = 'KEYWORD_FROM'; - case KEYWORD_IMPORT = 'KEYWORD_IMPORT'; - case KEYWORD_EXPORT = 'KEYWORD_EXPORT'; - case KEYWORD_ENUM = 'KEYWORD_ENUM'; - case KEYWORD_STRUCT = 'KEYWORD_STRUCT'; - case KEYWORD_COMPONENT = 'KEYWORD_COMPONENT'; - case KEYWORD_MATCH = 'KEYWORD_MATCH'; - case KEYWORD_DEFAULT = 'KEYWORD_DEFAULT'; - case KEYWORD_RETURN = 'KEYWORD_RETURN'; - case KEYWORD_TRUE = 'KEYWORD_TRUE'; - case KEYWORD_FALSE = 'KEYWORD_FALSE'; - case KEYWORD_NULL = 'KEYWORD_NULL'; - - case CONSTANT = 'CONSTANT'; - - case STRING = 'STRING'; - case STRING_QUOTED = 'STRING_QUOTED'; - - case NUMBER_BINARY = 'NUMBER_BINARY'; - case NUMBER_OCTAL = 'NUMBER_OCTAL'; - case NUMBER_DECIMAL = 'NUMBER_DECIMAL'; - case NUMBER_HEXADECIMAL = 'NUMBER_HEXADECIMAL'; - - case TEMPLATE_LITERAL_START = 'TEMPLATE_LITERAL_START'; - case TEMPLATE_LITERAL_END = 'TEMPLATE_LITERAL_END'; - - case OPERATOR_BOOLEAN_AND = 'OPERATOR_BOOLEAN_AND'; - case OPERATOR_BOOLEAN_OR = 'OPERATOR_BOOLEAN_OR'; - case OPERATOR_BOOLEAN_NOT = 'OPERATOR_BOOLEAN_NOT'; - - case COMPARATOR_EQUAL = 'COMPARATOR_EQUAL'; - case COMPARATOR_NOT_EQUAL = 'COMPARATOR_NOT_EQUAL'; - case COMPARATOR_GREATER_THAN = 'COMPARATOR_GREATER_THAN'; - case COMPARATOR_GREATER_THAN_OR_EQUAL = 'COMPARATOR_GREATER_THAN_OR_EQUAL'; - case COMPARATOR_LESS_THAN = 'COMPARATOR_LESS_THAN'; - case COMPARATOR_LESS_THAN_OR_EQUAL = 'COMPARATOR_LESS_THAN_OR_EQUAL'; - - case ARROW_SINGLE = 'ARROW_SINGLE'; - - case BRACKET_CURLY_OPEN = 'BRACKET_CURLY_OPEN'; - case BRACKET_CURLY_CLOSE = 'BRACKET_CURLY_CLOSE'; - case BRACKET_ROUND_OPEN = 'BRACKET_ROUND_OPEN'; - case BRACKET_ROUND_CLOSE = 'BRACKET_ROUND_CLOSE'; - case BRACKET_SQUARE_OPEN = 'BRACKET_SQUARE_OPEN'; - case BRACKET_SQUARE_CLOSE = 'BRACKET_SQUARE_CLOSE'; - - case TAG_START_OPENING = 'TAG_START_OPENING'; - case TAG_START_CLOSING = 'TAG_START_CLOSING'; - case TAG_SELF_CLOSE = 'TAG_SELF_CLOSE'; - case TAG_END = 'TAG_END'; - - case PERIOD = 'PERIOD'; - case COLON = 'COLON'; - case QUESTIONMARK = 'QUESTIONMARK'; - case COMMA = 'COMMA'; - case EQUALS = 'EQUALS'; - case SLASH_FORWARD = 'SLASH_FORWARD'; - case DOLLAR = 'DOLLAR'; - case PIPE = 'PIPE'; - - case OPTCHAIN = 'OPTCHAIN'; - case NULLISH_COALESCE = 'NULLISH_COALESCE'; - - case SPACE = 'SPACE'; - case END_OF_LINE = 'END_OF_LINE'; - - public static function fromBuffer(Buffer $buffer): TokenType - { - $value = $buffer->value(); - - return match (true) { - $value === 'from' => self::KEYWORD_FROM, - $value === 'import' => self::KEYWORD_IMPORT, - $value === 'export' => self::KEYWORD_EXPORT, - $value === 'enum' => self::KEYWORD_ENUM, - $value === 'struct' => self::KEYWORD_STRUCT, - $value === 'component' => self::KEYWORD_COMPONENT, - $value === 'match' => self::KEYWORD_MATCH, - $value === 'default' => self::KEYWORD_DEFAULT, - $value === 'return' => self::KEYWORD_RETURN, - $value === 'true' => self::KEYWORD_TRUE, - $value === 'false' => self::KEYWORD_FALSE, - $value === 'null' => self::KEYWORD_NULL, - - $value === '.' => self::PERIOD, - - (bool) preg_match( - '/^0[bB][0-1]+$/', - $value - ) => self::NUMBER_BINARY, - (bool) preg_match( - '/^0o[0-7]+$/', - $value - ) => self::NUMBER_OCTAL, - $value !== '' && preg_match( - '/^([-+]?[0-9]+)$/', - $value - ) => self::NUMBER_DECIMAL, - (bool) preg_match( - '/^0x[0-9a-fA-F]+$/', - $value - ) => self::NUMBER_HEXADECIMAL, - default => self::STRING - }; - } - - public static function tryBracketOpenFromFragment(Fragment $fragment): ?self - { - return match ($fragment->value) { - '{' => self::BRACKET_CURLY_OPEN, - '(' => self::BRACKET_ROUND_OPEN, - '[' => self::BRACKET_SQUARE_OPEN, - default => null - }; - } - - public function closingBracket(): TokenType - { - return match ($this) { - self::BRACKET_CURLY_OPEN => self::BRACKET_CURLY_CLOSE, - self::BRACKET_ROUND_OPEN => self::BRACKET_ROUND_CLOSE, - self::BRACKET_SQUARE_OPEN => self::BRACKET_SQUARE_CLOSE, - default => throw new \Exception('@TODO: Not a bracket.') - }; - } - - public function matchesString(string $string): bool - { - return match ($this) { - self::BRACKET_CURLY_CLOSE => $string === '}', - self::BRACKET_ROUND_CLOSE => $string === ')', - self::BRACKET_SQUARE_CLOSE => $string === ']', - default => false - }; - } - - public function toDebugString(): string - { - return $this->value . match ($this) { - self::COMMENT => ' (e.g. "# ...")', - self::KEYWORD_FROM => ' ("from")', - self::KEYWORD_IMPORT => ' ("import")', - self::KEYWORD_EXPORT => ' ("export")', - self::KEYWORD_ENUM => ' ("enum")', - self::KEYWORD_STRUCT => ' ("struct")', - self::KEYWORD_COMPONENT => ' ("component")', - self::KEYWORD_MATCH => ' ("match")', - self::KEYWORD_DEFAULT => ' ("default")', - self::KEYWORD_RETURN => ' ("return")', - self::KEYWORD_TRUE => ' ("true")', - self::KEYWORD_FALSE => ' ("false")', - self::KEYWORD_NULL => ' ("null")', - self::CONSTANT => '', - self::STRING => '', - self::STRING_QUOTED => '', - self::NUMBER_BINARY => ' (e.g. "0b1001")', - self::NUMBER_OCTAL => ' (e.g. "0o644")', - self::NUMBER_DECIMAL => ' (e.g. "42")', - self::NUMBER_HEXADECIMAL => ' (e.g. "0xABC")', - self::TEMPLATE_LITERAL_START => ' ("`")', - self::TEMPLATE_LITERAL_END => ' ("`")', - self::OPERATOR_BOOLEAN_AND => ' ("&&")', - self::OPERATOR_BOOLEAN_OR => ' ("||")', - self::OPERATOR_BOOLEAN_NOT => ' ("!")', - self::COMPARATOR_EQUAL => ' ("===")', - self::COMPARATOR_NOT_EQUAL => ' ("!==")', - self::COMPARATOR_GREATER_THAN => ' (">")', - self::COMPARATOR_GREATER_THAN_OR_EQUAL => ' (">=")', - self::COMPARATOR_LESS_THAN => ' ("<")', - self::COMPARATOR_LESS_THAN_OR_EQUAL => ' ("<=")', - self::ARROW_SINGLE => ' ("->")', - self::BRACKET_CURLY_OPEN => ' ("{")', - self::BRACKET_CURLY_CLOSE => ' ("}")', - self::BRACKET_ROUND_OPEN => ' ("(")', - self::BRACKET_ROUND_CLOSE => ' (")")', - self::BRACKET_SQUARE_OPEN => ' ("[")', - self::BRACKET_SQUARE_CLOSE => ' ("]")', - self::TAG_START_OPENING => ' ("<")', - self::TAG_START_CLOSING => ' (" ' ("/>")', - self::TAG_END => ' (">")', - self::PERIOD => ' (".")', - self::COLON => ' (":")', - self::QUESTIONMARK => ' ("?")', - self::COMMA => ' (",")', - self::EQUALS => ' ("=")', - self::SLASH_FORWARD => ' ("/")', - self::DOLLAR => ' ("$")', - self::PIPE => ' ("|")', - self::OPTCHAIN => ' ("?.")', - self::NULLISH_COALESCE => ' ("??")', - self::SPACE => '', - self::END_OF_LINE => '' - }; - } -} diff --git a/src/Parser/Tokenizer/TokenTypes.php b/src/Parser/Tokenizer/TokenTypes.php deleted file mode 100644 index 135e1d77..00000000 --- a/src/Parser/Tokenizer/TokenTypes.php +++ /dev/null @@ -1,66 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -final class TokenTypes -{ - /** - * @var TokenType[] - */ - private readonly array $items; - - private function __construct(TokenType ...$items) - { - assert(count($items) > 0); - - $this->items = $items; - } - - public static function from(TokenType ...$items): self - { - $items = array_unique($items, SORT_REGULAR); - $items = array_values($items); - - return new self(...$items); - } - - public function contains(TokenType $needle): bool - { - return in_array($needle, $this->items); - } - - public function toDebugString(): string - { - if (count($this->items) === 1) { - return $this->items[0]->toDebugString(); - } - - $leadingItems = array_slice($this->items, 0, -1); - $trailingItem = array_slice($this->items, -1)[0]; - - return join(', ', array_map( - static fn (TokenType $tokenType) => $tokenType->toDebugString(), - $leadingItems - )) . ' or ' . $trailingItem->toDebugString(); - } -} diff --git a/src/Parser/Tokenizer/Tokenizer.php b/src/Parser/Tokenizer/Tokenizer.php deleted file mode 100644 index c670e207..00000000 --- a/src/Parser/Tokenizer/Tokenizer.php +++ /dev/null @@ -1,484 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Fragment; -use PackageFactory\ComponentEngine\Parser\Source\Source; - -/** - * @implements \IteratorAggregate - */ -final class Tokenizer implements \IteratorAggregate -{ - private function __construct(private readonly Source $source) - { - } - - public static function fromSource(Source $source): Tokenizer - { - return new Tokenizer(source: $source); - } - - /** - * @return \Iterator - */ - public function getIterator(): \Iterator - { - $fragments = $this->source->getIterator(); - while ($fragments->valid()) { - yield from self::block($fragments); - } - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - private static function block(\Iterator $fragments): \Iterator - { - if (!$fragments->valid()) { - return; - } - - $bracket = TokenType::tryBracketOpenFromFragment($fragments->current()); - $buffer = Buffer::empty(); - - if ($bracket) { - yield from $buffer->append($fragments->current())->flush($bracket); - $fragments->next(); - } - - while ($fragments->valid()) { - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - - if ($bracket) { - $closingBracket = $bracket->closingBracket(); - - if ($closingBracket->matchesString($fragment->value)) { - yield from self::flushRemainder($buffer); - yield from $buffer->append($fragments->current())->flush($closingBracket); - $fragments->next(); - return; - } - } - - $delegate = match (CharacterType::get($fragment->value)) { - CharacterType::COMMENT_DELIMITER => self::comment($fragments), - CharacterType::STRING_DELIMITER => self::string($fragments), - CharacterType::TEMPLATE_LITERAL_DELIMITER => self::templateLiteral($fragments), - CharacterType::BRACKET_OPEN => self::block($fragments), - CharacterType::ANGLE_OPEN => self::angle($fragments), - CharacterType::PERIOD => self::period($fragments), - CharacterType::ANGLE_CLOSE, - CharacterType::FORWARD_SLASH, - CharacterType::SYMBOL => self::symbol($fragments), - CharacterType::SPACE => self::space($fragments), - default => null - }; - - if ($delegate) { - yield from self::flushRemainder($buffer); - yield from $delegate; - } else { - $buffer->append($fragment); - $fragments->next(); - } - } - - yield from self::flushRemainder($buffer); - } - - /** - * @param Buffer $buffer - * @return \Iterator - */ - private static function flushRemainder(Buffer $buffer): \Iterator - { - yield from $buffer->flush(TokenType::fromBuffer($buffer)); - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - private static function string(\Iterator $fragments): \Iterator - { - $delimiter = $fragments->current(); - $fragments->next(); - - $buffer = Buffer::empty(); - - while ($fragments->valid()) { - switch ($fragments->current()->value) { - case $delimiter->value: - if ($buffer->isEmpty()) { - yield Token::emptyFromDelimitingFragments( - TokenType::STRING_QUOTED, - $delimiter, - $fragments->current() - ); - } else { - yield from $buffer->flush(TokenType::STRING_QUOTED); - } - $fragments->next(); - return; - - case '\\': - $buffer->append($fragments->current()); - $fragments->next(); - - if (!$fragments->valid()) { - throw new \Exception("@TODO: Unexpected end of input"); - } - - $buffer->append($fragments->current()); - $fragments->next(); - break; - - default: - $buffer->append($fragments->current()); - $fragments->next(); - break; - } - } - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - public static function templateLiteral(\Iterator $fragments): \Iterator - { - $buffer = Buffer::empty(); - $buffer->append($fragments->current()); - - yield from $buffer->flush(TokenType::TEMPLATE_LITERAL_START); - - $fragments->next(); - - while ($fragments->valid()) { - - switch ($fragments->current()->value) { - case '`': - yield from $buffer->flush(TokenType::STRING_QUOTED); - $buffer->append($fragments->current()); - yield from $buffer->flush(TokenType::TEMPLATE_LITERAL_END); - $fragments->next(); - return; - - case '$': - $dollarSignBuffer = Buffer::empty()->append($fragments->current()); - $fragments->next(); - - if (!$fragments->valid()) { - throw new \Exception("@TODO: Unexpected end of input"); - } - - $nextFragment = $fragments->current(); - - if ($nextFragment->value === '{') { - yield from $buffer->flush(TokenType::STRING_QUOTED); - yield from $dollarSignBuffer->flush(TokenType::DOLLAR); - yield from self::block($fragments); - } - break; - - case '\\': - $buffer->append($fragments->current()); - $fragments->next(); - - if (!$fragments->valid()) { - throw new \Exception("@TODO: Unexpected end of input"); - } - - $buffer->append($fragments->current()); - $fragments->next(); - break; - - default: - $buffer->append($fragments->current()); - $fragments->next(); - break; - } - } - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - public static function period(\Iterator $fragments): \Iterator - { - $buffer = Buffer::empty()->append($fragments->current()); - $fragments->next(); - - yield from $buffer->flush(TokenType::PERIOD); - } - - /** - * @param \Iterator $fragments - */ - public static function symbol(\Iterator $fragments, ?Buffer $buffer = null): \Iterator - { - $buffer = $buffer ?? Buffer::empty(); - $capture = true; - - while ($capture && $fragments->valid()) { - $fragment = $fragments->current(); - - if ($buffer->value() === '!' && $fragment->value === '!') { - // chained `!` must be kept as individual fragments/tokens - break; - } - - $capture = match (CharacterType::get($fragment->value)) { - CharacterType::ANGLE_CLOSE, - CharacterType::FORWARD_SLASH, - CharacterType::PERIOD, - CharacterType::SYMBOL => (bool) $buffer->append($fragment), - default => false - }; - - if ($capture) $fragments->next(); - } - - yield from match ($buffer->value()) { - '&&' => $buffer->flush(TokenType::OPERATOR_BOOLEAN_AND), - '||' => $buffer->flush(TokenType::OPERATOR_BOOLEAN_OR), - '!' => $buffer->flush(TokenType::OPERATOR_BOOLEAN_NOT), - '>' => $buffer->flush(TokenType::COMPARATOR_GREATER_THAN), - '>=' => $buffer->flush(TokenType::COMPARATOR_GREATER_THAN_OR_EQUAL), - '<' => $buffer->flush(TokenType::COMPARATOR_LESS_THAN), - '<=' => $buffer->flush(TokenType::COMPARATOR_LESS_THAN_OR_EQUAL), - '===' => $buffer->flush(TokenType::COMPARATOR_EQUAL), - '!==' => $buffer->flush(TokenType::COMPARATOR_NOT_EQUAL), - '->' => $buffer->flush(TokenType::ARROW_SINGLE), - ':' => $buffer->flush(TokenType::COLON), - '?.' => $buffer->flush(TokenType::OPTCHAIN), - '.' => $buffer->flush(TokenType::PERIOD), - ',' => $buffer->flush(TokenType::COMMA), - '=' => $buffer->flush(TokenType::EQUALS), - '?' => $buffer->flush(TokenType::QUESTIONMARK), - '$' => $buffer->flush(TokenType::DOLLAR), - '|' => $buffer->flush(TokenType::PIPE), - default => self::flushRemainder($buffer) - }; - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - public static function angle(\Iterator $fragments): \Iterator - { - $buffer = Buffer::empty(); - - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - $buffer->append($fragment); - - $fragments->next(); - if ($fragments->valid()) { - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - yield from match (CharacterType::get($fragment->value)) { - CharacterType::SYMBOL => self::symbol($fragments, $buffer), - CharacterType::SPACE => $buffer->flush(TokenType::COMPARATOR_LESS_THAN), - default => self::tag($fragments, $buffer) - }; - } - } - - /** - * @param \Iterator $fragments - * @param null|Buffer $buffer - * @return \Iterator - */ - public static function tag(\Iterator $fragments, ?Buffer $buffer = null): \Iterator - { - $buffer = $buffer ?? Buffer::empty(); - $isClosing = false; - - while ($fragments->valid()) { - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - if ($buffer->value() === '<') { - if ($fragment->value === '/') { - yield from $buffer->append($fragment)->flush(TokenType::TAG_START_CLOSING); - $fragments->next(); - $isClosing = true; - continue; - } else { - yield from $buffer->flush(TokenType::TAG_START_OPENING); - } - } - - switch (true) { - case $fragment->value === '=': - yield from $buffer->flush(TokenType::STRING); - yield from $buffer->append($fragment)->flush(TokenType::EQUALS); - $fragments->next(); - break; - case $fragment->value === '{': - yield from $buffer->flush(TokenType::STRING); - yield from self::block($fragments); - break; - case $fragment->value === '"': - yield from $buffer->flush(TokenType::STRING); - yield from self::string($fragments); - break; - case $fragment->value === '/': - yield from $buffer->flush(TokenType::STRING); - $buffer->append($fragment); - $fragments->next(); - if (!$fragments->valid()) { - throw new \Exception("@TODO: Unexpected end of input"); - } - $nextFragment = $fragments->current(); - if ($nextFragment->value === '>') { - yield from $buffer->append($nextFragment)->flush(TokenType::TAG_SELF_CLOSE); - $fragments->next(); - } else { - throw new \Exception("@TODO: Illegal Character"); - } - - - return; - case $fragment->value === '>': - yield from $buffer->flush(TokenType::STRING); - yield from $buffer->append($fragment)->flush(TokenType::TAG_END); - $fragments->next(); - - if ($isClosing) { - return; - } else { - $buffer = (yield from self::tagContent($fragments)) ?? Buffer::empty(); - } - break; - case ctype_space($fragment->value): - yield from $buffer->flush(TokenType::STRING); - yield from self::space($fragments); - break; - default: - $buffer->append($fragment); - $fragments->next(); - break; - } - } - - yield from $buffer->flush(TokenType::STRING); - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - public static function tagContent(\Iterator $fragments): \Iterator - { - $buffer = Buffer::empty(); - while ($fragments->valid()) { - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - switch (true) { - case $fragment->value === '{': - yield from $buffer->flush(TokenType::STRING); - yield from self::block($fragments); - break; - case $fragment->value === '<': - $fragments->next(); - if (!$fragments->valid()) { - throw new \Exception("@TODO: Unexpected end of input"); - } - if ($fragments->current()->value === '/') { - yield from $buffer->flush(TokenType::STRING); - return Buffer::empty()->append($fragment); - } else if (!ctype_space($fragments->current()->value)) { - yield from self::tag($fragments, Buffer::empty()->append($fragment)); - } else { - $buffer->append($fragment); - } - case ctype_space($fragment->value): - yield from $buffer->flush(TokenType::STRING); - yield from self::space($fragments); - break; - default: - $buffer->append($fragment); - $fragments->next(); - break; - } - } - - yield from $buffer->flush(TokenType::STRING); - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - public static function space(\Iterator $fragments): \Iterator - { - $buffer = Buffer::empty(); - - while ($fragments->valid()) { - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - - if ($fragment->value === PHP_EOL) { - yield from $buffer->flush(TokenType::SPACE); - yield from $buffer->append($fragment)->flush(TokenType::END_OF_LINE); - } else if (ctype_space($fragment->value)) { - $buffer->append($fragment); - } else { - break; - } - - $fragments->next(); - } - - yield from $buffer->flush(TokenType::SPACE); - } - - /** - * @param \Iterator $fragments - * @return \Iterator - */ - public static function comment(\Iterator $fragments): \Iterator - { - $buffer = Buffer::empty(); - - while ($fragments->valid()) { - /** @var Fragment $fragment */ - $fragment = $fragments->current(); - - if ($fragment->value === PHP_EOL) { - break; - } - - $buffer->append($fragment); - $fragments->next(); - } - - yield from $buffer->flush(TokenType::COMMENT); - } -} diff --git a/test/Integration/PhpTranspilerIntegrationTest.php b/test/Integration/PhpTranspilerIntegrationTest.php index a025eb10..5617c733 100644 --- a/test/Integration/PhpTranspilerIntegrationTest.php +++ b/test/Integration/PhpTranspilerIntegrationTest.php @@ -27,7 +27,6 @@ use PackageFactory\ComponentEngine\Module\Loader\ModuleFile\ModuleFileLoader; use PackageFactory\ComponentEngine\Parser\Source\Path; use PackageFactory\ComponentEngine\Parser\Source\Source; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Tokenizer; use PackageFactory\ComponentEngine\Target\Php\Transpiler\Module\ModuleTranspiler; use PackageFactory\ComponentEngine\Test\Unit\Target\Php\Transpiler\Module\ModuleTestStrategy; use PackageFactory\ComponentEngine\TypeSystem\Scope\GlobalScope\GlobalScope; diff --git a/test/Unit/Language/Parser/ParserTestCase.php b/test/Unit/Language/Parser/ParserTestCase.php index 15120259..c473a165 100644 --- a/test/Unit/Language/Parser/ParserTestCase.php +++ b/test/Unit/Language/Parser/ParserTestCase.php @@ -25,25 +25,10 @@ use PackageFactory\ComponentEngine\Language\Parser\ParserException; use PackageFactory\ComponentEngine\Parser\Source\Position; use PackageFactory\ComponentEngine\Parser\Source\Range; -use PackageFactory\ComponentEngine\Parser\Source\Source; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Token; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Tokenizer; use PHPUnit\Framework\TestCase; abstract class ParserTestCase extends TestCase { - /** - * @param string $sourceAsString - * @return \Iterator - */ - protected function createTokenIterator(string $sourceAsString): \Iterator - { - $source = Source::fromString($sourceAsString); - $tokenizer = Tokenizer::fromSource($source); - - return $tokenizer->getIterator(); - } - /** * @param array{int,int} $startAsArray * @param array{int,int} $endAsArray diff --git a/test/Unit/Parser/Tokenizer/Fixtures.php b/test/Unit/Parser/Tokenizer/Fixtures.php deleted file mode 100644 index cfe7a57b..00000000 --- a/test/Unit/Parser/Tokenizer/Fixtures.php +++ /dev/null @@ -1,42 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Test\Unit\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Source; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Token; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Tokenizer; - -final class Fixtures -{ - /** - * @param string $sourceAsString - * @return \Iterator - */ - public static function tokens(string $sourceAsString): \Iterator - { - $source = Source::fromString($sourceAsString); - $tokenizer = Tokenizer::fromSource($source); - - return $tokenizer->getIterator(); - } -} diff --git a/test/Unit/Parser/Tokenizer/TokenTest.php b/test/Unit/Parser/Tokenizer/TokenTest.php deleted file mode 100644 index 44159ce8..00000000 --- a/test/Unit/Parser/Tokenizer/TokenTest.php +++ /dev/null @@ -1,55 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Test\Unit\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Path; -use PackageFactory\ComponentEngine\Parser\Source\Position; -use PackageFactory\ComponentEngine\Parser\Source\Range; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Token; -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType; -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenTypes; -use PHPUnit\Framework\TestCase; - -final class TokenTest extends TestCase -{ - /** - * @test - */ - public function providesDebugString(): void - { - $token = new Token( - type: TokenType::COMMENT, - value: '# This is a comment', - boundaries: Range::from( - new Position(0, 0), - new Position(0, 0) - ), - sourcePath: Path::createMemory() - ); - - $this->assertEquals( - 'COMMENT ("# This is a comment")', - $token->toDebugString() - ); - } -} diff --git a/test/Unit/Parser/Tokenizer/TokenTypesTest.php b/test/Unit/Parser/Tokenizer/TokenTypesTest.php deleted file mode 100644 index 5afe1e4d..00000000 --- a/test/Unit/Parser/Tokenizer/TokenTypesTest.php +++ /dev/null @@ -1,110 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Test\Unit\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType; -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenTypes; -use PHPUnit\Framework\TestCase; - -final class TokenTypesTest extends TestCase -{ - /** - * @test - */ - public function providesDebugStringForSingleItem(): void - { - $tokenTypes = TokenTypes::from(TokenType::COLON); - - $this->assertEquals( - 'COLON (":")', - $tokenTypes->toDebugString() - ); - } - - /** - * @test - */ - public function providesDebugStringForTwoItems(): void - { - $tokenTypes = TokenTypes::from(TokenType::PERIOD, TokenType::COMMA); - - $this->assertEquals( - 'PERIOD (".") or COMMA (",")', - $tokenTypes->toDebugString() - ); - } - - /** - * @test - */ - public function providesDebugStringForThreeOrMoreItems(): void - { - $tokenTypes = TokenTypes::from( - TokenType::PERIOD, - TokenType::COMMA, - TokenType::COLON, - TokenType::DOLLAR - ); - - $this->assertEquals( - 'PERIOD ("."), COMMA (","), COLON (":") or DOLLAR ("$")', - $tokenTypes->toDebugString() - ); - } - - /** - * @test - */ - public function containsReturnsTrueIfCollectionContainsGivenTokenType(): void - { - $tokenTypes = TokenTypes::from( - TokenType::PERIOD, - TokenType::COMMA, - TokenType::COLON, - TokenType::DOLLAR - ); - - $this->assertTrue($tokenTypes->contains(TokenType::PERIOD)); - $this->assertTrue($tokenTypes->contains(TokenType::COMMA)); - $this->assertTrue($tokenTypes->contains(TokenType::COLON)); - $this->assertTrue($tokenTypes->contains(TokenType::DOLLAR)); - } - - /** - * @test - */ - public function containsReturnsFalseIfCollectionDoesNotContainGivenTokenType(): void - { - $tokenTypes = TokenTypes::from( - TokenType::PERIOD, - TokenType::COMMA, - TokenType::COLON, - TokenType::DOLLAR - ); - - $this->assertFalse($tokenTypes->contains(TokenType::SLASH_FORWARD)); - $this->assertFalse($tokenTypes->contains(TokenType::COMMENT)); - $this->assertFalse($tokenTypes->contains(TokenType::STRING)); - $this->assertFalse($tokenTypes->contains(TokenType::EQUALS)); - } -} diff --git a/test/Unit/Parser/Tokenizer/TokenizerTest.php b/test/Unit/Parser/Tokenizer/TokenizerTest.php deleted file mode 100644 index 27ab6601..00000000 --- a/test/Unit/Parser/Tokenizer/TokenizerTest.php +++ /dev/null @@ -1,168 +0,0 @@ -. - */ - -declare(strict_types=1); - -namespace PackageFactory\ComponentEngine\Test\Unit\Parser\Tokenizer; - -use PackageFactory\ComponentEngine\Parser\Source\Source; -use PackageFactory\ComponentEngine\Parser\Tokenizer\Tokenizer; -use PackageFactory\ComponentEngine\Parser\Tokenizer\TokenType; -use PHPUnit\Framework\TestCase; - -final class TokenizerTest extends TestCase -{ - /** - * @test - */ - public function tokenizesEmptySourceToEmptyIterator(): void - { - $source = Source::fromString(''); - $tokenizer = Tokenizer::fromSource($source); - $iterator = $tokenizer->getIterator(); - - $this->assertFalse($iterator->valid()); - } - - /** - * @test - */ - public function tokenizesOpeningTag(): void - { - $source = Source::fromString(''); - $tokenizer = Tokenizer::fromSource($source); - $tokens = \iterator_to_array($tokenizer->getIterator(), false); - - $this->assertEquals(TokenType::TAG_START_OPENING, $tokens[0]->type); - $this->assertEquals(TokenType::STRING, $tokens[1]->type); - $this->assertEquals(TokenType::TAG_END, $tokens[2]->type); - } - - /** - * @test - */ - public function tokenizesClosingTag(): void - { - $source = Source::fromString(''); - $tokenizer = Tokenizer::fromSource($source); - $tokens = \iterator_to_array($tokenizer->getIterator(), false); - - $this->assertEquals(TokenType::TAG_START_CLOSING, $tokens[0]->type); - $this->assertEquals(TokenType::STRING, $tokens[1]->type); - $this->assertEquals(TokenType::TAG_END, $tokens[2]->type); - } - - /** - * @test - */ - public function tokenizesMultipleBracketedStatements(): void - { - $source = Source::fromString('(a ? b : c) ? (d ? e : f) : (g ? h : i)'); - $tokenizer = Tokenizer::fromSource($source); - $tokens = \iterator_to_array($tokenizer->getIterator(), false); - - $this->assertEquals(TokenType::BRACKET_ROUND_OPEN, $tokens[0]->type); - - $this->assertEquals(TokenType::STRING, $tokens[1]->type); - $this->assertEquals('a', $tokens[1]->value); - - $this->assertEquals(TokenType::SPACE, $tokens[2]->type); - - $this->assertEquals(TokenType::QUESTIONMARK, $tokens[3]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[4]->type); - - $this->assertEquals(TokenType::STRING, $tokens[5]->type); - $this->assertEquals('b', $tokens[5]->value); - - $this->assertEquals(TokenType::SPACE, $tokens[6]->type); - - $this->assertEquals(TokenType::COLON, $tokens[7]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[8]->type); - - $this->assertEquals(TokenType::STRING, $tokens[9]->type); - $this->assertEquals('c', $tokens[9]->value); - - $this->assertEquals(TokenType::BRACKET_ROUND_CLOSE, $tokens[10]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[11]->type); - - $this->assertEquals(TokenType::QUESTIONMARK, $tokens[12]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[13]->type); - - $this->assertEquals(TokenType::BRACKET_ROUND_OPEN, $tokens[14]->type); - - $this->assertEquals(TokenType::STRING, $tokens[15]->type); - $this->assertEquals('d', $tokens[15]->value); - - $this->assertEquals(TokenType::SPACE, $tokens[16]->type); - - $this->assertEquals(TokenType::QUESTIONMARK, $tokens[17]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[18]->type); - - $this->assertEquals(TokenType::STRING, $tokens[19]->type); - $this->assertEquals('e', $tokens[19]->value); - - $this->assertEquals(TokenType::SPACE, $tokens[20]->type); - - $this->assertEquals(TokenType::COLON, $tokens[21]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[22]->type); - - $this->assertEquals(TokenType::STRING, $tokens[23]->type); - $this->assertEquals('f', $tokens[23]->value); - - $this->assertEquals(TokenType::BRACKET_ROUND_CLOSE, $tokens[24]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[25]->type); - - $this->assertEquals(TokenType::COLON, $tokens[26]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[27]->type); - - $this->assertEquals(TokenType::BRACKET_ROUND_OPEN, $tokens[28]->type); - - $this->assertEquals(TokenType::STRING, $tokens[29]->type); - $this->assertEquals('g', $tokens[29]->value); - - $this->assertEquals(TokenType::SPACE, $tokens[30]->type); - - $this->assertEquals(TokenType::QUESTIONMARK, $tokens[31]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[32]->type); - - $this->assertEquals(TokenType::STRING, $tokens[33]->type); - $this->assertEquals('h', $tokens[33]->value); - - $this->assertEquals(TokenType::SPACE, $tokens[34]->type); - - $this->assertEquals(TokenType::COLON, $tokens[35]->type); - - $this->assertEquals(TokenType::SPACE, $tokens[36]->type); - - $this->assertEquals(TokenType::STRING, $tokens[37]->type); - $this->assertEquals('i', $tokens[37]->value); - - $this->assertEquals(TokenType::BRACKET_ROUND_CLOSE, $tokens[38]->type); - } -}