refactor: move Lexer to lexer-core package

nodejs-tools/nodejs-compiler-tools/formatter-xml/src/format_xml_element_attribute.test.ts

@@ -1,21 +1,22 @@
 import { test, expect } from 'vitest';
+import { Token } from '@vighnesh153/lexer-core';
+import { XmlTokenType } from '@vighnesh153/lexer-xml';
 import { formatXmlElementAttribute } from './format_xml_element_attribute';
-import { Token, TokenType, TokenTypes } from '@vighnesh153/lexer-xml';
 
-function createToken(tokenLiteral: string, tokenType: TokenType): Token {
+function createToken(tokenLiteral: string, tokenType: XmlTokenType): Token<XmlTokenType> {
   return {
     lineNumber: 1,
     columnNumber: 1,
     tokenLiteral,
     tokenType,
-  } satisfies Token;
+  } satisfies Token<XmlTokenType>;
 }
 
 test('should format attribute with single namespace', () => {
   expect(
     formatXmlElementAttribute({
-      namespaces: [createToken('pokemon', TokenTypes.IDENTIFIER)],
-      value: createToken('pikachu', TokenTypes.STRING_LITERAL),
+      namespaces: [createToken('pokemon', XmlTokenType.Identifier)],
+      value: createToken('pikachu', XmlTokenType.StringLiteral),
     })
   ).toMatchInlineSnapshot(`"pokemon="pikachu""`);
 });
@@ -24,11 +25,11 @@ test('should format attribute with multiple namespaces', () => {
   expect(
     formatXmlElementAttribute({
       namespaces: [
-        createToken('pokemon', TokenTypes.IDENTIFIER),
-        createToken('electric', TokenTypes.IDENTIFIER),
-        createToken('yellow', TokenTypes.IDENTIFIER),
+        createToken('pokemon', XmlTokenType.Identifier),
+        createToken('electric', XmlTokenType.Identifier),
+        createToken('yellow', XmlTokenType.Identifier),
       ],
-      value: createToken('pikachu', TokenTypes.STRING_LITERAL),
+      value: createToken('pikachu', XmlTokenType.StringLiteral),
     })
   ).toMatchInlineSnapshot(`"pokemon:electric:yellow="pikachu""`);
 });

nodejs-tools/nodejs-compiler-tools/formatter-xml/src/index.ts

@@ -1,8 +1,8 @@
-import { LexerError, LexerInputReader, StringLexerInput } from '@vighnesh153/lexer-core';
-import { XmlLexer } from '@vighnesh153/lexer-xml';
+import { Lexer, LexerError, LexerInputReader, StringLexerInput } from '@vighnesh153/lexer-core';
 import { ParserError, XmlParser } from '@vighnesh153/parser-xml';
 import { FormattingOptions } from './formatting_options';
 import { formatXmlProgram } from './format_xml_program';
+import { XmlTokenType } from '@vighnesh153/lexer-xml';
 
 export type FormatResponse =
   | {
@@ -26,7 +26,7 @@ export function format(rawXml: string, { indentation = 4, sortAttributes = true
   try {
     const input = new StringLexerInput(rawXml);
     const inputReader = new LexerInputReader(input);
-    const lexer = new XmlLexer(inputReader);
+    const lexer = new Lexer<XmlTokenType>(inputReader);
     const parser = new XmlParser(lexer);
     const program = parser.parseProgram();
 

nodejs-tools/nodejs-compiler-tools/formatter-xml/src/test_utils.ts

@@ -1,11 +1,11 @@
-import { LexerInputReader, StringLexerInput } from '@vighnesh153/lexer-core';
-import { XmlLexer } from '@vighnesh153/lexer-xml';
+import { Lexer, LexerInputReader, StringLexerInput } from '@vighnesh153/lexer-core';
+import { XmlTokenType } from '@vighnesh153/lexer-xml';
 import { XmlParser, XmlProgram } from '@vighnesh153/parser-xml';
 
 export function parseProgram(input: string): [XmlParser, XmlProgram] {
   const stringInput = new StringLexerInput(input);
   const inputReader = new LexerInputReader(stringInput);
-  const lexer = new XmlLexer(inputReader);
+  const lexer = new Lexer<XmlTokenType>(inputReader);
   const parser = new XmlParser(lexer);
 
   return [parser, parser.parseProgram()];

nodejs-tools/nodejs-compiler-tools/lexer-xml/src/Lexer.ts → nodejs-tools/nodejs-compiler-tools/lexer-core/src/Lexer.ts

@@ -1,10 +1,11 @@
-import { LexerError, LexerInputReader } from '@vighnesh153/lexer-core';
+import { LexerError } from './LexerError';
+import { LexerInputReader } from './LexerInputReader';
 import { Token } from './tokens';
 
-export class XmlLexer {
+export class Lexer<TokenType> {
   readonly #errors: Array<LexerError> = [];
 
-  currentToken: Token | null = null;
+  currentToken: Token<TokenType> | null = null;
 
   get errors(): Readonly<Array<LexerError>> {
     return this.#errors.map((error) => error.copy());

nodejs-tools/nodejs-compiler-tools/lexer-core/src/index.ts

@@ -1,5 +1,7 @@
+export * from './Lexer';
 export * from './LexerError';
 export * from './LexerInput';
 export * from './LexerInputReader';
 
+export * from './tokens';
 export * from './utils';

nodejs-tools/nodejs-compiler-tools/lexer-core/src/tokens.test.ts

@@ -0,0 +1,24 @@
+import { test, expect } from 'vitest';
+
+import { cloneToken, Token } from './tokens';
+
+test('should clone token', () => {
+  const token: Token<'dummy_token_type'> = {
+    lineNumber: 43,
+    columnNumber: 25,
+    tokenLiteral: 'dummy token literal',
+    tokenType: 'dummy_token_type',
+  };
+
+  const clonedToken = cloneToken(token);
+
+  // reference equality
+  expect(clonedToken).not.toBe(token);
+  // value equality
+  expect(clonedToken).toStrictEqual({
+    lineNumber: 43,
+    columnNumber: 25,
+    tokenLiteral: 'dummy token literal',
+    tokenType: 'dummy_token_type',
+  });
+});

nodejs-tools/nodejs-compiler-tools/lexer-core/src/tokens.ts

@@ -0,0 +1,10 @@
+export type Token<T> = {
+  tokenType: T;
+  tokenLiteral: string;
+  lineNumber: number;
+  columnNumber: number;
+};
+
+export function cloneToken<T>(token: Token<T>): Token<T> {
+  return { ...token };
+}

nodejs-tools/nodejs-compiler-tools/lexer-xml/src/index.ts

@@ -1,11 +1,2 @@
-export { XmlLexer } from './Lexer';
 export { nextToken } from './nextToken';
-export {
-  type Token,
-  TokenTypes,
-  cloneToken,
-  TokenType,
-  serializeToken,
-  type SerializedToken,
-  type SerializedTokenType,
-} from './tokens';
+export { XmlTokenType } from './tokens';