From 53923f23fd7c494498114a0133e051ac272b8eac Mon Sep 17 00:00:00 2001 From: Conrad Buck Date: Sat, 2 Jul 2022 11:25:33 -0400 Subject: [PATCH] tokens -> cstTokens Fixes #4 --- lib/commands.js | 4 ++-- lib/descriptors.js | 57 +++++++++++++++++++++++----------------------- lib/index.js | 14 ++++++------ lib/match.js | 16 ++++++------- lib/sources.js | 20 ++++++++-------- lib/utils/cst.js | 2 +- 6 files changed, 57 insertions(+), 56 deletions(-) diff --git a/lib/commands.js b/lib/commands.js index 5c2fdf7..28d22ea 100644 --- a/lib/commands.js +++ b/lib/commands.js @@ -1,8 +1,8 @@ const debug = require('debug'); -const emit = (tokens) => ({ +const emit = (cstTokens) => ({ type: 'emit', - value: tokens, + value: cstTokens, error: debug.enabled('cst-tokens') ? new Error() : undefined, }); const match = (...descriptors) => ({ diff --git a/lib/descriptors.js b/lib/descriptors.js index 4dff1a0..0d86ceb 100644 --- a/lib/descriptors.js +++ b/lib/descriptors.js @@ -23,8 +23,8 @@ const Optional = (descriptor) => { build() { return []; }, - matchTokens(tokens) { - const match = descriptor.matchTokens(tokens); + matchTokens(cstTokens) { + const match = descriptor.matchTokens(cstTokens); return match === null ? [] : match; }, matchChrs(chrs) { @@ -44,12 +44,12 @@ const Text = (matchValue) => (value) => { build(value) { return [{ type: 'Text', value: value || defaultValue }]; }, - matchTokens(tokens) { + matchTokens(cstTokens) { const match = []; - const matchSource = tokens.fork(); + const matchSource = cstTokens.fork(); while (!matchSource.done && matchSource.value.type === 'Text') { match.push(matchSource.value); - matchSource.advance([tokens.value]); + matchSource.advance([cstTokens.value]); } return match.length ? match : null; }, @@ -66,8 +66,8 @@ const CommentStart = (value) => { build() { return [{ type: 'CommentStart', value }]; }, - matchTokens(tokens) { - const token = tokens.value; + matchTokens(cstTokens) { + const token = cstTokens.value; const { type, value: tValue } = token; return type === 'CommentStart' && tValue === value ? [token] : null; }, @@ -84,8 +84,8 @@ const CommentEnd = (value) => { build() { return [{ type: 'CommentEnd', value }]; }, - matchTokens(tokens) { - const token = tokens.value; + matchTokens(cstTokens) { + const token = cstTokens.value; const { type, value: tValue } = token; return type === 'CommentEnd' && tValue === value ? [token] : null; }, @@ -120,12 +120,12 @@ const Comment = (value) => { throw new Error('Comment value was not a valid comment'); } }, - matchTokens(tokens) { + matchTokens(cstTokens) { return ( - match_([lcStart, lcText], tokens) || - match_([lcStart], tokens) || - match_([bcStart, bcText, bcEnd], tokens) || - match_([bcStart, bcEnd], tokens) + match_([lcStart, lcText], cstTokens) || + match_([lcStart], cstTokens) || + match_([bcStart, bcText, bcEnd], cstTokens) || + match_([bcStart, bcEnd], cstTokens) ); }, matchChrs(chrs) { @@ -144,10 +144,10 @@ const Whitespace = (value = ' ') => { build(value) { return [{ type: 'Whitespace', value: value || defaultValue }]; }, - matchTokens(tokens) { + matchTokens(cstTokens) { const match = []; // What should I do with '' whitespace values? - const matchSource = tokens.fork(); + const matchSource = cstTokens.fork(); while (!matchSource.done && matchSource.value.type === 'Whitespace') { match.push(matchSource.value); matchSource.advance([matchSource.value]); @@ -167,8 +167,8 @@ const Punctuator = (value) => ({ build() { return [{ type: 'Punctuator', value }]; }, - matchTokens(tokens) { - const token = tokens.value; + matchTokens(cstTokens) { + const token = cstTokens.value; const { type, value: tValue } = token; return type === 'Punctuator' && tValue === value ? [token] : null; }, @@ -185,8 +185,8 @@ const Keyword = (value) => { build() { return [{ type: 'Keyword', value }]; }, - matchTokens(tokens) { - const token = tokens.value; + matchTokens(cstTokens) { + const token = cstTokens.value; const { type, value: tValue } = token; return type === 'Keyword' && tValue === value ? [token] : null; }, @@ -205,8 +205,8 @@ const Identifier = (value) => { build(value) { return [{ type: 'Identifier', value: value || defaultValue }]; }, - matchTokens(tokens) { - const token = tokens.value; + matchTokens(cstTokens) { + const token = cstTokens.value; const { type, value: tValue } = token; return type === 'Identifier' && tValue === value ? [token] : null; }, @@ -222,8 +222,8 @@ const Reference = (value) => ({ build() { return [{ type: 'Reference', value }]; }, - matchTokens(tokens) { - const token = tokens.value; + matchTokens(cstTokens) { + const token = cstTokens.value; const { type, value: tValue } = token; return type === 'Reference' && tValue === value ? [token] : null; }, @@ -239,8 +239,8 @@ const Separator = () => ({ build() { return ws.build(); }, - matchTokens(tokens) { - const matchSource = tokens.fork(); + matchTokens(cstTokens) { + const matchSource = cstTokens.fork(); const matches = []; let match; while ((match = matchSource.match(ws) || matchSource.match(cmt))) { @@ -295,8 +295,9 @@ const String = (value) => { throw new Error('String value was not a valid string'); } }, - matchTokens(tokens) { - return match_([sQuot, sQuotText, sQuot], tokens) || match_([dQuot, dQuotText, dQuot], tokens); + matchTokens(cstTokens) { + // prettier-ignore + return match_([sQuot, sQuotText, sQuot], cstTokens) || match_([dQuot, dQuotText, dQuot], cstTokens); }, matchChrs(chrs) { return match_([sQuot, sQuotText, sQuot], chrs) || match_([dQuot, dQuotText, dQuot], chrs); diff --git a/lib/index.js b/lib/index.js index 043dd1c..54d6ba5 100644 --- a/lib/index.js +++ b/lib/index.js @@ -3,11 +3,11 @@ const { exec, buildContext } = require('./match.js'); const { sourceFor } = require('./sources.js'); function __updateTokens(matchNode, matchNodes) { - const { tokens, node } = matchNode; + const { cstTokens, node } = matchNode; - node.tokens = tokens; + node.cstTokens = cstTokens; - for (const token of tokens) { + for (const token of cstTokens) { if (token.type === 'Reference') { __updateTokens(matchNodes.get(token), matchNodes); } @@ -23,13 +23,13 @@ function updateTokens(node, options = {}) { } function __updateRanges(matchNode, matchNodes, position = 0) { - const { tokens, node } = matchNode; + const { cstTokens, node } = matchNode; node.range = []; node.range[0] = position; - for (const token of tokens) { + for (const token of cstTokens) { if (token.type === 'Reference') { position = __updateRanges(matchNodes.get(token), matchNodes, position); } else { @@ -51,11 +51,11 @@ function updateRanges(node, options = {}) { } function __print(matchNode, matchNodes) { - const { tokens } = matchNode; + const { cstTokens } = matchNode; let str = ''; - for (const token of tokens) { + for (const token of cstTokens) { if (token.type === 'Reference') { str += __print(matchNodes.get(token), matchNodes); } else { diff --git a/lib/match.js b/lib/match.js index ca7413f..063502f 100644 --- a/lib/match.js +++ b/lib/match.js @@ -8,18 +8,18 @@ const buildGrammar = (node, context) => { }; // Executes token matching. Serves as a coroutine to grammar generators. -// Takes tokens from `source` and puts them in `matchNode.tokens` +// Takes tokens from `source` and puts them in `matchNode.cstTokens` // `node` is an AST node that acts as the "pattern" to match // `grammar` defines which tokens can belong to a given `node` -// `source` could be either flat text or a tokens tree +// `source` could be either flat text or a CST const __exec = (node, source, context) => { const { matchNodes } = context; - const tokens = []; + const cstTokens = []; const matchNode = { type: node.type, node, - tokens, + cstTokens, source: { // This is as much of the source as I dare make public to the grammar type: source.type, @@ -38,7 +38,7 @@ const __exec = (node, source, context) => { const { type, value } = command; if (type === 'emit') { - tokens.push(...value); + cstTokens.push(...value); } else { const descriptors = value; const forkedResolver = resolver.fork(); @@ -113,12 +113,12 @@ const __exec = (node, source, context) => { if (type === 'match') { // Feeds matching tokens to the grammar generator. In the grammar it looks like: - // tokens = yield match(...descriptors); + // cstTokens = yield match(...descriptors); grammar.advance(match_); continue; } else if (type === 'take') { if (match_) { - tokens.push(...match_); + cstTokens.push(...match_); } else { let fallbackSource; try { @@ -147,7 +147,7 @@ const exec = (node, source, context = buildContext()) => { const result = __exec(node, source, context); if (context.separatorMatch) { - result.tokens.push(...context.separatorMatch); + result.cstTokens.push(...context.separatorMatch); context.separatorMatch = null; } diff --git a/lib/sources.js b/lib/sources.js index 31d7b87..6a18928 100644 --- a/lib/sources.js +++ b/lib/sources.js @@ -26,9 +26,9 @@ class TextSource { return descriptor.type === 'Reference' ? descriptor.build() : descriptor.matchChrs(this); } - advance(tokens, matchNodes) { + advance(cstTokens, matchNodes) { let n = 0; - for (const token of tokens) { + for (const token of cstTokens) { if (token.type === 'Reference') { const { end } = matchNodes.get(token).source; const delta = end - this.index; @@ -64,24 +64,24 @@ class TokensSource { } get done() { - return this.index >= this.node.tokens.length; + return this.index >= this.node.cstTokens.length; } get value() { - return this.node.tokens[this.index]; + return this.node.cstTokens[this.index]; } fork(node = this.node) { const reset = node !== this.node; - return isArray(node.tokens) ? new TokensSource(node, reset ? 0 : this.index) : noSource; + return isArray(node.cstTokens) ? new TokensSource(node, reset ? 0 : this.index) : noSource; } match(descriptor) { return descriptor.matchTokens(this); } - advance(tokens) { - this.index += tokens.length; + advance(cstTokens) { + this.index += cstTokens.length; } fallback() { @@ -90,8 +90,8 @@ class TokensSource { *[Symbol.iterator]() { const { index, node } = this; - for (let i = index; i < node.tokens.length; i++) { - yield node.tokens[i]; + for (let i = index; i < node.cstTokens.length; i++) { + yield node.cstTokens[i]; } } } @@ -137,7 +137,7 @@ const sourceFor = (node, options = {}) => { // prettier-ignore return sourceText != null ? new TextSource(sourceText) - : isArray(node.tokens) + : isArray(node.cstTokens) ? new TokensSource(node) : noSource; }; diff --git a/lib/utils/cst.js b/lib/utils/cst.js index 573140a..8466ad3 100644 --- a/lib/utils/cst.js +++ b/lib/utils/cst.js @@ -12,7 +12,7 @@ const tokensEqual = (a, b) => { const nodeTokensEqual = (a, b) => { const resolver = new RefResolver(a); - for (const [aTok, bTok] of zipAll(a.tokens, b.tokens)) { + for (const [aTok, bTok] of zipAll(a.cstTokens, b.cstTokens)) { if (!tokensEqual(aTok, bTok)) { return false; }