tokens -> cstTokens

Fixes #4
bablr-lang · Jul 2, 2022 · 53923f2 · 53923f2
1 parent 3e196bf
commit 53923f2
Show file tree

Hide file tree

Showing 6 changed files with 57 additions and 56 deletions.
diff --git a/lib/commands.js b/lib/commands.js
@@ -1,8 +1,8 @@
 const debug = require('debug');
 
-const emit = (tokens) => ({
+const emit = (cstTokens) => ({
   type: 'emit',
-  value: tokens,
+  value: cstTokens,
   error: debug.enabled('cst-tokens') ? new Error() : undefined,
 });
 const match = (...descriptors) => ({

diff --git a/lib/descriptors.js b/lib/descriptors.js
@@ -23,8 +23,8 @@ const Optional = (descriptor) => {
     build() {
       return [];
     },
-    matchTokens(tokens) {
-      const match = descriptor.matchTokens(tokens);
+    matchTokens(cstTokens) {
+      const match = descriptor.matchTokens(cstTokens);
       return match === null ? [] : match;
     },
     matchChrs(chrs) {
@@ -44,12 +44,12 @@ const Text = (matchValue) => (value) => {
     build(value) {
       return [{ type: 'Text', value: value || defaultValue }];
     },
-    matchTokens(tokens) {
+    matchTokens(cstTokens) {
       const match = [];
-      const matchSource = tokens.fork();
+      const matchSource = cstTokens.fork();
       while (!matchSource.done && matchSource.value.type === 'Text') {
         match.push(matchSource.value);
-        matchSource.advance([tokens.value]);
+        matchSource.advance([cstTokens.value]);
       }
       return match.length ? match : null;
     },
@@ -66,8 +66,8 @@ const CommentStart = (value) => {
     build() {
       return [{ type: 'CommentStart', value }];
     },
-    matchTokens(tokens) {
-      const token = tokens.value;
+    matchTokens(cstTokens) {
+      const token = cstTokens.value;
       const { type, value: tValue } = token;
       return type === 'CommentStart' && tValue === value ? [token] : null;
     },
@@ -84,8 +84,8 @@ const CommentEnd = (value) => {
     build() {
       return [{ type: 'CommentEnd', value }];
     },
-    matchTokens(tokens) {
-      const token = tokens.value;
+    matchTokens(cstTokens) {
+      const token = cstTokens.value;
       const { type, value: tValue } = token;
       return type === 'CommentEnd' && tValue === value ? [token] : null;
     },
@@ -120,12 +120,12 @@ const Comment = (value) => {
         throw new Error('Comment value was not a valid comment');
       }
     },
-    matchTokens(tokens) {
+    matchTokens(cstTokens) {
       return (
-        match_([lcStart, lcText], tokens) ||
-        match_([lcStart], tokens) ||
-        match_([bcStart, bcText, bcEnd], tokens) ||
-        match_([bcStart, bcEnd], tokens)
+        match_([lcStart, lcText], cstTokens) ||
+        match_([lcStart], cstTokens) ||
+        match_([bcStart, bcText, bcEnd], cstTokens) ||
+        match_([bcStart, bcEnd], cstTokens)
       );
     },
     matchChrs(chrs) {
@@ -144,10 +144,10 @@ const Whitespace = (value = ' ') => {
     build(value) {
       return [{ type: 'Whitespace', value: value || defaultValue }];
     },
-    matchTokens(tokens) {
+    matchTokens(cstTokens) {
       const match = [];
       // What should I do with '' whitespace values?
-      const matchSource = tokens.fork();
+      const matchSource = cstTokens.fork();
       while (!matchSource.done && matchSource.value.type === 'Whitespace') {
         match.push(matchSource.value);
         matchSource.advance([matchSource.value]);
@@ -167,8 +167,8 @@ const Punctuator = (value) => ({
   build() {
     return [{ type: 'Punctuator', value }];
   },
-  matchTokens(tokens) {
-    const token = tokens.value;
+  matchTokens(cstTokens) {
+    const token = cstTokens.value;
     const { type, value: tValue } = token;
     return type === 'Punctuator' && tValue === value ? [token] : null;
   },
@@ -185,8 +185,8 @@ const Keyword = (value) => {
     build() {
       return [{ type: 'Keyword', value }];
     },
-    matchTokens(tokens) {
-      const token = tokens.value;
+    matchTokens(cstTokens) {
+      const token = cstTokens.value;
       const { type, value: tValue } = token;
       return type === 'Keyword' && tValue === value ? [token] : null;
     },
@@ -205,8 +205,8 @@ const Identifier = (value) => {
     build(value) {
       return [{ type: 'Identifier', value: value || defaultValue }];
     },
-    matchTokens(tokens) {
-      const token = tokens.value;
+    matchTokens(cstTokens) {
+      const token = cstTokens.value;
       const { type, value: tValue } = token;
       return type === 'Identifier' && tValue === value ? [token] : null;
     },
@@ -222,8 +222,8 @@ const Reference = (value) => ({
   build() {
     return [{ type: 'Reference', value }];
   },
-  matchTokens(tokens) {
-    const token = tokens.value;
+  matchTokens(cstTokens) {
+    const token = cstTokens.value;
     const { type, value: tValue } = token;
     return type === 'Reference' && tValue === value ? [token] : null;
   },
@@ -239,8 +239,8 @@ const Separator = () => ({
   build() {
     return ws.build();
   },
-  matchTokens(tokens) {
-    const matchSource = tokens.fork();
+  matchTokens(cstTokens) {
+    const matchSource = cstTokens.fork();
     const matches = [];
     let match;
     while ((match = matchSource.match(ws) || matchSource.match(cmt))) {
@@ -295,8 +295,9 @@ const String = (value) => {
         throw new Error('String value was not a valid string');
       }
     },
-    matchTokens(tokens) {
-      return match_([sQuot, sQuotText, sQuot], tokens) || match_([dQuot, dQuotText, dQuot], tokens);
+    matchTokens(cstTokens) {
+      // prettier-ignore
+      return match_([sQuot, sQuotText, sQuot], cstTokens) || match_([dQuot, dQuotText, dQuot], cstTokens);
     },
     matchChrs(chrs) {
       return match_([sQuot, sQuotText, sQuot], chrs) || match_([dQuot, dQuotText, dQuot], chrs);

diff --git a/lib/index.js b/lib/index.js
@@ -3,11 +3,11 @@ const { exec, buildContext } = require('./match.js');
 const { sourceFor } = require('./sources.js');
 
 function __updateTokens(matchNode, matchNodes) {
-  const { tokens, node } = matchNode;
+  const { cstTokens, node } = matchNode;
 
-  node.tokens = tokens;
+  node.cstTokens = cstTokens;
 
-  for (const token of tokens) {
+  for (const token of cstTokens) {
     if (token.type === 'Reference') {
       __updateTokens(matchNodes.get(token), matchNodes);
     }
@@ -23,13 +23,13 @@ function updateTokens(node, options = {}) {
 }
 
 function __updateRanges(matchNode, matchNodes, position = 0) {
-  const { tokens, node } = matchNode;
+  const { cstTokens, node } = matchNode;
 
   node.range = [];
 
   node.range[0] = position;
 
-  for (const token of tokens) {
+  for (const token of cstTokens) {
     if (token.type === 'Reference') {
       position = __updateRanges(matchNodes.get(token), matchNodes, position);
     } else {
@@ -51,11 +51,11 @@ function updateRanges(node, options = {}) {
 }
 
 function __print(matchNode, matchNodes) {
-  const { tokens } = matchNode;
+  const { cstTokens } = matchNode;
 
   let str = '';
 
-  for (const token of tokens) {
+  for (const token of cstTokens) {
     if (token.type === 'Reference') {
       str += __print(matchNodes.get(token), matchNodes);
     } else {

diff --git a/lib/match.js b/lib/match.js
@@ -8,18 +8,18 @@ const buildGrammar = (node, context) => {
 };
 
 // Executes token matching. Serves as a coroutine to grammar generators.
-// Takes tokens from `source` and puts them in `matchNode.tokens`
+// Takes tokens from `source` and puts them in `matchNode.cstTokens`
 // `node` is an AST node that acts as the "pattern" to match
 // `grammar` defines which tokens can belong to a given `node`
-// `source` could be either flat text or a tokens tree
+// `source` could be either flat text or a CST
 const __exec = (node, source, context) => {
   const { matchNodes } = context;
-  const tokens = [];
+  const cstTokens = [];
 
   const matchNode = {
     type: node.type,
     node,
-    tokens,
+    cstTokens,
     source: {
       // This is as much of the source as I dare make public to the grammar
       type: source.type,
@@ -38,7 +38,7 @@ const __exec = (node, source, context) => {
     const { type, value } = command;
 
     if (type === 'emit') {
-      tokens.push(...value);
+      cstTokens.push(...value);
     } else {
       const descriptors = value;
       const forkedResolver = resolver.fork();
@@ -113,12 +113,12 @@ const __exec = (node, source, context) => {
 
       if (type === 'match') {
         // Feeds matching tokens to the grammar generator. In the grammar it looks like:
-        // tokens = yield match(...descriptors);
+        // cstTokens = yield match(...descriptors);
         grammar.advance(match_);
         continue;
       } else if (type === 'take') {
         if (match_) {
-          tokens.push(...match_);
+          cstTokens.push(...match_);
         } else {
           let fallbackSource;
           try {
@@ -147,7 +147,7 @@ const exec = (node, source, context = buildContext()) => {
   const result = __exec(node, source, context);
 
   if (context.separatorMatch) {
-    result.tokens.push(...context.separatorMatch);
+    result.cstTokens.push(...context.separatorMatch);
     context.separatorMatch = null;
   }
 

diff --git a/lib/sources.js b/lib/sources.js
@@ -26,9 +26,9 @@ class TextSource {
     return descriptor.type === 'Reference' ? descriptor.build() : descriptor.matchChrs(this);
   }
 
-  advance(tokens, matchNodes) {
+  advance(cstTokens, matchNodes) {
     let n = 0;
-    for (const token of tokens) {
+    for (const token of cstTokens) {
       if (token.type === 'Reference') {
         const { end } = matchNodes.get(token).source;
         const delta = end - this.index;
@@ -64,24 +64,24 @@ class TokensSource {
   }
 
   get done() {
-    return this.index >= this.node.tokens.length;
+    return this.index >= this.node.cstTokens.length;
   }
 
   get value() {
-    return this.node.tokens[this.index];
+    return this.node.cstTokens[this.index];
   }
 
   fork(node = this.node) {
     const reset = node !== this.node;
-    return isArray(node.tokens) ? new TokensSource(node, reset ? 0 : this.index) : noSource;
+    return isArray(node.cstTokens) ? new TokensSource(node, reset ? 0 : this.index) : noSource;
   }
 
   match(descriptor) {
     return descriptor.matchTokens(this);
   }
 
-  advance(tokens) {
-    this.index += tokens.length;
+  advance(cstTokens) {
+    this.index += cstTokens.length;
   }
 
   fallback() {
@@ -90,8 +90,8 @@ class TokensSource {
 
   *[Symbol.iterator]() {
     const { index, node } = this;
-    for (let i = index; i < node.tokens.length; i++) {
-      yield node.tokens[i];
+    for (let i = index; i < node.cstTokens.length; i++) {
+      yield node.cstTokens[i];
     }
   }
 }
@@ -137,7 +137,7 @@ const sourceFor = (node, options = {}) => {
   // prettier-ignore
   return sourceText != null
     ? new TextSource(sourceText)
-    : isArray(node.tokens)
+    : isArray(node.cstTokens)
       ? new TokensSource(node)
       : noSource;
 };

diff --git a/lib/utils/cst.js b/lib/utils/cst.js
@@ -12,7 +12,7 @@ const tokensEqual = (a, b) => {
 const nodeTokensEqual = (a, b) => {
   const resolver = new RefResolver(a);
 
-  for (const [aTok, bTok] of zipAll(a.tokens, b.tokens)) {
+  for (const [aTok, bTok] of zipAll(a.cstTokens, b.cstTokens)) {
     if (!tokensEqual(aTok, bTok)) {
       return false;
     }