From 4e6f172aa4801a414326f0580f91a48a3f1ba519 Mon Sep 17 00:00:00 2001 From: Matt Vague Date: Tue, 14 Jun 2022 09:18:00 -0700 Subject: [PATCH 1/2] Add source prop to nodes --- src/expression/node/AccessorNode.js | 4 +- src/expression/node/ArrayNode.js | 4 +- src/expression/node/AssignmentNode.js | 4 +- src/expression/node/BlockNode.js | 5 ++- src/expression/node/ConditionalNode.js | 4 +- src/expression/node/ConstantNode.js | 4 +- src/expression/node/FunctionAssignmentNode.js | 4 +- src/expression/node/FunctionNode.js | 4 +- src/expression/node/IndexNode.js | 4 +- src/expression/node/Node.js | 2 + src/expression/node/ObjectNode.js | 5 ++- src/expression/node/OperatorNode.js | 4 +- src/expression/node/ParenthesisNode.js | 4 +- src/expression/node/RangeNode.js | 4 +- src/expression/node/RelationalNode.js | 4 +- src/expression/node/SymbolNode.js | 4 +- types/index.d.ts | 43 ++++++++++++------- 17 files changed, 76 insertions(+), 31 deletions(-) diff --git a/src/expression/node/AccessorNode.js b/src/expression/node/AccessorNode.js index 6adce6c0b4..6b406fb966 100644 --- a/src/expression/node/AccessorNode.js +++ b/src/expression/node/AccessorNode.js @@ -30,8 +30,9 @@ export const createAccessorNode = /* #__PURE__ */ factory(name, dependencies, ({ * @param {Node} object The object from which to retrieve * a property or subset. * @param {IndexNode} index IndexNode containing ranges + * @param {SourceLocation | undefined} source Node source location */ - function AccessorNode (object, index) { + function AccessorNode (object, index, source) { if (!(this instanceof AccessorNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -45,6 +46,7 @@ export const createAccessorNode = /* #__PURE__ */ factory(name, dependencies, ({ this.object = object || null this.index = index + this.source = source // readonly property name Object.defineProperty(this, 'name', { diff --git a/src/expression/node/ArrayNode.js b/src/expression/node/ArrayNode.js index 25879c1530..42b2ed06aa 100644 --- a/src/expression/node/ArrayNode.js +++ b/src/expression/node/ArrayNode.js @@ -13,13 +13,15 @@ export const createArrayNode = /* #__PURE__ */ factory(name, dependencies, ({ No * @extends {Node} * Holds an 1-dimensional array with items * @param {Node[]} [items] 1 dimensional array with items + * @param {SourceLocation | undefined} source Node source location */ - function ArrayNode (items) { + function ArrayNode (items, source) { if (!(this instanceof ArrayNode)) { throw new SyntaxError('Constructor must be called with the new operator') } this.items = items || [] + this.source = source // validate input if (!Array.isArray(this.items) || !this.items.every(isNode)) { diff --git a/src/expression/node/AssignmentNode.js b/src/expression/node/AssignmentNode.js index 8572be6b4c..3ee00ff56b 100644 --- a/src/expression/node/AssignmentNode.js +++ b/src/expression/node/AssignmentNode.js @@ -41,8 +41,9 @@ export const createAssignmentNode = /* #__PURE__ */ factory(name, dependencies, * the property is assigned to the * global scope. * @param {Node} value The value to be assigned + * @param {SourceLocation | undefined} source Node source location */ - function AssignmentNode (object, index, value) { + function AssignmentNode (object, index, value, source) { if (!(this instanceof AssignmentNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -50,6 +51,7 @@ export const createAssignmentNode = /* #__PURE__ */ factory(name, dependencies, this.object = object this.index = value ? index : null this.value = value || index + this.source = source // validate input if (!isSymbolNode(object) && !isAccessorNode(object)) { diff --git a/src/expression/node/BlockNode.js b/src/expression/node/BlockNode.js index a55691e185..b3ff1593bb 100644 --- a/src/expression/node/BlockNode.js +++ b/src/expression/node/BlockNode.js @@ -17,8 +17,9 @@ export const createBlockNode = /* #__PURE__ */ factory(name, dependencies, ({ Re * An array with blocks, where a block is constructed as an Object * with properties block, which is a Node, and visible, which is * a boolean. The property visible is optional and is true by default + * @param {SourceLocation | undefined} source Node source location */ - function BlockNode (blocks) { + function BlockNode (blocks, source) { if (!(this instanceof BlockNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -34,6 +35,8 @@ export const createBlockNode = /* #__PURE__ */ factory(name, dependencies, ({ Re return { node, visible } }) + + this.source = source } BlockNode.prototype = new Node() diff --git a/src/expression/node/ConditionalNode.js b/src/expression/node/ConditionalNode.js index 8f0ead9221..b53f6f9e29 100644 --- a/src/expression/node/ConditionalNode.js +++ b/src/expression/node/ConditionalNode.js @@ -14,11 +14,12 @@ export const createConditionalNode = /* #__PURE__ */ factory(name, dependencies, * @param {Node} condition Condition, must result in a boolean * @param {Node} trueExpr Expression evaluated when condition is true * @param {Node} falseExpr Expression evaluated when condition is true + * @param {SourceLocation | undefined} source Node source location * * @constructor ConditionalNode * @extends {Node} */ - function ConditionalNode (condition, trueExpr, falseExpr) { + function ConditionalNode (condition, trueExpr, falseExpr, source) { if (!(this instanceof ConditionalNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -29,6 +30,7 @@ export const createConditionalNode = /* #__PURE__ */ factory(name, dependencies, this.condition = condition this.trueExpr = trueExpr this.falseExpr = falseExpr + this.source = source } ConditionalNode.prototype = new Node() diff --git a/src/expression/node/ConstantNode.js b/src/expression/node/ConstantNode.js index df20ed1219..557c31af83 100644 --- a/src/expression/node/ConstantNode.js +++ b/src/expression/node/ConstantNode.js @@ -18,15 +18,17 @@ export const createConstantNode = /* #__PURE__ */ factory(name, dependencies, ({ * new ConstantNode('hello') * * @param {*} value Value can be any type (number, BigNumber, string, ...) + * @param {SourceLocation | undefined} source Node source location * @constructor ConstantNode * @extends {Node} */ - function ConstantNode (value) { + function ConstantNode (value, source) { if (!(this instanceof ConstantNode)) { throw new SyntaxError('Constructor must be called with the new operator') } this.value = value + this.source = source } ConstantNode.prototype = new Node() diff --git a/src/expression/node/FunctionAssignmentNode.js b/src/expression/node/FunctionAssignmentNode.js index 07b280b9ce..21db25245f 100644 --- a/src/expression/node/FunctionAssignmentNode.js +++ b/src/expression/node/FunctionAssignmentNode.js @@ -25,8 +25,9 @@ export const createFunctionAssignmentNode = /* #__PURE__ */ factory(name, depend * array with objects containing the name * and type of the parameter * @param {Node} expr The function expression + * @param {SourceLocation | undefined} source Node source location */ - function FunctionAssignmentNode (name, params, expr) { + function FunctionAssignmentNode (name, params, expr, source) { if (!(this instanceof FunctionAssignmentNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -45,6 +46,7 @@ export const createFunctionAssignmentNode = /* #__PURE__ */ factory(name, depend return (param && param.type) || 'any' }) this.expr = expr + this.source = source } FunctionAssignmentNode.prototype = new Node() diff --git a/src/expression/node/FunctionNode.js b/src/expression/node/FunctionNode.js index 1383a14ec8..f97daa419e 100644 --- a/src/expression/node/FunctionNode.js +++ b/src/expression/node/FunctionNode.js @@ -21,8 +21,9 @@ export const createFunctionNode = /* #__PURE__ */ factory(name, dependencies, ({ * @param {./Node | string} fn Node resolving with a function on which to invoke * the arguments, typically a SymboNode or AccessorNode * @param {./Node[]} args + * @param {SourceLocation | undefined} source Node source location */ - function FunctionNode (fn, args) { + function FunctionNode (fn, args, source) { if (!(this instanceof FunctionNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -39,6 +40,7 @@ export const createFunctionNode = /* #__PURE__ */ factory(name, dependencies, ({ this.fn = fn this.args = args || [] + this.source = source // readonly property name Object.defineProperty(this, 'name', { diff --git a/src/expression/node/IndexNode.js b/src/expression/node/IndexNode.js index d39525baae..9ec2ce6d2a 100644 --- a/src/expression/node/IndexNode.js +++ b/src/expression/node/IndexNode.js @@ -25,14 +25,16 @@ export const createIndexNode = /* #__PURE__ */ factory(name, dependencies, ({ No * notation like `a.b`, or using bracket * notation like `a["b"]` (default). * Used to stringify an IndexNode. + * @param {SourceLocation | undefined} source Node source location */ - function IndexNode (dimensions, dotNotation) { + function IndexNode (dimensions, dotNotation, source) { if (!(this instanceof IndexNode)) { throw new SyntaxError('Constructor must be called with the new operator') } this.dimensions = dimensions this.dotNotation = dotNotation || false + this.source = source // validate input if (!Array.isArray(dimensions) || !dimensions.every(isNode)) { diff --git a/src/expression/node/Node.js b/src/expression/node/Node.js index 55eb06a62b..b002724ee9 100644 --- a/src/expression/node/Node.js +++ b/src/expression/node/Node.js @@ -33,6 +33,8 @@ export const createNode = /* #__PURE__ */ factory(name, dependencies, ({ mathWit Node.prototype.comment = '' + Node.prototype.source = null + /** * Compile the node into an optimized, evauatable JavaScript function * @return {{evaluate: function([Object])}} object diff --git a/src/expression/node/ObjectNode.js b/src/expression/node/ObjectNode.js index 5d6168252e..04d8eb89b6 100644 --- a/src/expression/node/ObjectNode.js +++ b/src/expression/node/ObjectNode.js @@ -15,8 +15,9 @@ export const createObjectNode = /* #__PURE__ */ factory(name, dependencies, ({ N * @extends {Node} * Holds an object with keys/values * @param {Object.} [properties] object with key/value pairs + * @param {SourceLocation | undefined} source Node source location */ - function ObjectNode (properties) { + function ObjectNode (properties, source) { if (!(this instanceof ObjectNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -31,6 +32,8 @@ export const createObjectNode = /* #__PURE__ */ factory(name, dependencies, ({ N throw new TypeError('Object containing Nodes expected') } } + + this.source = source } ObjectNode.prototype = new Node() diff --git a/src/expression/node/OperatorNode.js b/src/expression/node/OperatorNode.js index 5dc5c75df4..9e27185fed 100644 --- a/src/expression/node/OperatorNode.js +++ b/src/expression/node/OperatorNode.js @@ -22,8 +22,9 @@ export const createOperatorNode = /* #__PURE__ */ factory(name, dependencies, ({ * @param {Node[]} args Operator arguments * @param {boolean} [implicit] Is this an implicit multiplication? * @param {boolean} [isPercentage] Is this an percentage Operation? + * @param {SourceLocation | undefined} source Node source location */ - function OperatorNode (op, fn, args, implicit, isPercentage) { + function OperatorNode (op, fn, args, implicit, isPercentage, source) { if (!(this instanceof OperatorNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -44,6 +45,7 @@ export const createOperatorNode = /* #__PURE__ */ factory(name, dependencies, ({ this.op = op this.fn = fn this.args = args || [] + this.source = source } OperatorNode.prototype = new Node() diff --git a/src/expression/node/ParenthesisNode.js b/src/expression/node/ParenthesisNode.js index c59d4243cd..8ef8a17378 100644 --- a/src/expression/node/ParenthesisNode.js +++ b/src/expression/node/ParenthesisNode.js @@ -12,9 +12,10 @@ export const createParenthesisNode = /* #__PURE__ */ factory(name, dependencies, * @extends {Node} * A parenthesis node describes manual parenthesis from the user input * @param {Node} content + * @param {SourceLocation | undefined} source Node source location * @extends {Node} */ - function ParenthesisNode (content) { + function ParenthesisNode (content, source) { if (!(this instanceof ParenthesisNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -25,6 +26,7 @@ export const createParenthesisNode = /* #__PURE__ */ factory(name, dependencies, } this.content = content + this.source = source } ParenthesisNode.prototype = new Node() diff --git a/src/expression/node/RangeNode.js b/src/expression/node/RangeNode.js index 849f7ba153..c8afd7d544 100644 --- a/src/expression/node/RangeNode.js +++ b/src/expression/node/RangeNode.js @@ -15,8 +15,9 @@ export const createRangeNode = /* #__PURE__ */ factory(name, dependencies, ({ No * @param {Node} start included lower-bound * @param {Node} end included upper-bound * @param {Node} [step] optional step + * @param {SourceLocation | undefined} source Node source location */ - function RangeNode (start, end, step) { + function RangeNode (start, end, step, source) { if (!(this instanceof RangeNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -30,6 +31,7 @@ export const createRangeNode = /* #__PURE__ */ factory(name, dependencies, ({ No this.start = start // included lower-bound this.end = end // included upper-bound this.step = step || null // optional step + this.source = source } RangeNode.prototype = new Node() diff --git a/src/expression/node/RelationalNode.js b/src/expression/node/RelationalNode.js index 80af63bb3c..fe5f838e8e 100644 --- a/src/expression/node/RelationalNode.js +++ b/src/expression/node/RelationalNode.js @@ -15,11 +15,12 @@ export const createRelationalNode = /* #__PURE__ */ factory(name, dependencies, * * @param {String[]} conditionals An array of conditional operators used to compare the parameters * @param {Node[]} params The parameters that will be compared + * @param {SourceLocation | undefined} source Node source location * * @constructor RelationalNode * @extends {Node} */ - function RelationalNode (conditionals, params) { + function RelationalNode (conditionals, params, source) { if (!(this instanceof RelationalNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -29,6 +30,7 @@ export const createRelationalNode = /* #__PURE__ */ factory(name, dependencies, this.conditionals = conditionals this.params = params + this.source = source } RelationalNode.prototype = new Node() diff --git a/src/expression/node/SymbolNode.js b/src/expression/node/SymbolNode.js index 57b4c098f0..2d46762553 100644 --- a/src/expression/node/SymbolNode.js +++ b/src/expression/node/SymbolNode.js @@ -14,6 +14,7 @@ export const createSymbolNode = /* #__PURE__ */ factory(name, dependencies, ({ m /** * Check whether some name is a valueless unit like "inch". * @param {string} name + * @param {SourceLocation | undefined} source Node source location * @return {boolean} */ function isValuelessUnit (name) { @@ -27,7 +28,7 @@ export const createSymbolNode = /* #__PURE__ */ factory(name, dependencies, ({ m * @param {string} name * @extends {Node} */ - function SymbolNode (name) { + function SymbolNode (name, source) { if (!(this instanceof SymbolNode)) { throw new SyntaxError('Constructor must be called with the new operator') } @@ -36,6 +37,7 @@ export const createSymbolNode = /* #__PURE__ */ factory(name, dependencies, ({ m if (typeof name !== 'string') throw new TypeError('String expected for parameter "name"') this.name = name + this.source = source } SymbolNode.prototype = new Node() diff --git a/types/index.d.ts b/types/index.d.ts index 1e4f7510eb..e4759856af 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -169,7 +169,7 @@ declare namespace math { name: string } interface AccessorNodeCtor { - new (object: MathNode, index: IndexNode): AccessorNode + new (object: MathNode, index: IndexNode, source?: SourceLocation): AccessorNode } interface ArrayNode extends MathNodeCommon { @@ -178,7 +178,7 @@ declare namespace math { items: MathNode[] } interface ArrayNodeCtor { - new (items: MathNode[]): ArrayNode + new (items: MathNode[], source?: SourceLocation): ArrayNode } interface AssignmentNode extends MathNodeCommon { @@ -190,7 +190,7 @@ declare namespace math { name: string } interface AssignmentNodeCtor { - new (object: SymbolNode, value: MathNode): AssignmentNode + new (object: SymbolNode, value: MathNode, source?: SourceLocation): AssignmentNode new ( object: SymbolNode | AccessorNode, index: IndexNode, @@ -205,7 +205,8 @@ declare namespace math { } interface BlockNodeCtor { new ( - arr: Array<{ node: MathNode } | { node: MathNode; visible: boolean }> + arr: Array<{ node: MathNode } | { node: MathNode; visible: boolean }>, + source?: SourceLocation ): BlockNode } @@ -220,7 +221,8 @@ declare namespace math { new ( condition: MathNode, trueExpr: MathNode, - falseExpr: MathNode + falseExpr: MathNode, + source?: SourceLocation ): ConditionalNode } @@ -232,7 +234,7 @@ declare namespace math { } interface ConstantNodeCtor { - new (constant: number): ConstantNode + new (constant: number, source?: SourceLocation): ConstantNode } interface FunctionAssignmentNode extends MathNodeCommon { @@ -243,7 +245,7 @@ declare namespace math { expr: MathNode } interface FunctionAssignmentNodeCtor { - new (name: string, params: string[], expr: MathNode): FunctionAssignmentNode + new (name: string, params: string[], expr: MathNode, source?: SourceLocation): FunctionAssignmentNode } interface FunctionNode extends MathNodeCommon { @@ -253,7 +255,7 @@ declare namespace math { args: MathNode[] } interface FunctionNodeCtor { - new (fn: MathNode | string, args: MathNode[]): FunctionNode + new (fn: MathNode | string, args: MathNode[], source?: SourceLocation): FunctionNode } interface IndexNode extends MathNodeCommon { @@ -263,8 +265,8 @@ declare namespace math { dotNotation: boolean } interface IndexNodeCtor { - new (dimensions: MathNode[]): IndexNode - new (dimensions: MathNode[], dotNotation: boolean): IndexNode + new (dimensions: MathNode[], source?: SourceLocation): IndexNode + new (dimensions: MathNode[], dotNotation: boolean, source?: SourceLocation): IndexNode } interface ObjectNode extends MathNodeCommon { @@ -273,7 +275,7 @@ declare namespace math { properties: Record } interface ObjectNodeCtor { - new (properties: Record): ObjectNode + new (properties: Record, source?: SourceLocation): ObjectNode } type OperatorNodeMap = { @@ -334,7 +336,8 @@ declare namespace math { op: TOp, fn: TFn, args: TArgs, - implicit?: boolean + implicit?: boolean, + source?: SourceLocation ): OperatorNode } interface ParenthesisNode @@ -345,7 +348,7 @@ declare namespace math { } interface ParenthesisNodeCtor { new ( - content: TContent + content: TContent, source?: SourceLocation ): ParenthesisNode } @@ -357,7 +360,7 @@ declare namespace math { step: MathNode | null } interface RangeNodeCtor { - new (start: MathNode, end: MathNode, step?: MathNode): RangeNode + new (start: MathNode, end: MathNode, step?: MathNode, source?: SourceLocation): RangeNode } interface RelationalNode extends MathNodeCommon { @@ -367,7 +370,7 @@ declare namespace math { params: MathNode[] } interface RelationalNodeCtor { - new (conditionals: string[], params: MathNode[]): RelationalNode + new (conditionals: string[], params: MathNode[], source?: SourceLocation): RelationalNode } interface SymbolNode extends MathNodeCommon { @@ -376,7 +379,7 @@ declare namespace math { name: string } interface SymbolNodeCtor { - new (name: string): SymbolNode + new (name: string, source?: SourceLocation): SymbolNode } type MathNode = @@ -3806,10 +3809,18 @@ declare namespace math { evaluate(scope?: any): any } + interface SourceLocation { + start: { + start: number + end: number + } + } + interface MathNodeCommon { isNode: true comment: string type: string + source?: SourceLocation isUpdateNode?: boolean From 7377bffb1d3c8179db81414898d073108412acdd Mon Sep 17 00:00:00 2001 From: Matt Vague Date: Tue, 14 Jun 2022 11:03:32 -0700 Subject: [PATCH 2/2] Spike on source locations --- .mocharc.json | 2 +- src/expression/parse.js | 141 +++++++++++++++-------- test/unit-tests/expression/parse.test.js | 91 +++++++++++++++ types/index.d.ts | 62 ++++++---- 4 files changed, 223 insertions(+), 73 deletions(-) diff --git a/.mocharc.json b/.mocharc.json index 8211a6189c..eaa79c4c3c 100644 --- a/.mocharc.json +++ b/.mocharc.json @@ -1,6 +1,6 @@ { "reporter": "dot", "timeout": 60000, - "forbid-only": true, + "forbid-only": false, "recursive": true } diff --git a/src/expression/parse.js b/src/expression/parse.js index 229c39f549..7c58235bb0 100644 --- a/src/expression/parse.js +++ b/src/expression/parse.js @@ -110,7 +110,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ }) } - // token types enumeration + /** token types enumeration + * @type ParserTokenType + */ const TOKENTYPE = { NULL: 0, DELIMITER: 1, @@ -185,6 +187,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ 'Infinity' ] + /** + * Return intitial parser state + * @returns {ParserState} + */ function initialState () { return { extraNodes: {}, // current extra nodes, must be careful not to mutate @@ -210,6 +216,28 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ return state.expression.substr(state.index, length) } + /** + * Convert parser state to source meta for attaching to nodes + * + * @param {Object} state + * @returns {SourceMeta} + * @private + */ + function toSourceMeta(state) { + return { + start: state.index - state.token.length, + end: state.index, + expression: state.token + } + } + function toSourceMetaNew(start, end, expression) { + return { + start, + end, + expression, + } + } + /** * View the current character. Returns '' if end of expression is reached. * @@ -634,10 +662,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } if (blocks.length > 0) { - return new BlockNode(blocks) + return new BlockNode(blocks, toSourceMeta(state)) } else { if (!node) { - node = new ConstantNode(undefined) + node = new ConstantNode(undefined, toSourceMeta(state)) node.comment = state.comment } @@ -664,12 +692,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ name = node.name getTokenSkipNewline(state) value = parseAssignment(state) - return new AssignmentNode(new SymbolNode(name), value) + return new AssignmentNode(new SymbolNode(name), value, toSourceMeta(state)) } else if (isAccessorNode(node)) { // parse a matrix subset assignment like 'A[1,2] = 4' getTokenSkipNewline(state) value = parseAssignment(state) - return new AssignmentNode(node.object, node.index, value) + return new AssignmentNode(node.object, node.index, value, toSourceMeta(state)) } else if (isFunctionNode(node) && isSymbolNode(node.fn)) { // parse function assignment like 'f(x) = x^2' valid = true @@ -687,7 +715,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (valid) { getTokenSkipNewline(state) value = parseAssignment(state) - return new FunctionAssignmentNode(name, args, value) + return new FunctionAssignmentNode(name, args, value, toSourceMeta(state)) } } @@ -727,7 +755,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ const falseExpr = parseAssignment(state) // Note: check for conditional operator again, right associativity - node = new ConditionalNode(condition, trueExpr, falseExpr) + node = new ConditionalNode(condition, trueExpr, falseExpr, toSourceMeta(state)) // restore the previous conditional level state.conditionalLevel = prev @@ -738,6 +766,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ /** * logical or, 'x or y' + * @param {ParserState} state * @return {Node} node * @private */ @@ -746,7 +775,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (state.token === 'or') { // eslint-disable-line no-unmodified-loop-condition getTokenSkipNewline(state) - node = new OperatorNode('or', 'or', [node, parseLogicalXor(state)]) + node = new OperatorNode('or', 'or', [node, parseLogicalXor(state)], false, false, toSourceMeta(state)) } return node @@ -762,7 +791,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (state.token === 'xor') { // eslint-disable-line no-unmodified-loop-condition getTokenSkipNewline(state) - node = new OperatorNode('xor', 'xor', [node, parseLogicalAnd(state)]) + node = new OperatorNode('xor', 'xor', [node, parseLogicalAnd(state)], false, false, toSourceMeta(state)) } return node @@ -778,7 +807,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (state.token === 'and') { // eslint-disable-line no-unmodified-loop-condition getTokenSkipNewline(state) - node = new OperatorNode('and', 'and', [node, parseBitwiseOr(state)]) + node = new OperatorNode('and', 'and', [node, parseBitwiseOr(state)], false, false, toSourceMeta(state)) } return node @@ -794,7 +823,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (state.token === '|') { // eslint-disable-line no-unmodified-loop-condition getTokenSkipNewline(state) - node = new OperatorNode('|', 'bitOr', [node, parseBitwiseXor(state)]) + node = new OperatorNode('|', 'bitOr', [node, parseBitwiseXor(state)], false ,false, toSourceMeta(state)) } return node @@ -810,7 +839,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (state.token === '^|') { // eslint-disable-line no-unmodified-loop-condition getTokenSkipNewline(state) - node = new OperatorNode('^|', 'bitXor', [node, parseBitwiseAnd(state)]) + node = new OperatorNode('^|', 'bitXor', [node, parseBitwiseAnd(state)], false, false, toSourceMeta(state)) } return node @@ -826,7 +855,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ while (state.token === '&') { // eslint-disable-line no-unmodified-loop-condition getTokenSkipNewline(state) - node = new OperatorNode('&', 'bitAnd', [node, parseRelational(state)]) + node = new OperatorNode('&', 'bitAnd', [node, parseRelational(state)], false, false, toSourceMeta(state)) } return node @@ -859,9 +888,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (params.length === 1) { return params[0] } else if (params.length === 2) { - return new OperatorNode(conditionals[0].name, conditionals[0].fn, params) + return new OperatorNode(conditionals[0].name, conditionals[0].fn, params, false, false, toSourceMeta(state)) } else { - return new RelationalNode(conditionals.map(c => c.fn), params) + return new RelationalNode(conditionals.map(c => c.fn), params, toSourceMeta(state)) } } @@ -887,7 +916,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getTokenSkipNewline(state) params = [node, parseConversion(state)] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, toSourceMeta(state)) } return node @@ -916,11 +945,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (name === 'in' && state.token === '') { // end of expression -> this is the unit 'in' ('inch') - node = new OperatorNode('*', 'multiply', [node, new SymbolNode('in')], true) + node = new OperatorNode('*', 'multiply', [node, new SymbolNode('in')], true, false, toSourceMeta(state)) } else { // operator 'a to b' or 'a in b' params = [node, parseRange(state)] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, toSourceMeta(state)) } } @@ -936,9 +965,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ let node const params = [] + console.log(`parseRange`, state) + if (state.token === ':') { // implicit start=1 (one-based) - node = new ConstantNode(1) + node = new ConstantNode(1, toSourceMeta(state)) } else { // explicit start node = parseAddSubtract(state) @@ -954,7 +985,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.token === ')' || state.token === ']' || state.token === ',' || state.token === '') { // implicit end - params.push(new SymbolNode('end')) + params.push(new SymbolNode('end'), toSourceMeta(state)) } else { // explicit end params.push(parseAddSubtract(state)) @@ -963,10 +994,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (params.length === 3) { // params = [start, step, end] - node = new RangeNode(params[0], params[2], params[1]) // start, end, step + node = new RangeNode(params[0], params[2], params[1], toSourceMeta(state)) // start, end, step } else { // length === 2 // params = [start, end] - node = new RangeNode(params[0], params[1]) // start, end + node = new RangeNode(params[0], params[1], toSourceMeta(state)) // start, end } } @@ -979,7 +1010,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ * @private */ function parseAddSubtract (state) { - let node, name, fn, params + let node, name, fn, params, sourceMeta + + console.log(`parseAddSubtract`, state) + + sourceMeta = toSourceMetaNew(state) node = parseMultiplyDivide(state) @@ -987,6 +1022,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ '+': 'add', '-': 'subtract' } + while (hasOwnProperty(operators, state.token)) { name = state.token fn = operators[name] @@ -994,11 +1030,12 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getTokenSkipNewline(state) const rightNode = parseMultiplyDivide(state) if (rightNode.isPercentage) { - params = [node, new OperatorNode('*', 'multiply', [node, rightNode])] + params = [node, new OperatorNode('*', 'multiply', [node, rightNode], false, false, sourceMeta)] } else { params = [node, rightNode] } - node = new OperatorNode(name, fn, params) + + node = new OperatorNode(name, fn, params, false, false, sourceMeta) } return node @@ -1031,7 +1068,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getTokenSkipNewline(state) last = parseImplicitMultiplication(state) - node = new OperatorNode(name, fn, [node, last]) + node = new OperatorNode(name, fn, [node, last], toSourceMeta(state)) } else { break } @@ -1064,7 +1101,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // number: implicit multiplication like '(2+3)2' // parenthesis: implicit multiplication like '2(3+4)', '(3+4)(1+2)' last = parseRule2(state) - node = new OperatorNode('*', 'multiply', [node, last], true /* implicit */) + node = new OperatorNode('*', 'multiply', [node, last], true /* implicit */, toSourceMeta(state)) } else { break } @@ -1105,7 +1142,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ Object.assign(state, tokenStates.pop()) tokenStates.pop() last = parsePercentage(state) - node = new OperatorNode('/', 'divide', [node, last]) + node = new OperatorNode('/', 'divide', [node, last], toSourceMeta(state)) } else { // Not a match, so rewind tokenStates.pop() @@ -1147,10 +1184,10 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (name === '%' && state.tokenType === TOKENTYPE.DELIMITER && state.token !== '(') { // If the expression contains only %, then treat that as /100 - node = new OperatorNode('/', 'divide', [node, new ConstantNode(100)], false, true) + node = new OperatorNode('/', 'divide', [node, new ConstantNode(100)], false, true, toSourceMeta(state)) } else { params = [node, parseUnary(state)] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, toSourceMeta(state)) } } @@ -1178,7 +1215,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getTokenSkipNewline(state) params = [parseUnary(state)] - return new OperatorNode(name, fn, params) + return new OperatorNode(name, fn, params, false, false, toSourceMeta(state)) } return parsePow(state) @@ -1201,7 +1238,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getTokenSkipNewline(state) params = [node, parseUnary(state)] // Go back to unary, we can have '2^-3' - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, toSourceMeta(state)) } return node @@ -1229,7 +1266,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getToken(state) params = [node] - node = new OperatorNode(name, fn, params) + node = new OperatorNode(name, fn, params, false, false, toSourceMeta(state)) node = parseAccessors(state, node) } @@ -1298,7 +1335,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // create a new custom node // noinspection JSValidateTypes - return new CustomNode(params) + return new CustomNode(params, toSourceMeta(state)) } return parseSymbol(state) @@ -1319,11 +1356,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ getToken(state) if (hasOwnProperty(CONSTANTS, name)) { // true, false, null, ... - node = new ConstantNode(CONSTANTS[name]) + node = new ConstantNode(CONSTANTS[name], toSourceMeta(sta)) } else if (NUMERIC_CONSTANTS.indexOf(name) !== -1) { // NaN, Infinity - node = new ConstantNode(numeric(name, 'number')) + node = new ConstantNode(numeric(name, 'number'), toSourceMeta(state)) } else { - node = new SymbolNode(name) + node = new SymbolNode(name, toSourceMeta(state)) } // parse function parameters and matrix index @@ -1377,7 +1414,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ closeParams(state) getToken(state) - node = new FunctionNode(node, params) + node = new FunctionNode(node, params, toSourceMeta(state)) } else { // implicit multiplication like (2+3)(4+5) or sqrt(2)(1+2) // don't parse it here but let it be handled by parseImplicitMultiplication @@ -1405,7 +1442,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ closeParams(state) getToken(state) - node = new AccessorNode(node, new IndexNode(params)) + node = new AccessorNode(node, new IndexNode(params), toSourceMeta(state)) } else { // dot notation like variable.prop getToken(state) @@ -1413,11 +1450,11 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ if (state.tokenType !== TOKENTYPE.SYMBOL) { throw createSyntaxError(state, 'Property name expected after dot') } - params.push(new ConstantNode(state.token)) + params.push(new ConstantNode(state.token), toSourceMeta(state)) getToken(state) const dotNotation = true - node = new AccessorNode(node, new IndexNode(params, dotNotation)) + node = new AccessorNode(node, new IndexNode(params, dotNotation), toSourceMeta(state)) } } @@ -1436,7 +1473,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ str = parseDoubleQuotesStringToken(state) // create constant - node = new ConstantNode(str) + node = new ConstantNode(str, toSourceMeta(state)) // parse index parameters node = parseAccessors(state, node) @@ -1487,7 +1524,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ str = parseSingleQuotesStringToken(state) // create constant - node = new ConstantNode(str) + node = new ConstantNode(str, toSourceMeta(state)) // parse index parameters node = parseAccessors(state, node) @@ -1571,7 +1608,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ } } - array = new ArrayNode(params) + array = new ArrayNode(params, toSourceMeta(state)) } else { // 1 dimensional vector if (state.token !== ']') { @@ -1586,7 +1623,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ // this is an empty matrix "[ ]" closeParams(state) getToken(state) - array = new ArrayNode([]) + array = new ArrayNode([], toSourceMeta(state)) } return parseAccessors(state, array) @@ -1611,7 +1648,9 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ len++ } - return new ArrayNode(params) + console.log('parseRow', state) + + return new ArrayNode(params, toSourceMeta(state)) } /** @@ -1659,7 +1698,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ closeParams(state) getToken(state) - let node = new ObjectNode(properties) + let node = new ObjectNode(properties, toSourceMeta(state)) // parse index parameters node = parseAccessors(state, node) @@ -1676,14 +1715,16 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ * @private */ function parseNumber (state) { - let numberStr + let numberStr, sourceMeta if (state.tokenType === TOKENTYPE.NUMBER) { // this is a number numberStr = state.token + sourceMeta = toSourceMeta(state) + getToken(state) - return new ConstantNode(numeric(numberStr, config.number)) + return new ConstantNode(numeric(numberStr, config.number), sourceMeta) } return parseParentheses(state) @@ -1711,7 +1752,7 @@ export const createParse = /* #__PURE__ */ factory(name, dependencies, ({ closeParams(state) getToken(state) - node = new ParenthesisNode(node) + node = new ParenthesisNode(node, toSourceMeta(state)) node = parseAccessors(state, node) return node } diff --git a/test/unit-tests/expression/parse.test.js b/test/unit-tests/expression/parse.test.js index 2d99efb027..e3189b6bda 100644 --- a/test/unit-tests/expression/parse.test.js +++ b/test/unit-tests/expression/parse.test.js @@ -2171,6 +2171,97 @@ describe('parse', function () { }) }) + describe.only('source meta', function () { + it('should attach source meta info to nodes', () => { + // assert.deepStrictEqual(parse('0').source, { + // start: 0, + // end: 1, + // expression: '0' + // }) + + // assert.deepStrictEqual(parse('12345').source, { + // start: 0, + // end: 5, + // expression: '12345' + // }) + + // assert.deepStrictEqual(parse('"hello"').source, { + // start: 0, + // end: 7, + // expression: '"hello"' + // }) + + assert.deepStrictEqual(parse('1 + 2').source, { + start: 0, + end: 5, + expression: '1 + 2' + }) + + // assert.deepStrictEqual(parse('1 + 2').args.map(a => a.source), [ + // { + // start: 0, + // end: 1, + // expression: "1" + // }, + // { + // start: 4, + // end: 5, + // expression: "2" + // } + // ]) + + // assert.deepStrictEqual(parse('(1 + 2)').source, { + // start: 0, + // end: 6, + // expression: '(1 + 2)' + // }) + + // assert.deepStrictEqual(parse('(1 + 2)').content.source, { + // start: 1, + // end: 5, + // expression: '1 + 2' + // }) + + // assert.deepStrictEqual(parse('(1 + 2)').args.map(a => a.source), [ + // { + // start: 1, + // end: 2, + // expression: "1" + // }, + // { + // start: 5, + // end: 6, + // expression: "2" + // } + // ]) + + // assert.deepStrictEqual(parse('[1, 2 + 3i, 4]').source, { + // start: 0, + // end: 14, + // expression: '[1, 2 + 3i, 4]' + // }) + // assert.deepStrictEqual(parse('[1, 2 + 3i, 4]').items.map(a => a.source), [ + // { + // start: 1, + // end: 7, + // expression: 1 + // }, + // { + // start: 1, + // end: 7, + // expression: '2 + 3i' + // }, + // { + // start: 1, + // end: 7, + // expression: 4 + // } + // ]) + + // assert.strictEqual(parse('1/2a').toString(), '1 / 2 a') + }) + }) + describe('expose test functions', function () { it('should expose isAlpha', function () { assert.ok('should expose isAlpha', typeof math.parse.isAlpha === 'function') diff --git a/types/index.d.ts b/types/index.d.ts index e4759856af..0728e33490 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -28,6 +28,25 @@ declare namespace math { [key: string]: FactoryFunction | FactoryFunctionMap } + enum ParserTokenType { + NULL, + DELIMITER, + NUMBER, + SYMBOL, + UNKNOWN + } + + interface ParserState { + extraNodes: any + expression: string + comment: string + index: 0 + token: string + tokenType: ParserTokenType + nestingLevel: number + conditionalLevel: number | null + } + /** Available options for parse */ interface ParseOptions { /** a set of custom nodes */ @@ -169,7 +188,7 @@ declare namespace math { name: string } interface AccessorNodeCtor { - new (object: MathNode, index: IndexNode, source?: SourceLocation): AccessorNode + new (object: MathNode, index: IndexNode, source?: SourceMeta): AccessorNode } interface ArrayNode extends MathNodeCommon { @@ -178,7 +197,7 @@ declare namespace math { items: MathNode[] } interface ArrayNodeCtor { - new (items: MathNode[], source?: SourceLocation): ArrayNode + new (items: MathNode[], source?: SourceMeta): ArrayNode } interface AssignmentNode extends MathNodeCommon { @@ -190,7 +209,7 @@ declare namespace math { name: string } interface AssignmentNodeCtor { - new (object: SymbolNode, value: MathNode, source?: SourceLocation): AssignmentNode + new (object: SymbolNode, value: MathNode, source?: SourceMeta): AssignmentNode new ( object: SymbolNode | AccessorNode, index: IndexNode, @@ -206,7 +225,7 @@ declare namespace math { interface BlockNodeCtor { new ( arr: Array<{ node: MathNode } | { node: MathNode; visible: boolean }>, - source?: SourceLocation + source?: SourceMeta ): BlockNode } @@ -222,7 +241,7 @@ declare namespace math { condition: MathNode, trueExpr: MathNode, falseExpr: MathNode, - source?: SourceLocation + source?: SourceMeta ): ConditionalNode } @@ -234,7 +253,7 @@ declare namespace math { } interface ConstantNodeCtor { - new (constant: number, source?: SourceLocation): ConstantNode + new (constant: number, source?: SourceMeta): ConstantNode } interface FunctionAssignmentNode extends MathNodeCommon { @@ -245,7 +264,7 @@ declare namespace math { expr: MathNode } interface FunctionAssignmentNodeCtor { - new (name: string, params: string[], expr: MathNode, source?: SourceLocation): FunctionAssignmentNode + new (name: string, params: string[], expr: MathNode, source?: SourceMeta): FunctionAssignmentNode } interface FunctionNode extends MathNodeCommon { @@ -255,7 +274,7 @@ declare namespace math { args: MathNode[] } interface FunctionNodeCtor { - new (fn: MathNode | string, args: MathNode[], source?: SourceLocation): FunctionNode + new (fn: MathNode | string, args: MathNode[], source?: SourceMeta): FunctionNode } interface IndexNode extends MathNodeCommon { @@ -265,8 +284,8 @@ declare namespace math { dotNotation: boolean } interface IndexNodeCtor { - new (dimensions: MathNode[], source?: SourceLocation): IndexNode - new (dimensions: MathNode[], dotNotation: boolean, source?: SourceLocation): IndexNode + new (dimensions: MathNode[], source?: SourceMeta): IndexNode + new (dimensions: MathNode[], dotNotation: boolean, source?: SourceMeta): IndexNode } interface ObjectNode extends MathNodeCommon { @@ -275,7 +294,7 @@ declare namespace math { properties: Record } interface ObjectNodeCtor { - new (properties: Record, source?: SourceLocation): ObjectNode + new (properties: Record, source?: SourceMeta): ObjectNode } type OperatorNodeMap = { @@ -337,7 +356,7 @@ declare namespace math { fn: TFn, args: TArgs, implicit?: boolean, - source?: SourceLocation + source?: SourceMeta ): OperatorNode } interface ParenthesisNode @@ -348,7 +367,7 @@ declare namespace math { } interface ParenthesisNodeCtor { new ( - content: TContent, source?: SourceLocation + content: TContent, source?: SourceMeta ): ParenthesisNode } @@ -360,7 +379,7 @@ declare namespace math { step: MathNode | null } interface RangeNodeCtor { - new (start: MathNode, end: MathNode, step?: MathNode, source?: SourceLocation): RangeNode + new (start: MathNode, end: MathNode, step?: MathNode, source?: SourceMeta): RangeNode } interface RelationalNode extends MathNodeCommon { @@ -370,7 +389,7 @@ declare namespace math { params: MathNode[] } interface RelationalNodeCtor { - new (conditionals: string[], params: MathNode[], source?: SourceLocation): RelationalNode + new (conditionals: string[], params: MathNode[], source?: SourceMeta): RelationalNode } interface SymbolNode extends MathNodeCommon { @@ -379,7 +398,7 @@ declare namespace math { name: string } interface SymbolNodeCtor { - new (name: string, source?: SourceLocation): SymbolNode + new (name: string, source?: SourceMeta): SymbolNode } type MathNode = @@ -3809,18 +3828,17 @@ declare namespace math { evaluate(scope?: any): any } - interface SourceLocation { - start: { - start: number - end: number - } + interface SourceMeta { + start: number + end: number + expression: string } interface MathNodeCommon { isNode: true comment: string type: string - source?: SourceLocation + source?: SourceMeta isUpdateNode?: boolean