diff --git a/example/executionStatus.json b/example/executionStatus.json index 3bd533ea..21ed67c6 100644 --- a/example/executionStatus.json +++ b/example/executionStatus.json @@ -120,7 +120,8 @@ "expr__": "['luke', 'han', 'leia', 'chewbacca', 'Lando'].($forked('/name',$))", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": null + "data__": null, + "variables__": ["forked"] }, { "materialized__": true, @@ -156,7 +157,8 @@ "treeHasExpressions__": false, "parent__": "/homeworldDetails/properties" } - } + }, + "variables__": ["fetch"] }, { "materialized__": true, @@ -177,7 +179,8 @@ "exprRootPath__": null, "expr__": " homeworldDetails!=null?$joined('/homeworlds/-', homeworldDetails.properties.name):null ", "exprTargetJsonPointer__": "", - "compiledExpr__": "--compiled expression--" + "compiledExpr__": "--compiled expression--", + "variables__": ["joined"] }, { "materialized__": true, @@ -263,7 +266,8 @@ "treeHasExpressions__": false, "parent__": "/personDetails/properties" } - } + }, + "variables__": ["fetch", "save"] }, { "materialized__": false, diff --git a/example/restoreSnapshot.json b/example/restoreSnapshot.json index 2aa70a16..4cc554b7 100644 --- a/example/restoreSnapshot.json +++ b/example/restoreSnapshot.json @@ -64,7 +64,8 @@ "expr__": " $setInterval(function(){$set('/count', count+1)}, 10) ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "--interval/timeout--" + "data__": "--interval/timeout--", + "variables__": ["setInterval","set"] }, { "materialized__": true, @@ -86,7 +87,8 @@ "expr__": " count=10?($clearInterval($$.counter);'done'):'not done' ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "not done" + "data__": "not done", + "variables__": ["clearInterval"] } ] } diff --git a/src/DependencyFinder.ts b/src/DependencyFinder.ts index e578055a..a8cf6cce 100644 --- a/src/DependencyFinder.ts +++ b/src/DependencyFinder.ts @@ -45,6 +45,7 @@ export default class DependencyFinder { private readonly currentSteps: StepRecord[][]; //logically, [[a,b,c],[d,e,f]] private nodeStack: GeneratedExprNode[]; private readonly dependencies: string[][]; //during tree walking we collect these dependencies like [["a", "b", "c"], ["foo", "bar"]] which means the dependencies are a.b.c and foo.bar + public readonly variables: Set = new Set(); /** * program can be either a string to be compiled, or an already-compiled AST * @param program @@ -102,6 +103,7 @@ export default class DependencyFinder { const { type, } = node; + this.captureBuiltInFunctionNames(node); this.capturePathExpressions(node); this.captureArrayIndexes(node); this.nodeStack.push(node); @@ -143,6 +145,21 @@ export default class DependencyFinder { return this.dependencies; } + /** + * Function calls like $count(...) are recorded in this.variables so that the MetaInf can have a record + * of what context variables (in this case functions) were accessed by the expression. + * @param node + * @private + */ + private captureBuiltInFunctionNames(node:GeneratedExprNode) { + if(node.type === 'function'){ + const name = node.procedure?.value; + if(name !== undefined){ + this.variables.add(name); + } + } + } + private markScopeWhenFunctionReturnsValue(scopeWeExited:GeneratedExprNode) { if (scopeWeExited?.type === 'function') { //function has returned const currentScope = DependencyFinder.peek(this.nodeStack); @@ -238,6 +255,7 @@ export default class DependencyFinder { if (type === "variable") { //if we are here then the variable must be an ordinary locally named variable since it is neither $$ or $. //variables local to a closure cannot cause/imply a dependency for this expression + this.variables.add(value); if (!this.hasParent("function")) { //the function name is actually a variable, we want to skip such variables //@ts-ignore last(this.currentSteps).push({type, value, "emit": false}); diff --git a/src/MetaInfoProducer.ts b/src/MetaInfoProducer.ts index 44294a83..c7fa6579 100644 --- a/src/MetaInfoProducer.ts +++ b/src/MetaInfoProducer.ts @@ -31,6 +31,7 @@ export interface MetaInfo{ exprTargetJsonPointer__?:JsonPointerStructureArray|JsonPointerString //the pointer to the object that this expression executes on data__?:any isFunction__?:boolean + variables__?:string[] } export type JsonPointerStructureArray = (string|number)[]; diff --git a/src/TemplateProcessor.ts b/src/TemplateProcessor.ts index a2181818..fa569108 100644 --- a/src/TemplateProcessor.ts +++ b/src/TemplateProcessor.ts @@ -84,7 +84,8 @@ export type Transaction ={ * a FunctionGenerator is used to generate functions that need the context of which expression they were called from * which is made available to them in the MetaInf */ -export type FunctionGenerator = (metaInfo: MetaInfo, templateProcessor: TemplateProcessor) => (Promise<(arg: any) => Promise>); +export type FunctionGenerator = (context: T, templateProcessor?: TemplateProcessor) => Promise<(...args: any[]) => Promise> | ((...args: any[]) => any); + @@ -310,10 +311,11 @@ export default class TemplateProcessor { * it generates are asynchronous functions (ie they return a promise). * $import is an example of this kind of behavior. * When $import('http://mytemplate.com/foo.json') is called, the import function - * is actually genrated on the fly, using knowledge of the json path that it was + * is actually generated on the fly, using knowledge of the json path that it was * called at, to replace the content of the template at that path with the downloaded * content.*/ - functionGenerators: Map; + functionGenerators: Map> = new Map(); + planStepFunctionGenerators: Map> = new Map(); /** for every json pointer, we have multiple callbacks that are stored in a Set * @private @@ -388,7 +390,7 @@ export default class TemplateProcessor { this.options = options; this.isInitializing = false; this.changeCallbacks = new Map(); - this.functionGenerators = new Map(); + //this.functionGenerators = new Map(); this.tagSet = new Set(); this.onInitialize = new Map(); this.executionStatus = new ExecutionStatus(this); @@ -431,6 +433,7 @@ export default class TemplateProcessor { } } + this.setupFunctionGenerators(); } @@ -636,7 +639,7 @@ export default class TemplateProcessor { public static NOOP = Symbol('NOOP'); - private getImport(metaInfo: MetaInfo):(templateToImport:string)=>Promise { //we provide the JSON Pointer that targets where the imported content will go + private getImport = (metaInfo: MetaInfo):(templateToImport:string)=>Promise => { //we provide the JSON Pointer that targets where the imported content will go //import the template to the location pointed to by jsonPtr return async (importMe) => { let resp; @@ -789,6 +792,7 @@ export default class TemplateProcessor { metaInfo.compiledExpr__ = depFinder.compiledExpression; //we have to filter out "" from the dependencies as these are akin to 'no-op' path steps metaInfo.dependencies__ = depFinder.findDependencies().map(depArray => depArray.filter(pathPart => pathPart !== "")); + metaInfo.variables__ = Array.from(depFinder.variables); acc.push(metaInfo); } catch (e) { this.logger.error(JSON.stringify(e)); @@ -1191,7 +1195,7 @@ export default class TemplateProcessor { if (plan.op === "snapshot") { (plan as SnapshotPlan).generatedSnapshot = this.executionStatus.toJsonString();; } else if(plan.op === "transaction" ){ - this.applyTransaction(plan as Transaction); + this.applyTransaction(plan as Transaction); //should this await? }else{ await this.executePlan(plan as Plan); } @@ -1592,42 +1596,12 @@ export default class TemplateProcessor { const {jsonPtr, output} = planStep; let evaluated: AsyncGenerator | any; const metaInfo = jp.get(this.templateMeta, jsonPtr) as MetaInfo; - const {compiledExpr__, exprTargetJsonPointer__, expr__} = metaInfo; + const {compiledExpr__, exprTargetJsonPointer__, expr__, variables__=[]} = metaInfo; let target; try { + const context = {...this.context}; + await this.populateContextWithGeneratedFunctions(context, variables__, metaInfo, planStep); target = jp.get(output, exprTargetJsonPointer__ as JsonPointerString); //an expression is always relative to a target - const safe = this.withErrorHandling.bind(this); - const jittedFunctions: { [key: string]: (arg: any) => Promise } = {}; - for (const k of this.functionGenerators.keys()) { - const generator: FunctionGenerator | undefined = this.functionGenerators.get(k); - if (generator) { // Check if generator is not undefined - try { - jittedFunctions[k] = await safe(await generator(metaInfo, this)); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : "Unknown error"; - const msg = `Function generator '${k}' failed to generate a function and erred with:"${errorMessage}"`; - this.logger.error(msg); - throw new Error(msg); - } - } else { - // Optionally handle the case where generator is undefined - const msg = `Function generator for key '${k}' is undefined.`; - this.logger.error(msg); - throw new Error(msg); - } - } - - const context ={...this.context, - ...{"errorReport": this.generateErrorReportFunction(metaInfo)}, - ...{"defer": safe(this.generateDeferFunction(metaInfo))}, - ...{"import": safe(this.getImport(metaInfo))}, - ...{"forked": safe(this.generateForked(planStep))}, - ...{"joined": safe(this.generateJoined(planStep))}, - ...{"set": safe(this.generateSet(planStep))}, - ...{"setInterval": safe(this.timerManager.generateSetInterval(planStep))}, - ...{"clearInterval": safe(this.timerManager.generateClearInterval(planStep))}, - ...jittedFunctions - }; evaluated = await compiledExpr__?.evaluate( target, context @@ -1655,6 +1629,52 @@ export default class TemplateProcessor { } + private setupFunctionGenerators(){ + this.functionGenerators.set("errorReport", this.generateErrorReportFunction); + this.functionGenerators.set("defer", this.generateDeferFunction); + this.functionGenerators.set("import", this.getImport); + this.planStepFunctionGenerators.set("forked", this.generateForked); + this.planStepFunctionGenerators.set("joined", this.generateJoined); + this.planStepFunctionGenerators.set("set", this.generateSet); + this.planStepFunctionGenerators.set("setInterval", this.timerManager.generateSetInterval); + this.planStepFunctionGenerators.set("clearInterval", this.timerManager.generateClearInterval); + } + + /** + * Certain functions callable in a JSONata expression must be dynamically generated. They cannot be static + * generated because the function instance needs to hold a reference to some kind of runtime state, either + * a MetaInfo or a PlanStep (see FunctionGenerator type). This method, for a given list of function names, + * generates the function by finding and calling the corresponding FunctionGenerator. + * @param context + * @param functionNames + * @param metaInf + * @param planStep + * @private + */ + private async populateContextWithGeneratedFunctions( context:any, functionNames:string[], metaInf:MetaInfo, planStep:PlanStep):Promise{ + const safe = this.withErrorHandling.bind(this); + for (const name of functionNames) { + try { + let generator:any = this.functionGenerators.get(name); + if (generator) { + const generated:any = await generator(metaInf, this); + context[name] = safe(generated); + } else { + generator = this.planStepFunctionGenerators.get(name); + if (generator) { + const generated = await generator(planStep); + context[name] = safe(generated); + } + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "Unknown error"; + const msg = `Function generator '${name}' failed to generate a function and erred with:"${errorMessage}"`; + this.logger.error(msg); + throw new Error(msg); + } + } + } + private allTagsPresent(tagSetOnTheExpression:Set) { if(tagSetOnTheExpression.size === 0 && this.tagSet.size > 0){ return false; @@ -1948,7 +1968,7 @@ export default class TemplateProcessor { return wrappedFunction; } - private async generateErrorReportFunction(metaInfo: MetaInfo){ + private generateErrorReportFunction = async (metaInfo: MetaInfo) => { return async (message:string, name?:string, stack?:any):Promise=>{ const error:StatedError = { error: { @@ -1974,19 +1994,18 @@ export default class TemplateProcessor { - private generateDeferFunction(metaInfo: any) { - const deferFunc = (jsonPointer:JsonPointerString, timeoutMs:number)=>{ + private generateDeferFunction = (metaInfo: MetaInfo) => { + return (jsonPointer: JsonPointerString, timeoutMs: number) => { - if(jp.has(this.output, jsonPointer)){ - const dataChangeCallback = debounce(async (data)=>{ - this.setData(metaInfo.jsonPointer__, data, "forceSetInternal"); //sets the value into the location in the template where the $defer() call is made + if (jp.has(this.output, jsonPointer)) { + const dataChangeCallback = debounce(async (data) => { + this.setData(metaInfo.jsonPointer__ as JsonPointerString, data, "forceSetInternal"); //sets the value into the location in the template where the $defer() call is made }, timeoutMs); this.setDataChangeCallback(jsonPointer, dataChangeCallback); return jp.get(this.output, jsonPointer); //returns the current value of the location $defer is called on } throw new Error(`$defer called on non-existant field: ${jsonPointer}`); - } - return deferFunc; + }; } /** @@ -2047,7 +2066,7 @@ export default class TemplateProcessor { * @private * @param planStep */ - public generateForked(planStep: PlanStep) { + public generateForked = (planStep: PlanStep) => { return async (jsonPtr:JsonPointerString, data:any, op:Op='set')=>{ const {output=this.output, forkStack, forkId} = planStep; //defaulting output to this.output is important for when this call is used by ExecutionStatus to restore const mvccSnapshot = TemplateProcessor.deepCopy(output); //every call to $forked creates a new planStep with its own output copy @@ -2076,7 +2095,7 @@ export default class TemplateProcessor { * @param planStep * @private */ - private generateSet(planStep: PlanStep) { + private generateSet = (planStep: PlanStep) => { const isInsideFork = planStep.forkStack.length > 0; if(!isInsideFork){ return this.setData @@ -2092,7 +2111,7 @@ export default class TemplateProcessor { * @param planStep * @private */ - private generateJoined(planStep: PlanStep) { + private generateJoined = (planStep: PlanStep) => { return async (jsonPtr:JsonPointerString, data:any, op:Op='set')=>{ const {output, forkId} = planStep.forkStack.pop() || {output:this.output, forkId:"ROOT"}; if(forkId === "ROOT"){ diff --git a/src/TimerManager.ts b/src/TimerManager.ts index 17067f14..fb67029e 100644 --- a/src/TimerManager.ts +++ b/src/TimerManager.ts @@ -46,7 +46,7 @@ class TimerManager { return timeout; } - public generateSetInterval(planStep:PlanStep) { + public generateSetInterval = (planStep:PlanStep) => { return (callback: (...args: any[]) => void, delay: number, ...args: any[]): Interval => { // TODO: wrap the callback to track last run time, run counter, and other stats const interval: Interval = setInterval(callback, delay, ...args); @@ -56,7 +56,7 @@ class TimerManager { } } - public generateClearInterval(planStep:PlanStep) { + public generateClearInterval = (planStep:PlanStep) => { return async (interval: Interval): Promise => { this.clearInterval(interval); const jsonPointerStr: string = this.jsonPointerByInterval.get(interval) as string; diff --git a/src/test/DependencyFinder.test.js b/src/test/DependencyFinder.test.js index 44bec58c..e8071eda 100644 --- a/src/test/DependencyFinder.test.js +++ b/src/test/DependencyFinder.test.js @@ -45,6 +45,7 @@ test('$$.aaa', () => { test('$merge($.a.b, $i)', () => { const df = new DependencyFinder('$merge($.a.b, $i)'); expect(df.findDependencies()).toEqual([["", 'a', 'b']]); + expect(Array.from(df.variables)).toEqual(["merge", "i"]) }); test(`'$reduce(function($acc, $i){(x.y.z)})`, () => { @@ -52,6 +53,7 @@ test(`'$reduce(function($acc, $i){(x.y.z)})`, () => { ' x.y.z\n' + ' )})'); expect(df.findDependencies()).toEqual([["x", "y", "z"]]); + expect(Array.from(df.variables)).toEqual(["reduce", "acc", "i"]) }); test('reduce 2', () => { const df = new DependencyFinder('$reduce(function($acc, $i){(\n' + @@ -59,6 +61,7 @@ test('reduce 2', () => { ' x.y.z\n' + ' )})'); expect(df.findDependencies()).toEqual([["", 'a', 'b'], ["x", "y", "z"]]); + expect(Array.from(df.variables)).toEqual(["reduce", "acc", "i", "merge"]) }); test("transform - pattern should be ignored", () => { const program = `k.z~>|$|{'foo':nozzle~>|bingus|{"dingus":klunk}|, 'zap':$$.aaaa}|` @@ -245,6 +248,19 @@ test("complex program 1", () => { "z" ] ]); + expect(Array.from(df.variables)).toEqual([ + "gorp", + "dink", + "loop", + "map", + "i", + "a", + "b", + "gimp", + "reduce", + "acc", + "merge" + ]) }); test("subtract", () => { @@ -399,6 +415,7 @@ test("products.$sum(quantity * price)", () => { "products" ] ]); + expect(Array.from(df.variables)).toEqual(["sum"]); }); test("$sum(quantity * price)", () => { @@ -412,6 +429,7 @@ test("$sum(quantity * price)", () => { "price" ] ]); + expect(Array.from(df.variables)).toEqual(["sum"]); }); test("count.{'cloud.provider': $$.providerName}", () => { @@ -540,6 +558,7 @@ test("chained function", () => { "handleRes" ] ]); + expect(Array.from(df.variables)).toEqual(["urlArray", "fetch", "join"]); }); test("function/procedure name should chain to path dependency", () => { @@ -676,6 +695,7 @@ test("matrix3", () => { "chars" ] ]); + expect(Array.from(df.variables)).toEqual(["set", "count"]); }); test("matrix4", () => { @@ -725,6 +745,18 @@ test("big test expression", () => { "compare" ] ]); + expect(Array.from(df.variables)).toEqual([ + "actual", + "expected", + "path", + "type", + "count", + "map", + "item", + "i", + "reduce", + "append" + ]); }); test("rando", () => { @@ -794,6 +826,22 @@ test("acc^($.val)", () => { ]); }); +test("viz ~> |props|{'x':'../../../../${$$.replacementProp}'}| ~> $import", () => { + const program = "viz ~> |props|{'x':'../../../../${$$.replacementProp}'}| ~> $import"; + const df = new DependencyFinder(program); + const deps = df.findDependencies(); + expect(Array.from(df.variables)).toEqual(["import"]); +}); + +test("$setInterval(counter, 1000)", () => { + const program = "$setInterval(counter, 1000)"; + const df = new DependencyFinder(program); + const deps = df.findDependencies(); + expect(Array.from(df.variables)).toEqual(["setInterval"]); +}); + + + diff --git a/src/test/TemplateProcessor.test.js b/src/test/TemplateProcessor.test.js index 6b550486..f7bbaa92 100644 --- a/src/test/TemplateProcessor.test.js +++ b/src/test/TemplateProcessor.test.js @@ -2289,7 +2289,8 @@ test("interval snapshot", async () => { "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", "isFunction__": true, - "data__": "{function:}" + "data__": "{function:}", + "variables__": ["set"] }, { "materialized__": true, @@ -2311,7 +2312,8 @@ test("interval snapshot", async () => { "expr__": " $setInterval(counter, 1000)", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "--interval/timeout--" + "data__": "--interval/timeout--", + "variables__": ["setInterval"] }, { "materialized__": true, @@ -2333,7 +2335,8 @@ test("interval snapshot", async () => { "expr__": " count>=2?($clearInterval($$.rapidCaller);'done'):'not done' ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "not done" + "data__": "not done", + "variables__": ["clearInterval"] } ] }, @@ -2489,7 +2492,8 @@ test("snapshot and restore", async () => { "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", "isFunction__": true, - "data__": "{function:}" + "data__": "{function:}", + "variables__": ["set"] }, { "materialized__": true, @@ -2511,7 +2515,8 @@ test("snapshot and restore", async () => { "expr__": " $setInterval(counter, 100)", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "--interval/timeout--" + "data__": "--interval/timeout--", + "variables__": ["setInterval"] }, { "materialized__": true, @@ -2533,7 +2538,8 @@ test("snapshot and restore", async () => { "expr__": " count>=10?($clearInterval($$.rapidCaller);'done'):'not done' ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "not done" + "data__": "not done", + "variables__": ["clearInterval"] } ] }, @@ -3197,7 +3203,7 @@ test("test generate function result", async () => { } }); -test("test generate verbose function result", async () => { +test("test_generate_verbose_function_result", async () => { const o = { "a":"${$generate(function(){10}, {'valueOnly':false})}", "b": "${a}" diff --git a/src/utils/GeneratorManager.ts b/src/utils/GeneratorManager.ts index 1b36d87c..7d46e001 100644 --- a/src/utils/GeneratorManager.ts +++ b/src/utils/GeneratorManager.ts @@ -55,7 +55,7 @@ export class GeneratorManager{ else if(typeof input === 'function'){ g = async function*(){ if(interval < 0){ //no interval so call function once - return (input as ()=>any)();//return not yield, for done:true + return await (input as ()=>any)();//return not yield, for done:true }else{ //an interval is specified so we sit in a loop calling the function let count = 0; while(maxYield < 0 || count++ < maxYield-1){