From 41c857364d186a09f9ad8ba9e5415aa8ae57c75b Mon Sep 17 00:00:00 2001 From: Geoffrey Hendrey Date: Mon, 4 Nov 2024 12:07:11 -0800 Subject: [PATCH] wip (#89) * wip * bump version 0.2.43 * refactor so that takes options * optimize FunctionGenerator so that only the functions actually used by a given expression are created and injected into the context. --- README.md | 95 +++++++++++++++-- example/executionStatus.json | 12 ++- example/generate.json | 2 +- example/myGenerator.mjs | 3 +- example/myGenerator3.yaml | 8 ++ example/myGenerator4.yaml | 3 + example/myGeneratorVerbose.json | 3 + example/restoreSnapshot.json | 6 +- package.json | 2 +- src/DependencyFinder.ts | 18 ++++ src/MetaInfoProducer.ts | 1 + src/TemplateProcessor.ts | 132 +++++++++++++---------- src/TimerManager.ts | 4 +- src/test/DependencyFinder.test.js | 48 +++++++++ src/test/TemplateProcessor.test.js | 111 ++++++++++++++++++-- src/utils/GeneratorManager.ts | 161 ++++++++++++++++++----------- 16 files changed, 462 insertions(+), 147 deletions(-) create mode 100644 example/myGenerator3.yaml create mode 100644 example/myGenerator4.yaml create mode 100644 example/myGeneratorVerbose.json diff --git a/README.md b/README.md index 25e06cf2..1524e8ec 100644 --- a/README.md +++ b/README.md @@ -1763,14 +1763,35 @@ keep in mind that all the values were sequentially pushed into the `generated` f "generated": 10 } ``` +If you are familiar with generators in JS, you know that generated values take a verbose form in which +a `{value, done}` object is [returned](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncGenerator/next#return_value). +If you wish to use the more verbose JS style of yielded/returned object, you can pass your generator +to `$generate` and disable `valuesOnly`, as follows. Notice how the yielded values now contain the JS style +`{value, done}` +```json +> .init -f example/myGeneratorVerbose.json --xf example/myGenerator.mjs +{ + "generated": "${$myGenerator()~>$generate({'valueOnly':false})}" +} +> .out +{ + "generated": { + "value": 10, + "done": true, + "return": "{function:}" + } +} +``` + A slight variation on the example accumulates every value yielded by the generator: ```json -> .init -f example/myGenerator2.json --xf example/myGenerator.mjs +> .init -f example/myGenerator2.json --xf example/myGenerator.mjs { - "generated": "${$myGenerator()}", - "onGenerated": "${$set('/accumulator/-', $$.generated)}", - "accumulator": [] + "generated": "${$myGenerator()}", + "onGenerated": "${$set('/accumulator/-', $$.generated)}", + "accumulator": [] } + ``` ```json ["data=[1,2,3,4,5,6,7,8,9,10]"] > .init -f example/myGenerator2.json --xf example/myGenerator.mjs --tail "/accumulator until $=[1,2,3,4,5,6,7,8,9,10]" @@ -1789,14 +1810,15 @@ Started tailing... Press Ctrl+C to stop. ] ``` -Or, you can use the built in `$generate` method, which takes an optional delay in ms, and turns the array -into an AsyncGenerator. In the example below the values 1 to 10 are pumped into the `generated` field -with 10 ms temporal separation. +You already saw hoe the built-in `$generate` function can accept a JS AsyncGeneraotr, and options. But $generate +can also be used to convert ordinary arrays or functions into async generators. When provided, the `interval` option, +causes the provided array to yield its elements periodically. When a function is provided, as opposed to an array, the +function is called periodically. ```json > .init -f example/generate.json { "delayMs": 250, - "generated":"${[1..10]~>$generate(delayMs)}" + "generated":"${[1..10]~>$generate({'interval':delayMs})}" } ``` ```json ["data.generated=10"] @@ -1808,6 +1830,63 @@ Started tailing... Press Ctrl+C to stop. } ``` +This `example/myGenerator3.yaml` shows how you can call `return` and stop a generator. +```yaml +generated: ${$generate($random, {'interval':10, 'valueOnly':false})} +onGenerated: | + ${ + $count(accumulator)<3 + ? $set('/accumulator/-', $$.generated.value) + : generated.return() /* shut off the generator when the accumulator has 10 items */ + } +accumulator: [] +``` +```json ["$count(data.accumulator)=3"] +> .init -f example/myGenerator3.yaml --tail "/ until $count(accumulator)=3" +Started tailing... Press Ctrl+C to stop. +{ + "generated": { + "value": 0.23433826655570145, + "done": false, + "return": "{function:}" + }, + "onGenerated": { + "value": null, + "done": true + }, + "accumulator": [ + 0.23433826655570145, + 0.23433826655570145, + 0.23433826655570145 + ] +} + + +``` +The `maxYield` parameter can also be used to stop a generator: +```json ["$count(data.accumulator)=0", "$count(data.accumulator)=5"] +> .init -f example/myGenerator4.yaml +{ + "generated": "${$generate($random, {'interval':10, 'maxYield':5})}", + "onGenerated": "${$set('/accumulator/-', $$.generated)}", + "accumulator": [] +} +> .init -f example/myGenerator4.yaml --tail "/ until $count(accumulator)=5" +{ + "generated": 0.5289126250886866, + "onGenerated": [ + "/accumulator/-" + ], + "accumulator": [ + 0.3260049204634301, + 0.4477190160739559, + 0.9414436597923774, + 0.8593436891141426, + 0.5289126250886866 + ] +} +``` + ### $setTimeout `$setTimeout` is the JavaScript `setTimeout` function. It receives a function and an timeout diff --git a/example/executionStatus.json b/example/executionStatus.json index 3bd533ea..21ed67c6 100644 --- a/example/executionStatus.json +++ b/example/executionStatus.json @@ -120,7 +120,8 @@ "expr__": "['luke', 'han', 'leia', 'chewbacca', 'Lando'].($forked('/name',$))", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": null + "data__": null, + "variables__": ["forked"] }, { "materialized__": true, @@ -156,7 +157,8 @@ "treeHasExpressions__": false, "parent__": "/homeworldDetails/properties" } - } + }, + "variables__": ["fetch"] }, { "materialized__": true, @@ -177,7 +179,8 @@ "exprRootPath__": null, "expr__": " homeworldDetails!=null?$joined('/homeworlds/-', homeworldDetails.properties.name):null ", "exprTargetJsonPointer__": "", - "compiledExpr__": "--compiled expression--" + "compiledExpr__": "--compiled expression--", + "variables__": ["joined"] }, { "materialized__": true, @@ -263,7 +266,8 @@ "treeHasExpressions__": false, "parent__": "/personDetails/properties" } - } + }, + "variables__": ["fetch", "save"] }, { "materialized__": false, diff --git a/example/generate.json b/example/generate.json index 0862c6d0..f93f2dc3 100644 --- a/example/generate.json +++ b/example/generate.json @@ -1,4 +1,4 @@ { "delayMs": 250, - "generated":"${[1..10]~>$generate(delayMs)}" + "generated":"${[1..10]~>$generate({'interval':delayMs})}" } \ No newline at end of file diff --git a/example/myGenerator.mjs b/example/myGenerator.mjs index ab507f2f..b652b36e 100644 --- a/example/myGenerator.mjs +++ b/example/myGenerator.mjs @@ -1,5 +1,6 @@ export async function* myGenerator() { - for (let i = 1; i <= 10; i++) { + for (let i = 1; i < 10; i++) { yield i; } + return 10; // Last value with `done: true` } \ No newline at end of file diff --git a/example/myGenerator3.yaml b/example/myGenerator3.yaml new file mode 100644 index 00000000..6d1b73e3 --- /dev/null +++ b/example/myGenerator3.yaml @@ -0,0 +1,8 @@ +generated: ${$generate($random, {'interval':10, 'valueOnly':false})} +onGenerated: | + ${ + $count(accumulator)<3 + ? $set('/accumulator/-', $$.generated.value) + : generated.return() /* shut off the generator when the accumulator has 10 items */ + } +accumulator: [] diff --git a/example/myGenerator4.yaml b/example/myGenerator4.yaml new file mode 100644 index 00000000..e79ae1c7 --- /dev/null +++ b/example/myGenerator4.yaml @@ -0,0 +1,3 @@ +generated: ${$generate($random, {'interval':10, 'maxYield':5})} +onGenerated: ${$set('/accumulator/-', $$.generated)} +accumulator: [] diff --git a/example/myGeneratorVerbose.json b/example/myGeneratorVerbose.json new file mode 100644 index 00000000..94ac2107 --- /dev/null +++ b/example/myGeneratorVerbose.json @@ -0,0 +1,3 @@ +{ + "generated": "${$myGenerator()~>$generate({'valueOnly':false})}" +} \ No newline at end of file diff --git a/example/restoreSnapshot.json b/example/restoreSnapshot.json index 2aa70a16..4cc554b7 100644 --- a/example/restoreSnapshot.json +++ b/example/restoreSnapshot.json @@ -64,7 +64,8 @@ "expr__": " $setInterval(function(){$set('/count', count+1)}, 10) ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "--interval/timeout--" + "data__": "--interval/timeout--", + "variables__": ["setInterval","set"] }, { "materialized__": true, @@ -86,7 +87,8 @@ "expr__": " count=10?($clearInterval($$.counter);'done'):'not done' ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "not done" + "data__": "not done", + "variables__": ["clearInterval"] } ] } diff --git a/package.json b/package.json index e5f36bf5..9d191642 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stated-js", - "version": "0.1.42", + "version": "0.1.43", "license": "Apache-2.0", "description": "JSONata embedded in JSON", "main": "./dist/src/index.js", diff --git a/src/DependencyFinder.ts b/src/DependencyFinder.ts index e578055a..a8cf6cce 100644 --- a/src/DependencyFinder.ts +++ b/src/DependencyFinder.ts @@ -45,6 +45,7 @@ export default class DependencyFinder { private readonly currentSteps: StepRecord[][]; //logically, [[a,b,c],[d,e,f]] private nodeStack: GeneratedExprNode[]; private readonly dependencies: string[][]; //during tree walking we collect these dependencies like [["a", "b", "c"], ["foo", "bar"]] which means the dependencies are a.b.c and foo.bar + public readonly variables: Set = new Set(); /** * program can be either a string to be compiled, or an already-compiled AST * @param program @@ -102,6 +103,7 @@ export default class DependencyFinder { const { type, } = node; + this.captureBuiltInFunctionNames(node); this.capturePathExpressions(node); this.captureArrayIndexes(node); this.nodeStack.push(node); @@ -143,6 +145,21 @@ export default class DependencyFinder { return this.dependencies; } + /** + * Function calls like $count(...) are recorded in this.variables so that the MetaInf can have a record + * of what context variables (in this case functions) were accessed by the expression. + * @param node + * @private + */ + private captureBuiltInFunctionNames(node:GeneratedExprNode) { + if(node.type === 'function'){ + const name = node.procedure?.value; + if(name !== undefined){ + this.variables.add(name); + } + } + } + private markScopeWhenFunctionReturnsValue(scopeWeExited:GeneratedExprNode) { if (scopeWeExited?.type === 'function') { //function has returned const currentScope = DependencyFinder.peek(this.nodeStack); @@ -238,6 +255,7 @@ export default class DependencyFinder { if (type === "variable") { //if we are here then the variable must be an ordinary locally named variable since it is neither $$ or $. //variables local to a closure cannot cause/imply a dependency for this expression + this.variables.add(value); if (!this.hasParent("function")) { //the function name is actually a variable, we want to skip such variables //@ts-ignore last(this.currentSteps).push({type, value, "emit": false}); diff --git a/src/MetaInfoProducer.ts b/src/MetaInfoProducer.ts index 44294a83..c7fa6579 100644 --- a/src/MetaInfoProducer.ts +++ b/src/MetaInfoProducer.ts @@ -31,6 +31,7 @@ export interface MetaInfo{ exprTargetJsonPointer__?:JsonPointerStructureArray|JsonPointerString //the pointer to the object that this expression executes on data__?:any isFunction__?:boolean + variables__?:string[] } export type JsonPointerStructureArray = (string|number)[]; diff --git a/src/TemplateProcessor.ts b/src/TemplateProcessor.ts index 6e652803..fa569108 100644 --- a/src/TemplateProcessor.ts +++ b/src/TemplateProcessor.ts @@ -84,7 +84,8 @@ export type Transaction ={ * a FunctionGenerator is used to generate functions that need the context of which expression they were called from * which is made available to them in the MetaInf */ -export type FunctionGenerator = (metaInfo: MetaInfo, templateProcessor: TemplateProcessor) => (Promise<(arg: any) => Promise>); +export type FunctionGenerator = (context: T, templateProcessor?: TemplateProcessor) => Promise<(...args: any[]) => Promise> | ((...args: any[]) => any); + @@ -310,10 +311,11 @@ export default class TemplateProcessor { * it generates are asynchronous functions (ie they return a promise). * $import is an example of this kind of behavior. * When $import('http://mytemplate.com/foo.json') is called, the import function - * is actually genrated on the fly, using knowledge of the json path that it was + * is actually generated on the fly, using knowledge of the json path that it was * called at, to replace the content of the template at that path with the downloaded * content.*/ - functionGenerators: Map; + functionGenerators: Map> = new Map(); + planStepFunctionGenerators: Map> = new Map(); /** for every json pointer, we have multiple callbacks that are stored in a Set * @private @@ -388,7 +390,7 @@ export default class TemplateProcessor { this.options = options; this.isInitializing = false; this.changeCallbacks = new Map(); - this.functionGenerators = new Map(); + //this.functionGenerators = new Map(); this.tagSet = new Set(); this.onInitialize = new Map(); this.executionStatus = new ExecutionStatus(this); @@ -431,6 +433,7 @@ export default class TemplateProcessor { } } + this.setupFunctionGenerators(); } @@ -636,7 +639,7 @@ export default class TemplateProcessor { public static NOOP = Symbol('NOOP'); - private getImport(metaInfo: MetaInfo):(templateToImport:string)=>Promise { //we provide the JSON Pointer that targets where the imported content will go + private getImport = (metaInfo: MetaInfo):(templateToImport:string)=>Promise => { //we provide the JSON Pointer that targets where the imported content will go //import the template to the location pointed to by jsonPtr return async (importMe) => { let resp; @@ -789,6 +792,7 @@ export default class TemplateProcessor { metaInfo.compiledExpr__ = depFinder.compiledExpression; //we have to filter out "" from the dependencies as these are akin to 'no-op' path steps metaInfo.dependencies__ = depFinder.findDependencies().map(depArray => depArray.filter(pathPart => pathPart !== "")); + metaInfo.variables__ = Array.from(depFinder.variables); acc.push(metaInfo); } catch (e) { this.logger.error(JSON.stringify(e)); @@ -1191,7 +1195,7 @@ export default class TemplateProcessor { if (plan.op === "snapshot") { (plan as SnapshotPlan).generatedSnapshot = this.executionStatus.toJsonString();; } else if(plan.op === "transaction" ){ - this.applyTransaction(plan as Transaction); + this.applyTransaction(plan as Transaction); //should this await? }else{ await this.executePlan(plan as Plan); } @@ -1592,48 +1596,18 @@ export default class TemplateProcessor { const {jsonPtr, output} = planStep; let evaluated: AsyncGenerator | any; const metaInfo = jp.get(this.templateMeta, jsonPtr) as MetaInfo; - const {compiledExpr__, exprTargetJsonPointer__, expr__} = metaInfo; + const {compiledExpr__, exprTargetJsonPointer__, expr__, variables__=[]} = metaInfo; let target; try { + const context = {...this.context}; + await this.populateContextWithGeneratedFunctions(context, variables__, metaInfo, planStep); target = jp.get(output, exprTargetJsonPointer__ as JsonPointerString); //an expression is always relative to a target - const safe = this.withErrorHandling.bind(this); - const jittedFunctions: { [key: string]: (arg: any) => Promise } = {}; - for (const k of this.functionGenerators.keys()) { - const generator: FunctionGenerator | undefined = this.functionGenerators.get(k); - if (generator) { // Check if generator is not undefined - try { - jittedFunctions[k] = await safe(await generator(metaInfo, this)); - } catch (error) { - const errorMessage = error instanceof Error ? error.message : "Unknown error"; - const msg = `Function generator '${k}' failed to generate a function and erred with:"${errorMessage}"`; - this.logger.error(msg); - throw new Error(msg); - } - } else { - // Optionally handle the case where generator is undefined - const msg = `Function generator for key '${k}' is undefined.`; - this.logger.error(msg); - throw new Error(msg); - } - } - - const context ={...this.context, - ...{"errorReport": this.generateErrorReportFunction(metaInfo)}, - ...{"defer": safe(this.generateDeferFunction(metaInfo))}, - ...{"import": safe(this.getImport(metaInfo))}, - ...{"forked": safe(this.generateForked(planStep))}, - ...{"joined": safe(this.generateJoined(planStep))}, - ...{"set": safe(this.generateSet(planStep))}, - ...{"setInterval": safe(this.timerManager.generateSetInterval(planStep))}, - ...{"clearInterval": safe(this.timerManager.generateClearInterval(planStep))}, - ...jittedFunctions - }; evaluated = await compiledExpr__?.evaluate( target, context ); if (evaluated?._jsonata_lambda) { - evaluated = this.wrapInOrdinaryFunction(evaluated); + evaluated = TemplateProcessor.wrapInOrdinaryFunction(evaluated); metaInfo.isFunction__ = true; } } catch (error: any) { @@ -1647,14 +1621,60 @@ export default class TemplateProcessor { _error.name = "JSONata evaluation exception"; throw _error; } - if (GeneratorManager.isGenerator(evaluated)) { - //returns the first item, and begins pumping remaining items into execution queue - evaluated = this.generatorManager.pumpItems(evaluated as AsyncGenerator, metaInfo, this); + if (GeneratorManager.isAsyncGenerator(evaluated)) { + //awaits and returns the first item. And pumpItems begins pumping remaining items into execution queue asynchronously + evaluated = await this.generatorManager.pumpItems(evaluated as AsyncGenerator, metaInfo, this); } return evaluated } + private setupFunctionGenerators(){ + this.functionGenerators.set("errorReport", this.generateErrorReportFunction); + this.functionGenerators.set("defer", this.generateDeferFunction); + this.functionGenerators.set("import", this.getImport); + this.planStepFunctionGenerators.set("forked", this.generateForked); + this.planStepFunctionGenerators.set("joined", this.generateJoined); + this.planStepFunctionGenerators.set("set", this.generateSet); + this.planStepFunctionGenerators.set("setInterval", this.timerManager.generateSetInterval); + this.planStepFunctionGenerators.set("clearInterval", this.timerManager.generateClearInterval); + } + + /** + * Certain functions callable in a JSONata expression must be dynamically generated. They cannot be static + * generated because the function instance needs to hold a reference to some kind of runtime state, either + * a MetaInfo or a PlanStep (see FunctionGenerator type). This method, for a given list of function names, + * generates the function by finding and calling the corresponding FunctionGenerator. + * @param context + * @param functionNames + * @param metaInf + * @param planStep + * @private + */ + private async populateContextWithGeneratedFunctions( context:any, functionNames:string[], metaInf:MetaInfo, planStep:PlanStep):Promise{ + const safe = this.withErrorHandling.bind(this); + for (const name of functionNames) { + try { + let generator:any = this.functionGenerators.get(name); + if (generator) { + const generated:any = await generator(metaInf, this); + context[name] = safe(generated); + } else { + generator = this.planStepFunctionGenerators.get(name); + if (generator) { + const generated = await generator(planStep); + context[name] = safe(generated); + } + } + } catch (error) { + const errorMessage = error instanceof Error ? error.message : "Unknown error"; + const msg = `Function generator '${name}' failed to generate a function and erred with:"${errorMessage}"`; + this.logger.error(msg); + throw new Error(msg); + } + } + } + private allTagsPresent(tagSetOnTheExpression:Set) { if(tagSetOnTheExpression.size === 0 && this.tagSet.size > 0){ return false; @@ -1869,7 +1889,6 @@ export default class TemplateProcessor { if (callbacks) { const promises = Array.from(callbacks).map(cbFn => Promise.resolve().then(() => { - //to do ... return here so asyn functions are actually awaited by Promise.all cbFn(data, jsonPointer as JsonPointerString, removed, op); }) //works with cbFn that is either sync or async by wrapping in promise ); @@ -1934,7 +1953,7 @@ export default class TemplateProcessor { } } - private wrapInOrdinaryFunction(jsonataLambda:any) { + public static wrapInOrdinaryFunction(jsonataLambda:any) { const wrappedFunction = (...args:any[])=> { // Call the 'apply' method of jsonataLambda with the captured arguments return jsonataLambda.apply(jsonataLambda, args); @@ -1949,7 +1968,7 @@ export default class TemplateProcessor { return wrappedFunction; } - private async generateErrorReportFunction(metaInfo: MetaInfo){ + private generateErrorReportFunction = async (metaInfo: MetaInfo) => { return async (message:string, name?:string, stack?:any):Promise=>{ const error:StatedError = { error: { @@ -1975,19 +1994,18 @@ export default class TemplateProcessor { - private generateDeferFunction(metaInfo: any) { - const deferFunc = (jsonPointer:JsonPointerString, timeoutMs:number)=>{ + private generateDeferFunction = (metaInfo: MetaInfo) => { + return (jsonPointer: JsonPointerString, timeoutMs: number) => { - if(jp.has(this.output, jsonPointer)){ - const dataChangeCallback = debounce(async (data)=>{ - this.setData(metaInfo.jsonPointer__, data, "forceSetInternal"); //sets the value into the location in the template where the $defer() call is made + if (jp.has(this.output, jsonPointer)) { + const dataChangeCallback = debounce(async (data) => { + this.setData(metaInfo.jsonPointer__ as JsonPointerString, data, "forceSetInternal"); //sets the value into the location in the template where the $defer() call is made }, timeoutMs); this.setDataChangeCallback(jsonPointer, dataChangeCallback); return jp.get(this.output, jsonPointer); //returns the current value of the location $defer is called on } throw new Error(`$defer called on non-existant field: ${jsonPointer}`); - } - return deferFunc; + }; } /** @@ -2048,7 +2066,7 @@ export default class TemplateProcessor { * @private * @param planStep */ - public generateForked(planStep: PlanStep) { + public generateForked = (planStep: PlanStep) => { return async (jsonPtr:JsonPointerString, data:any, op:Op='set')=>{ const {output=this.output, forkStack, forkId} = planStep; //defaulting output to this.output is important for when this call is used by ExecutionStatus to restore const mvccSnapshot = TemplateProcessor.deepCopy(output); //every call to $forked creates a new planStep with its own output copy @@ -2062,7 +2080,7 @@ export default class TemplateProcessor { forkId: TemplateProcessor.simpleUniqueId() }; //do not await setData...$forked runs async - this.setDataForked (mvccSnapshotPlanStep); + void this.setDataForked (mvccSnapshotPlanStep); } } @@ -2077,7 +2095,7 @@ export default class TemplateProcessor { * @param planStep * @private */ - private generateSet(planStep: PlanStep) { + private generateSet = (planStep: PlanStep) => { const isInsideFork = planStep.forkStack.length > 0; if(!isInsideFork){ return this.setData @@ -2093,7 +2111,7 @@ export default class TemplateProcessor { * @param planStep * @private */ - private generateJoined(planStep: PlanStep) { + private generateJoined = (planStep: PlanStep) => { return async (jsonPtr:JsonPointerString, data:any, op:Op='set')=>{ const {output, forkId} = planStep.forkStack.pop() || {output:this.output, forkId:"ROOT"}; if(forkId === "ROOT"){ diff --git a/src/TimerManager.ts b/src/TimerManager.ts index 17067f14..fb67029e 100644 --- a/src/TimerManager.ts +++ b/src/TimerManager.ts @@ -46,7 +46,7 @@ class TimerManager { return timeout; } - public generateSetInterval(planStep:PlanStep) { + public generateSetInterval = (planStep:PlanStep) => { return (callback: (...args: any[]) => void, delay: number, ...args: any[]): Interval => { // TODO: wrap the callback to track last run time, run counter, and other stats const interval: Interval = setInterval(callback, delay, ...args); @@ -56,7 +56,7 @@ class TimerManager { } } - public generateClearInterval(planStep:PlanStep) { + public generateClearInterval = (planStep:PlanStep) => { return async (interval: Interval): Promise => { this.clearInterval(interval); const jsonPointerStr: string = this.jsonPointerByInterval.get(interval) as string; diff --git a/src/test/DependencyFinder.test.js b/src/test/DependencyFinder.test.js index 44bec58c..e8071eda 100644 --- a/src/test/DependencyFinder.test.js +++ b/src/test/DependencyFinder.test.js @@ -45,6 +45,7 @@ test('$$.aaa', () => { test('$merge($.a.b, $i)', () => { const df = new DependencyFinder('$merge($.a.b, $i)'); expect(df.findDependencies()).toEqual([["", 'a', 'b']]); + expect(Array.from(df.variables)).toEqual(["merge", "i"]) }); test(`'$reduce(function($acc, $i){(x.y.z)})`, () => { @@ -52,6 +53,7 @@ test(`'$reduce(function($acc, $i){(x.y.z)})`, () => { ' x.y.z\n' + ' )})'); expect(df.findDependencies()).toEqual([["x", "y", "z"]]); + expect(Array.from(df.variables)).toEqual(["reduce", "acc", "i"]) }); test('reduce 2', () => { const df = new DependencyFinder('$reduce(function($acc, $i){(\n' + @@ -59,6 +61,7 @@ test('reduce 2', () => { ' x.y.z\n' + ' )})'); expect(df.findDependencies()).toEqual([["", 'a', 'b'], ["x", "y", "z"]]); + expect(Array.from(df.variables)).toEqual(["reduce", "acc", "i", "merge"]) }); test("transform - pattern should be ignored", () => { const program = `k.z~>|$|{'foo':nozzle~>|bingus|{"dingus":klunk}|, 'zap':$$.aaaa}|` @@ -245,6 +248,19 @@ test("complex program 1", () => { "z" ] ]); + expect(Array.from(df.variables)).toEqual([ + "gorp", + "dink", + "loop", + "map", + "i", + "a", + "b", + "gimp", + "reduce", + "acc", + "merge" + ]) }); test("subtract", () => { @@ -399,6 +415,7 @@ test("products.$sum(quantity * price)", () => { "products" ] ]); + expect(Array.from(df.variables)).toEqual(["sum"]); }); test("$sum(quantity * price)", () => { @@ -412,6 +429,7 @@ test("$sum(quantity * price)", () => { "price" ] ]); + expect(Array.from(df.variables)).toEqual(["sum"]); }); test("count.{'cloud.provider': $$.providerName}", () => { @@ -540,6 +558,7 @@ test("chained function", () => { "handleRes" ] ]); + expect(Array.from(df.variables)).toEqual(["urlArray", "fetch", "join"]); }); test("function/procedure name should chain to path dependency", () => { @@ -676,6 +695,7 @@ test("matrix3", () => { "chars" ] ]); + expect(Array.from(df.variables)).toEqual(["set", "count"]); }); test("matrix4", () => { @@ -725,6 +745,18 @@ test("big test expression", () => { "compare" ] ]); + expect(Array.from(df.variables)).toEqual([ + "actual", + "expected", + "path", + "type", + "count", + "map", + "item", + "i", + "reduce", + "append" + ]); }); test("rando", () => { @@ -794,6 +826,22 @@ test("acc^($.val)", () => { ]); }); +test("viz ~> |props|{'x':'../../../../${$$.replacementProp}'}| ~> $import", () => { + const program = "viz ~> |props|{'x':'../../../../${$$.replacementProp}'}| ~> $import"; + const df = new DependencyFinder(program); + const deps = df.findDependencies(); + expect(Array.from(df.variables)).toEqual(["import"]); +}); + +test("$setInterval(counter, 1000)", () => { + const program = "$setInterval(counter, 1000)"; + const df = new DependencyFinder(program); + const deps = df.findDependencies(); + expect(Array.from(df.variables)).toEqual(["setInterval"]); +}); + + + diff --git a/src/test/TemplateProcessor.test.js b/src/test/TemplateProcessor.test.js index ad83e777..f7bbaa92 100644 --- a/src/test/TemplateProcessor.test.js +++ b/src/test/TemplateProcessor.test.js @@ -2289,7 +2289,8 @@ test("interval snapshot", async () => { "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", "isFunction__": true, - "data__": "{function:}" + "data__": "{function:}", + "variables__": ["set"] }, { "materialized__": true, @@ -2311,7 +2312,8 @@ test("interval snapshot", async () => { "expr__": " $setInterval(counter, 1000)", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "--interval/timeout--" + "data__": "--interval/timeout--", + "variables__": ["setInterval"] }, { "materialized__": true, @@ -2333,7 +2335,8 @@ test("interval snapshot", async () => { "expr__": " count>=2?($clearInterval($$.rapidCaller);'done'):'not done' ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "not done" + "data__": "not done", + "variables__": ["clearInterval"] } ] }, @@ -2489,7 +2492,8 @@ test("snapshot and restore", async () => { "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", "isFunction__": true, - "data__": "{function:}" + "data__": "{function:}", + "variables__": ["set"] }, { "materialized__": true, @@ -2511,7 +2515,8 @@ test("snapshot and restore", async () => { "expr__": " $setInterval(counter, 100)", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "--interval/timeout--" + "data__": "--interval/timeout--", + "variables__": ["setInterval"] }, { "materialized__": true, @@ -2533,7 +2538,8 @@ test("snapshot and restore", async () => { "expr__": " count>=10?($clearInterval($$.rapidCaller);'done'):'not done' ", "exprTargetJsonPointer__": "", "compiledExpr__": "--compiled expression--", - "data__": "not done" + "data__": "not done", + "variables__": ["clearInterval"] } ] }, @@ -3110,14 +3116,14 @@ test("test close", async () => { } }); -test("test generate", async () => { +test("test generate array", async () => { const o = { - "delayMs": 10, - "a":"${[1..10]~>$generate(delayMs)}", + "options": {"interval":10, "valueOnly":true}, + "a":"${[1..10]~>$generate(options)}", "b": "${a}" }; - const callCount = 10; + const callCount = 0; let resolvePromise; const allCallsMade = new Promise((resolve) => { resolvePromise = resolve; @@ -3141,6 +3147,91 @@ test("test generate", async () => { } }); +test("test generate single item", async () => { + const o = { + "a":"${$generate(10)}", + "b": "${a}" + }; + + const callCount = 0; + let resolvePromise; + const allCallsMade = new Promise((resolve) => { + resolvePromise = resolve; + }); + + const changeHandler = jest.fn((data, ptr) => { + expect(ptr).toBe("/b"); // Ensure correct pointer + resolvePromise(); // Resolve the promise when callCount is reached + }); + const tp = new TemplateProcessor(o); + tp.setDataChangeCallback('/b', changeHandler); + try { + await tp.initialize(); + await allCallsMade; + expect(changeHandler).toHaveBeenCalledTimes(1); + expect(tp.output.b).toBe(10); + } finally { + await tp.close(); + } +}); + +test("test generate function result", async () => { + const o = { + "a":"${$generate(function(){10})}", + "b": "${a}" + }; + + let resolvePromise; + const allCallsMade = new Promise((resolve) => { + resolvePromise = resolve; + }); + + const changeHandler = jest.fn((data, ptr) => { + expect(ptr).toBe("/b"); // Ensure correct pointer + resolvePromise(); // Resolve the promise when callCount is reached + }); + + const tp = new TemplateProcessor(o); + tp.setDataChangeCallback('/b', changeHandler); + try { + await tp.initialize(); + await allCallsMade; + expect(changeHandler).toHaveBeenCalledTimes(1); + expect(tp.output.b).toBe(10); + } finally { + await tp.close(); + } +}); + +test("test_generate_verbose_function_result", async () => { + const o = { + "a":"${$generate(function(){10}, {'valueOnly':false})}", + "b": "${a}" + }; + + let resolvePromise; + const allCallsMade = new Promise((resolve) => { + resolvePromise = resolve; + }); + + const changeHandler = jest.fn((data, ptr) => { + expect(ptr).toBe("/b"); // Ensure correct pointer + resolvePromise(); // Resolve the promise when callCount is reached + }); + + const tp = new TemplateProcessor(o); + tp.setDataChangeCallback('/b', changeHandler); + try { + await tp.initialize(); + await allCallsMade; + expect(changeHandler).toHaveBeenCalledTimes(1); + expect(tp.output.b).toMatchObject({value:10, done:true}); + expect(tp.output.b.return).toBeDefined(); //make sure 'return' function is provided + } finally { + await tp.close(); + } +}); + test("test lifecycle manager", async () => { const o = { diff --git a/src/utils/GeneratorManager.ts b/src/utils/GeneratorManager.ts index 586726e8..7d46e001 100644 --- a/src/utils/GeneratorManager.ts +++ b/src/utils/GeneratorManager.ts @@ -9,48 +9,76 @@ export class GeneratorManager{ } /** - * Generates an asynchronous generator that yields items from the provided array, + * Generates an asynchronous generator that yields items from the provided input, * separated by the specified timeout, and automatically registers the generator. * - * @param array The array of items to yield. - * @param timeout The delay in milliseconds between yields. Defaults to 0. + * @param input The item or array of items to yield. If generate me is a function, the function is called to + * get a result, and recursively passed to generate() + * @param options {valueOnly:boolean, interval?:number}={valueOnly:true, interval:-1} by default the generator will + * yield/return only the value and not the verbose {value, done} object that is yielded from a JS AsyncGenerator. + * The interval parameter is used to temporally space the yielding of array values when input is an array. When input + * is a function .or simple value, the interval is used to repeatedly call the function or yield the value * @returns The registered asynchronous generator. */ - public generate = (array: any[], timeout: number = 0): AsyncGenerator => { + public generate = (input: AsyncGenerator|any[]|any| (() => any), options:{valueOnly:boolean, interval?:number, maxYield?:number}={valueOnly:true, interval:-1, maxYield:-1}): AsyncGenerator => { if (this.templateProcessor.isClosed) { throw new Error("generate() cannot be called on a closed TemplateProcessor"); } + const {interval=-1, valueOnly=true, maxYield=-1} = options; + if(maxYield === 0){ + throw new Error('maxYield must be greater than zero'); + } + if(GeneratorManager.isAsyncGenerator(input)){ //wrapping an existing async generator + input["valueOnly"] = options.valueOnly; //in effect, annotate the generator with the options, so that down the pike we can determine if we should pump just the value, or the entire shebang + return input + } const timerManager = this.templateProcessor.timerManager; - - const generator = async function* () { - for (let i = 0; i < array.length; i++) { - yield array[i]; - if (timeout > 0 && i < array.length - 1) { - await new Promise((resolve) => timerManager.setTimeout(resolve, timeout)); + let g; + //yield array items separated by timeout + if(Array.isArray(input)) { + g = async function* () { + let max = input.length; + if(maxYield > 0){ + max = Math.min(max, maxYield); } - } - }(); - return generator; - }; - - - - /** - * Extracts the first item from a generator, whether it is synchronous or asynchronous. - * Automatically registers the generator. - * - * @param gen The generator (sync or async) to extract the first item from. - * @returns A promise that resolves to the first item or `undefined` if empty. - */ - public async firstItem(gen: AsyncGenerator | Generator): Promise { - if (!GeneratorManager.isGenerator(gen)) { - throw new Error('The provided generator is not an AsynchronousGenerator, it is a `{typeof generator}`}.'); + let i; + for (i=0; i < max-1; i++) { + yield input[i]; + if (interval >= 0) { + await new Promise((resolve) => timerManager.setTimeout(resolve, interval)); + } + } + return input[i]; //last item is returned, not yielded, so don't is true with last item + }(); } - - const asyncGen = gen as AsyncGenerator; - const result = await asyncGen.next(); - return result.done ? undefined : result.value; - } + //call function and return result + else if(typeof input === 'function'){ + g = async function*(){ + if(interval < 0){ //no interval so call function once + return await (input as ()=>any)();//return not yield, for done:true + }else{ //an interval is specified so we sit in a loop calling the function + let count = 0; + while(maxYield < 0 || count++ < maxYield-1){ + yield await (input as ()=>any)(); + await new Promise((resolve) => timerManager.setTimeout(resolve, interval)); + } + return await (input as ()=>any)(); + } + }(); + }else { + //yield individual item + g = async function* () { + if(interval < 0){ //no interval + return input; //return not yield, for done:true + }else{ + //interval is not supported for ordinary value as this will pump a duplicate same value over and over which will be deduped and ignored anyway + throw new Error("$generate cannot be used to repeat the same value on an 'interval' since Stated would simply dedup/ignore the repeated values."); + } + }(); + } + (g as any)["valueOnly"] = valueOnly; + return g; + }; /** * Checks if the provided object is a generator (synchronous or asynchronous). @@ -58,7 +86,7 @@ export class GeneratorManager{ * @param obj The object to check. * @returns `true` if the object is a generator; otherwise `false`. */ - public static isGenerator(obj: any): boolean { + public static isAsyncGenerator(obj: any): boolean { if (obj == null) return false; if (typeof obj.next === 'function' && typeof obj[Symbol.asyncIterator] === 'function') { return true; @@ -66,46 +94,66 @@ export class GeneratorManager{ return false; } + /** - * Pumps the remaining items from the generator into the TemplateProcessor. - * Automatically registers the generator and returns the first item. + * Pumps the remaining items (after the first item) from the generator into the TemplateProcessor. + * Automatically returns the first item. * * @param generator The generator to pump items from. * @param metaInfo The meta info for processing. * @param templateProcessor The TemplateProcessor to set data in. - * @returns The first generated item. + * @returns The first generated item wrapped as {value, done, return} */ public async pumpItems( - generator: AsyncGenerator | Generator, + generator: AsyncGenerator, metaInfo: any, templateProcessor: any ): Promise { - const first = await this.firstItem(generator); // Get the first item from the generator - // Check if the generator is asynchronous - if (this.isAsyncGenerator(generator)) { - // Handle asynchronous generator. Do not await, because we want items to be pumped into the template async - void this.handleAsyncGenerator(generator, metaInfo, templateProcessor); - } else { - // Handle synchronous generator - throw new Error('The provided generator is not an AsynchronousGenerator, it is a `{typeof generator}`}.'); + const {valueOnly=true} = generator as any; + const first = await generator.next(); // Get the first item from the generator + const {done} = first; + if(!done) { + if (GeneratorManager.isAsyncGenerator(generator)) { + // Handle asynchronous generator. Do not await, because we want items to be pumped into the template async. + // Also, fear not, pumpItems can only queue items which will queue the remaining items which won't be + //drained out of the queue until the item returned by this method has been processed + void this.pumpRemaining(generator, metaInfo, templateProcessor); + } else { + // Handle synchronous generator + throw new Error('The provided generator is not an AsynchronousGenerator, it is a `{typeof generator}`}.'); + } } - - return first; + return valueOnly?first.value: {...first, return: TemplateProcessor.wrapInOrdinaryFunction(generator.return.bind(generator))}; } /** - * Handles asynchronous generators, pumping values into the template processor. + * Handles asynchronous generators, pumping remaining values into the template processor. */ - private async handleAsyncGenerator( + private async pumpRemaining( generator: AsyncGenerator, metaInfo: any, templateProcessor: any ): Promise { - for await (const item of generator) { + while (true) { try { + const result = await generator.next(); + const {valueOnly=true} = generator as any; + const { value, done } = result; + + // Create an object that includes value, done, and the return function + const item = valueOnly?value:{ + value, + done, + return: TemplateProcessor.wrapInOrdinaryFunction(generator.return.bind(generator)) + }; + + // Pass the entire item object to setData await templateProcessor.setData(metaInfo.jsonPointer__ as string, item, "forceSetInternal"); - }catch(error:any){ - if(error.message === "Attempt to setData on a closed TemplateProcessor."){ + + // Break the loop if the generator is done + if (done) break; + } catch (error: any) { + if (error.message === "Attempt to setData on a closed TemplateProcessor.") { await generator.return(); break; } @@ -114,13 +162,4 @@ export class GeneratorManager{ } } - - /** - * Determines if a generator is asynchronous. - */ - private isAsyncGenerator(generator: any): generator is AsyncGenerator { - return typeof generator[Symbol.asyncIterator] === 'function'; - } - - }