wip (#89)

* wip

* bump version 0.2.43

* refactor so that  takes options

* optimize FunctionGenerator so that only the functions actually used by a given expression are created and injected into the context.

README.md

@@ -1763,14 +1763,35 @@ keep in mind that all the values were sequentially pushed into the `generated` f
   "generated": 10
 }
 ```
+If you are familiar with generators in JS, you know that generated values take a verbose form in which 
+a `{value, done}` object is [returned](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/AsyncGenerator/next#return_value).
+If you wish to use the more verbose JS style of yielded/returned object, you can pass your generator 
+to `$generate` and disable `valuesOnly`, as follows. Notice how the yielded values now contain the JS style 
+`{value, done}`
+```json
+> .init -f example/myGeneratorVerbose.json --xf example/myGenerator.mjs
+{
+  "generated": "${$myGenerator()~>$generate({'valueOnly':false})}"
+}
+> .out
+{
+  "generated": {
+    "value": 10,
+    "done": true,
+    "return": "{function:}"
+  }
+}
+```
+
 A slight variation on the example accumulates every value yielded by the generator:
 ```json
-> .init -f example/myGenerator2.json --xf example/myGenerator.mjs
+>  .init -f example/myGenerator2.json --xf example/myGenerator.mjs
 {
-   "generated": "${$myGenerator()}",
-   "onGenerated": "${$set('/accumulator/-', $$.generated)}",
-   "accumulator": []
+  "generated": "${$myGenerator()}",
+  "onGenerated": "${$set('/accumulator/-', $$.generated)}",
+  "accumulator": []
 }
+
 ```
 ```json ["data=[1,2,3,4,5,6,7,8,9,10]"]
 > .init -f example/myGenerator2.json --xf example/myGenerator.mjs --tail "/accumulator until $=[1,2,3,4,5,6,7,8,9,10]"
@@ -1789,14 +1810,15 @@ Started tailing... Press Ctrl+C to stop.
 ]
 
 ```
-Or, you can use the built in `$generate` method, which takes an optional delay in ms, and turns the array
-into an AsyncGenerator. In the example below the values 1 to 10 are pumped into the `generated` field
-with 10 ms temporal separation.
+You already saw hoe the built-in `$generate` function can accept a JS AsyncGeneraotr, and options. But $generate 
+can also be used to convert ordinary arrays or functions into async generators. When provided, the `interval` option, 
+causes the provided array to yield its elements periodically. When a function is provided, as opposed to an array, the
+function is called periodically.
 ```json
 > .init -f example/generate.json
 {
   "delayMs": 250,
-  "generated":"${[1..10]~>$generate(delayMs)}"
+  "generated":"${[1..10]~>$generate({'interval':delayMs})}"
 }
 ```
 ```json ["data.generated=10"]
@@ -1808,6 +1830,63 @@ Started tailing... Press Ctrl+C to stop.
 }
 
 ```
+This `example/myGenerator3.yaml` shows how you can call `return` and stop a generator.
+```yaml
+generated: ${$generate($random, {'interval':10, 'valueOnly':false})}
+onGenerated: |
+  ${
+    $count(accumulator)<3
+      ? $set('/accumulator/-', $$.generated.value)
+      : generated.return() /* shut off the generator when the accumulator has 10 items */
+  }
+accumulator: []
+```
+```json ["$count(data.accumulator)=3"]
+> .init -f example/myGenerator3.yaml --tail "/ until $count(accumulator)=3"
+Started tailing... Press Ctrl+C to stop.
+{
+  "generated": {
+    "value": 0.23433826655570145,
+    "done": false,
+    "return": "{function:}"
+  },
+  "onGenerated": {
+    "value": null,
+    "done": true
+  },
+  "accumulator": [
+    0.23433826655570145,
+    0.23433826655570145,
+    0.23433826655570145
+  ]
+}
+
+
+```
+The `maxYield` parameter can also be used to stop a generator:
+```json ["$count(data.accumulator)=0", "$count(data.accumulator)=5"]
+> .init -f example/myGenerator4.yaml
+{
+  "generated": "${$generate($random, {'interval':10, 'maxYield':5})}",
+  "onGenerated": "${$set('/accumulator/-', $$.generated)}",
+  "accumulator": []
+}
+> .init -f example/myGenerator4.yaml --tail "/ until $count(accumulator)=5"
+{
+  "generated": 0.5289126250886866,
+  "onGenerated": [
+    "/accumulator/-"
+  ],
+  "accumulator": [
+    0.3260049204634301,
+    0.4477190160739559,
+    0.9414436597923774,
+    0.8593436891141426,
+    0.5289126250886866
+  ]
+}
+```
+
 
 ### $setTimeout
 `$setTimeout` is the JavaScript `setTimeout` function. It receives a function and an timeout

example/executionStatus.json

@@ -120,7 +120,8 @@
         "expr__": "['luke', 'han', 'leia', 'chewbacca', 'Lando'].($forked('/name',$))",
         "exprTargetJsonPointer__": "",
         "compiledExpr__": "--compiled expression--",
-        "data__": null
+        "data__": null,
+        "variables__": ["forked"]
       },
       {
         "materialized__": true,
@@ -156,7 +157,8 @@
             "treeHasExpressions__": false,
             "parent__": "/homeworldDetails/properties"
           }
-        }
+        },
+        "variables__": ["fetch"]
       },
       {
         "materialized__": true,
@@ -177,7 +179,8 @@
         "exprRootPath__": null,
         "expr__": " homeworldDetails!=null?$joined('/homeworlds/-', homeworldDetails.properties.name):null ",
         "exprTargetJsonPointer__": "",
-        "compiledExpr__": "--compiled expression--"
+        "compiledExpr__": "--compiled expression--",
+        "variables__": ["joined"]
       },
       {
         "materialized__": true,
@@ -263,7 +266,8 @@
             "treeHasExpressions__": false,
             "parent__": "/personDetails/properties"
           }
-        }
+        },
+        "variables__": ["fetch", "save"]
       },
       {
         "materialized__": false,

example/generate.json

@@ -1,4 +1,4 @@
 {
   "delayMs": 250,
-  "generated":"${[1..10]~>$generate(delayMs)}"
+  "generated":"${[1..10]~>$generate({'interval':delayMs})}"
 }

example/myGenerator.mjs

@@ -1,5 +1,6 @@
 export async function* myGenerator() {
-    for (let i = 1; i <= 10; i++) {
+    for (let i = 1; i < 10; i++) {
         yield i;
     }
+    return 10; // Last value with `done: true`
 }

example/myGenerator3.yaml

@@ -0,0 +1,8 @@
+generated: ${$generate($random, {'interval':10, 'valueOnly':false})}
+onGenerated: |
+  ${
+    $count(accumulator)<3
+      ? $set('/accumulator/-', $$.generated.value)
+      : generated.return() /* shut off the generator when the accumulator has 10 items */
+  }
+accumulator: []

example/myGenerator4.yaml

@@ -0,0 +1,3 @@
+generated: ${$generate($random, {'interval':10, 'maxYield':5})}
+onGenerated:  ${$set('/accumulator/-', $$.generated)}
+accumulator: []

example/myGeneratorVerbose.json

@@ -0,0 +1,3 @@
+{
+  "generated": "${$myGenerator()~>$generate({'valueOnly':false})}"
+}

example/restoreSnapshot.json

@@ -64,7 +64,8 @@
         "expr__": " $setInterval(function(){$set('/count', count+1)}, 10) ",
         "exprTargetJsonPointer__": "",
         "compiledExpr__": "--compiled expression--",
-        "data__": "--interval/timeout--"
+        "data__": "--interval/timeout--",
+        "variables__": ["setInterval","set"]
       },
       {
         "materialized__": true,
@@ -86,7 +87,8 @@
         "expr__": " count=10?($clearInterval($$.counter);'done'):'not done'  ",
         "exprTargetJsonPointer__": "",
         "compiledExpr__": "--compiled expression--",
-        "data__": "not done"
+        "data__": "not done",
+        "variables__": ["clearInterval"]
       }
     ]
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "stated-js",
-  "version": "0.1.42",
+  "version": "0.1.43",
   "license": "Apache-2.0",
   "description": "JSONata embedded in JSON",
   "main": "./dist/src/index.js",

src/DependencyFinder.ts

@@ -45,6 +45,7 @@ export default class DependencyFinder {
     private readonly currentSteps: StepRecord[][]; //logically, [[a,b,c],[d,e,f]]
     private nodeStack: GeneratedExprNode[];
     private readonly dependencies: string[][]; //during tree walking we collect these dependencies like [["a", "b", "c"], ["foo", "bar"]] which means the dependencies are a.b.c and foo.bar
+    public readonly variables: Set<string> = new Set<string>();
     /**
      * program can be either a string to be compiled, or an already-compiled AST
      * @param program
@@ -102,6 +103,7 @@ export default class DependencyFinder {
             const {
                 type,
             } = node;
+            this.captureBuiltInFunctionNames(node);
             this.capturePathExpressions(node);
             this.captureArrayIndexes(node);
             this.nodeStack.push(node);
@@ -143,6 +145,21 @@ export default class DependencyFinder {
         return this.dependencies;
     }
 
+    /**
+     * Function calls like $count(...) are recorded in this.variables so that the MetaInf can have a record
+     * of what context variables (in this case functions) were accessed by the expression.
+     * @param node
+     * @private
+     */
+    private captureBuiltInFunctionNames(node:GeneratedExprNode) {
+        if(node.type === 'function'){
+            const name = node.procedure?.value;
+            if(name !== undefined){
+                this.variables.add(name);
+            }
+        }
+    }
+
     private markScopeWhenFunctionReturnsValue(scopeWeExited:GeneratedExprNode) {
         if (scopeWeExited?.type === 'function') { //function has returned
             const currentScope = DependencyFinder.peek(this.nodeStack);
@@ -238,6 +255,7 @@ export default class DependencyFinder {
         if (type === "variable") {
             //if we are here then the variable must be an ordinary locally named variable since it is neither $$ or $.
             //variables local to a closure cannot cause/imply a dependency for this expression
+            this.variables.add(value);
             if (!this.hasParent("function")) { //the function name is actually a variable, we want to skip such variables
                 //@ts-ignore
                 last(this.currentSteps).push({type, value, "emit": false});

src/MetaInfoProducer.ts

@@ -31,6 +31,7 @@ export interface MetaInfo{
     exprTargetJsonPointer__?:JsonPointerStructureArray|JsonPointerString //the pointer to the object that this expression executes on
     data__?:any
     isFunction__?:boolean
+    variables__?:string[]
 }
 
 export type JsonPointerStructureArray = (string|number)[];