From 4da38e78c282511cbb7f885928b64de9c0758783 Mon Sep 17 00:00:00 2001 From: Geoffrey Hendrey Date: Mon, 18 Nov 2024 16:41:01 -0800 Subject: [PATCH] flow command (#92) --- package.json | 2 +- src/CliCoreBase.ts | 10 ++ src/DataFlow.ts | 129 ++++++++++++++ src/StatedREPL.ts | 1 + src/TemplateProcessor.ts | 13 +- src/test/TemplateProcessor.test.js | 275 ++++++++++++++++++++++++++++- 6 files changed, 427 insertions(+), 3 deletions(-) create mode 100644 src/DataFlow.ts diff --git a/package.json b/package.json index 55d30d60..0ea38081 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stated-js", - "version": "0.1.45", + "version": "0.1.46", "license": "Apache-2.0", "description": "JSONata embedded in JSON", "main": "./dist/src/index.js", diff --git a/src/CliCoreBase.ts b/src/CliCoreBase.ts index 6c406def..9f85faa8 100644 --- a/src/CliCoreBase.ts +++ b/src/CliCoreBase.ts @@ -23,6 +23,7 @@ import { exec } from 'child_process'; import http from 'http'; import * as child_process from "child_process"; import os from "os"; +import {FlowOpt} from "./DataFlow.js"; /** * Base class for building CLIs. By itself can be used for a CLI that does not support the tail command. Tail command @@ -293,6 +294,15 @@ export class CliCoreBase { return option === '--shallow' ? this.templateProcessor.getDependencies(jsonPtr) : this.templateProcessor.to(jsonPtr); } + async flow(replCmdInputStr:string) { + if (!this.templateProcessor) { + throw new Error('Initialize the template first.'); + } + const {level=0} = CliCoreBase.minimistArgs(replCmdInputStr); //--level=5 + return this.templateProcessor.flow(level as FlowOpt); + } + + async plan() { if (!this.templateProcessor) { throw new Error('Initialize the template first.'); diff --git a/src/DataFlow.ts b/src/DataFlow.ts new file mode 100644 index 00000000..99e2bac9 --- /dev/null +++ b/src/DataFlow.ts @@ -0,0 +1,129 @@ +import {JsonPointerString, MetaInfo} from "./MetaInfoProducer.js"; +import TemplateProcessor from "./TemplateProcessor.js"; +import {default as jp} from "./JsonPointer.js"; +import {JsonPointer} from "./index.js"; +//type TreeNode = {metaInfo:MetaInfo, dependees:TreeNode[]}; + +/** + * Represents a node in a data flow structure. + * A DataFlowNode can either be a JsonPointerString or an object with a location + * and an optional 'to' field which points to one or more subsequent DataFlowNodes or a JsonPointerString. + * + * @typedef {Object} DataFlowNode + * @property {JsonPointerString} location - The location of the data in a JSON structure. + * @property {DataFlowNode[]|DataFlowNode|JsonPointerString} [to] - Optional field that indicates the next node or nodes in the data flow. + * + * @typedef {string} JsonPointerString - A string that represents a JSON Pointer. + */ +export type DataFlowNode = { + location:JsonPointerString, + to?:DataFlowNode[]|DataFlowNode|JsonPointerString +} | JsonPointerString + +export type FlowOpt = 0|1; + + +/** + * Class representing a DataFlow, managing dependencies and data flow nodes. + */ +export class DataFlow { + private templateProcessor: TemplateProcessor; + private visited:Map; + private roots:Set; + + constructor(templateProcessor: TemplateProcessor) { + this.visited = new Map(); + this.roots = new Set(); + this.templateProcessor = templateProcessor; + } + + /** + * Links the given metaInfo node with its dependees, creating and returning a new DataFlowNode. + * + * @param {MetaInfo} metaInfo - The metadata information object containing dependencies and location data. + * @return {DataFlowNode} - The created DataFlowNode with linked dependees. + */ + private linkDependees(metaInfo: MetaInfo): DataFlowNode { + let n:any = this.visited.get(metaInfo); + if(n){ + return n; + } + const {absoluteDependencies__:deps, jsonPointer__:location} = metaInfo; + n = {location, to:[]}; + this.visited.set(metaInfo, n); + for (const jsonPtr of metaInfo.dependees__) { + const dependeeMeta: MetaInfo = jp.get(this.templateProcessor.templateMeta, jsonPtr) as MetaInfo; + const dependeeTreeNode = this.linkDependees(dependeeMeta); + const dependees = n.to; + dependees.push(dependeeTreeNode); + } + + //a root has no dependencies and at least one dependee (todo: degenerate case of expression like ${42} that has np dependencies) + if(metaInfo.absoluteDependencies__.length === 0 && metaInfo.dependees__.length > 0){ + this.roots.add(n); + } + + return n; + } + + + /** + * Recursively compacts the given DataFlowNode, reducing the 'to' field to a single object + * if there is only one child node, or omitting it if there are no child nodes. + * + * @param {DataFlowNode} node - The node to compact. This node may have a 'to' field + * that is an array of child nodes. + * @return {DataFlowNode} The compacted node, which may have a simplified 'to' field or none at all. + */ + private level1Compact(node:DataFlowNode):DataFlowNode { + const kids = (node as any).to? (node as any).to as DataFlowNode[]:[]; + const compactKids = kids.map((kid) => this.level1Compact(kid)); + const noKids = compactKids.length === 0; + const oneKid = compactKids.length === 1; + let compactNode; + let location: JsonPointerString; + if (typeof node === "string"){ + location = node as JsonPointerString; + }else{ + location = node.location; + } + if(noKids){ + compactNode = location; //no 'to' at all, just a raw location + }else if (oneKid) { + const onlyKid = compactKids[0]; + compactNode = {location, to: onlyKid}; //'to' is just a single thing, not an array + }else{ + compactNode = {location, to:compactKids} + } + return compactNode as DataFlowNode; + + } + + /** + * Retrieves the roots of the data flow nodes based on the specified option. + * + * @param {string} level - The option for retrieving roots, it can be either "l0" or "l1". + * "l0" returns the roots as they are. + * "l1" returns the roots in a compacted form. + * @return {DataFlowNode[]} - An array of data flow node roots. + * @throws {Error} - Throws an error if the specified option is unknown. + */ + getRoots(level:FlowOpt=0):DataFlowNode[] { + this.visited = new Map(); + this.roots = new Set(); + const metas = this.templateProcessor.metaInfoByJsonPointer["/"]; + for (const m of metas) { + if(m.jsonPointer__ !== "") { //skip root json pointer + this.linkDependees(m); + } + } + if(level===0){ + return Array.from(this.roots); + }else if (level===1){ + return Array.from(this.roots).map(root=>this.level1Compact(root)); + }else{ + throw new Error(`Unknown option ${level}`); + } + + } +} diff --git a/src/StatedREPL.ts b/src/StatedREPL.ts index fd3c5f5b..f7d1ee44 100755 --- a/src/StatedREPL.ts +++ b/src/StatedREPL.ts @@ -40,6 +40,7 @@ export default class StatedREPL { ["state", 'Show the current state of the templateMeta'], ["from", 'Show the dependents of a given JSON pointer'], ["to", 'Show the dependencies of a given JSON pointer'], + ["flow", 'return an array or tree structures showing the data flow from sources to destinations'], ["plan", 'Show the evaluation plan'], ["note", "returns ═══ ... for creating documentation"], ["log", "set the log level [debug, info, warn, error]"], diff --git a/src/TemplateProcessor.ts b/src/TemplateProcessor.ts index 680205d8..af804539 100644 --- a/src/TemplateProcessor.ts +++ b/src/TemplateProcessor.ts @@ -37,6 +37,7 @@ import {LifecycleManager} from "./LifecycleManager.js"; import {accumulate} from "./utils/accumulate.js"; import {defaulter} from "./utils/default.js"; import {CliCoreBase} from "./CliCoreBase.js"; +import {DataFlow, DataFlowNode, FlowOpt} from "./DataFlow.js"; declare const BUILD_TARGET: string | undefined; @@ -986,7 +987,6 @@ export default class TemplateProcessor { } } - private topologicalSort(metaInfos:MetaInfo[], exprsOnly = true, fanout=true):JsonPointerString[] { const visited = new Set(); const recursionStack:Set = new Set(); //for circular dependency detection @@ -1878,6 +1878,17 @@ export default class TemplateProcessor { return []; } + + /** + * Controls the flow of data and retrieves root nodes based on the specified level. + * + * @param {FlowOpt} level - The level specifying the granularity of the data flow. + * @return {DataFlowNode[]} An array of root nodes that are computed based on the specified level. + */ + flow(level:FlowOpt):DataFlowNode[]{ + return new DataFlow(this).getRoots(level); + } + /** * Sets a data change callback function that will be called whenever the value at the json pointer has changed * @param jsonPtr diff --git a/src/test/TemplateProcessor.test.js b/src/test/TemplateProcessor.test.js index 3627470a..7b860518 100644 --- a/src/test/TemplateProcessor.test.js +++ b/src/test/TemplateProcessor.test.js @@ -25,6 +25,7 @@ import { default as jp } from "../../dist/src/JsonPointer.js"; import StatedREPL from "../../dist/src/StatedREPL.js"; import { jest, expect, describe, beforeEach, afterEach, test} from '@jest/globals'; import {LifecycleState} from "../../dist/src/Lifecycle.js"; +import {DataFlow} from "../../dist/src/DataFlow.js"; if (typeof Bun !== 'undefined') { // Dynamically import Jest's globals if in Bun.js environment @@ -3376,4 +3377,276 @@ test("test transaction", async () => { } finally { await tp.close(); } -}); \ No newline at end of file +}); + +test("test data flow 1", async () => { + const o = { + "a": 1, + "b": "${a}", + "c": "${a}", + "d": "${b}", + "e": "${b}", + "x": 42, + "y": "${x}", + "z": "${x}" + }; + const tp = new TemplateProcessor(o); + try { + await tp.initialize(); + let flows = tp.flow(0); + expect(flows).toStrictEqual([ + { + "location": "/a", + "to": [ + { + "location": "/b", + "to": [ + { + "location": "/d", + "to": [] + }, + { + "location": "/e", + "to": [] + } + ] + }, + { + "location": "/c", + "to": [] + } + ] + }, + { + "location": "/x", + "to": [ + { + "location": "/y", + "to": [] + }, + { + "location": "/z", + "to": [] + } + ] + } + ]); + flows = tp.flow(1); + expect(flows).toStrictEqual([ + { + "location": "/a", + "to": [ + { + "location": "/b", + "to": [ + "/d", + "/e" + ] + }, + "/c" + ] + }, + { + "location": "/x", + "to": [ + "/y", + "/z" + ] + } + ]); + } finally { + await tp.close(); + } +}); + +test("test data flow 2", async () => { + const o = { + "a": "${c}", + "b": "${d+1+e}", + "c": "${b+1}", + "d": "${e+1}", + "e": 1 + }; + const tp = new TemplateProcessor(o); + try { + await tp.initialize(); + let flows = tp.flow(); + expect(flows).toStrictEqual([ + { + "location": "/e", + "to": [ + { + "location": "/b", + "to": [ + { + "location": "/c", + "to": [ + { + "location": "/a", + "to": [] + } + ] + } + ] + }, + { + "location": "/d", + "to": [ + { + "location": "/b", + "to": [ + { + "location": "/c", + "to": [ + { + "location": "/a", + "to": [] + } + ] + } + ] + } + ] + } + ] + } + ] + ); + flows = tp.flow(1); + expect(flows).toStrictEqual([ + { + "location": "/e", + "to": [ + { + "location": "/b", + "to": { + "location": "/c", + "to": "/a" + } + }, + { + "location": "/d", + "to": { + "location": "/b", + "to": { + "location": "/c", + "to": "/a" + } + } + } + ] + } + ] + ); + } finally { + await tp.close(); + } +}); + +test("test data flow 3", async () => { + const o = { + "commanderDetails": { + "fullName": "../${commander.firstName & ' ' & commander.lastName}", + "salutation": "../${$join([commander.rank, commanderDetails.fullName], ' ')}", + "systemsUnderCommand": "../${$count(systems)}" + }, + "organization": "NORAD", + "location": "Cheyenne Mountain Complex, Colorado", + "commander": { + "firstName": "Jack", + "lastName": "Beringer", + "rank": "General" + }, + "purpose": "Provide aerospace warning, air sovereignty, and defense for North America", + "systems": [ + "Ballistic Missile Early Warning System (BMEWS)", + "North Warning System (NWS)", + "Space-Based Infrared System (SBIRS)", + "Cheyenne Mountain Complex" + ] + }; + const tp = new TemplateProcessor(o); + try { + await tp.initialize(); + let flows = tp.flow(); + expect(flows).toStrictEqual([ + { + "location": "/commander/firstName", + "to": [ + { + "location": "/commanderDetails/fullName", + "to": [ + { + "location": "/commanderDetails/salutation", + "to": [] + } + ] + } + ] + }, + { + "location": "/commander/lastName", + "to": [ + { + "location": "/commanderDetails/fullName", + "to": [ + { + "location": "/commanderDetails/salutation", + "to": [] + } + ] + } + ] + }, + { + "location": "/commander/rank", + "to": [ + { + "location": "/commanderDetails/salutation", + "to": [] + } + ] + }, + { + "location": "/systems", + "to": [ + { + "location": "/commanderDetails/systemsUnderCommand", + "to": [] + } + ] + } + ] + ); + flows = tp.flow(1); + expect(flows).toStrictEqual([ + { + "location": "/commander/firstName", + "to": { + "location": "/commanderDetails/fullName", + "to": "/commanderDetails/salutation" + } + }, + { + "location": "/commander/lastName", + "to": { + "location": "/commanderDetails/fullName", + "to": "/commanderDetails/salutation" + } + }, + { + "location": "/commander/rank", + "to": "/commanderDetails/salutation" + }, + { + "location": "/systems", + "to": "/commanderDetails/systemsUnderCommand" + } + ] + ); + } finally { + await tp.close(); + } +}); + +