From e84ba878cb31449842c36365dabc2c48053de552 Mon Sep 17 00:00:00 2001 From: Eli Sherer Date: Sat, 27 Jul 2024 18:23:43 +0300 Subject: [PATCH] refactor, use Sequency for streams --- .../javascript-json-transform-test.yml | 33 ++++ .../json-transform-core/package-lock.json | 14 +- javascript/json-transform-core/package.json | 4 +- javascript/json-transform/package-lock.json | 34 +++- javascript/json-transform/package.json | 8 +- .../json-transform/src/JsonElementStreamer.ts | 67 ++++++++ javascript/json-transform/src/JsonHelpers.ts | 35 +++- .../json-transform/src/JsonTransformer.ts | 13 +- .../src/__tests__/JsonTransformer.test.ts | 3 +- .../functions/TransformerFunctionAnd.test.ts | 22 +++ .../functions/TransformerFunctionAt.test.ts | 23 +++ .../functions/TransformerFunctionAvg.test.ts | 29 ++++ .../TransformerFunctionBoolean.test.ts | 46 +++++ .../functions/TransformerFunctionIs.test.ts | 130 ++++++++++++++ .../src/functions/TransformerFunctionAnd.ts | 6 +- .../src/functions/TransformerFunctionAt.ts | 12 +- .../src/functions/TransformerFunctionAvg.ts | 23 ++- .../functions/TransformerFunctionBoolean.ts | 31 ++++ .../src/functions/TransformerFunctionIs.ts | 158 ++++++++++++++++++ .../src/functions/common/ArgumentType.ts | 3 +- .../src/functions/common/FunctionContext.ts | 96 +++++++++-- .../src/functions/common/FunctionHelpers.ts | 8 + .../functions/common/TransformerFunction.ts | 13 +- .../src/transformerFunctions.ts | 7 +- 24 files changed, 753 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/javascript-json-transform-test.yml create mode 100644 javascript/json-transform/src/JsonElementStreamer.ts create mode 100644 javascript/json-transform/src/__tests__/functions/TransformerFunctionAnd.test.ts create mode 100644 javascript/json-transform/src/__tests__/functions/TransformerFunctionAt.test.ts create mode 100644 javascript/json-transform/src/__tests__/functions/TransformerFunctionAvg.test.ts create mode 100644 javascript/json-transform/src/__tests__/functions/TransformerFunctionBoolean.test.ts create mode 100644 javascript/json-transform/src/__tests__/functions/TransformerFunctionIs.test.ts create mode 100644 javascript/json-transform/src/functions/TransformerFunctionBoolean.ts create mode 100644 javascript/json-transform/src/functions/TransformerFunctionIs.ts create mode 100644 javascript/json-transform/src/functions/common/FunctionHelpers.ts diff --git a/.github/workflows/javascript-json-transform-test.yml b/.github/workflows/javascript-json-transform-test.yml new file mode 100644 index 0000000..8bdd3c9 --- /dev/null +++ b/.github/workflows/javascript-json-transform-test.yml @@ -0,0 +1,33 @@ +name: JavaScript json-transform Test + +on: + pull_request: + branches: + - main + paths: + - javascript/json-transform/** + +# cancel previous tests if new commit is pushed to PR branch +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 18 + cache: npm + cache-dependency-path: ./javascript/json-transform/package-lock.json + + - name: Install dependencies + working-directory: ./javascript/json-transform + run: npm ci + + - name: Run tests + working-directory: ./javascript/json-transform + run: npm test diff --git a/javascript/json-transform-core/package-lock.json b/javascript/json-transform-core/package-lock.json index e4ae781..015710c 100644 --- a/javascript/json-transform-core/package-lock.json +++ b/javascript/json-transform-core/package-lock.json @@ -1,22 +1,22 @@ { "name": "@nlighten/json-transform-core", - "version": "1.0.0", + "version": "1.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@nlighten/json-transform-core", - "version": "1.0.0", + "version": "1.0.4", "license": "MIT", "devDependencies": { - "@nlighten/json-schema-utils": "^1.0.0", + "@nlighten/json-schema-utils": "^1.0.2", "microbundle": "^0.15.1", "prettier": "3.1.1", "typescript": "^5.3.3", "vitest": "1.2.2" }, "peerDependencies": { - "@nlighten/json-schema-utils": "^1.0.0" + "@nlighten/json-schema-utils": "^1.0.2" } }, "node_modules/@ampproject/remapping": { @@ -2304,9 +2304,9 @@ } }, "node_modules/@nlighten/json-schema-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@nlighten/json-schema-utils/-/json-schema-utils-1.0.1.tgz", - "integrity": "sha512-2rhtw99i625TPr7lj8PR2H+iSg+z2zRaF06KXA7G6aichfpRBlSZZxIjUiZ21VNjluMyQoscpS5zskRl1ZPPNQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@nlighten/json-schema-utils/-/json-schema-utils-1.0.2.tgz", + "integrity": "sha512-2Og9g+IsE9+edk8ahtwSOHDDwh6cW2vT2dQrRt6c8gAxtNaiVJBp7f6ImdgY8CZ5u6RmaV/0MaJRPohvj2g1HA==", "dev": true }, "node_modules/@rollup/plugin-alias": { diff --git a/javascript/json-transform-core/package.json b/javascript/json-transform-core/package.json index 88fe9ab..9904ddb 100644 --- a/javascript/json-transform-core/package.json +++ b/javascript/json-transform-core/package.json @@ -40,10 +40,10 @@ }, "homepage": "https://github.com/nlighten-oss/json-transform#readme", "peerDependencies": { - "@nlighten/json-schema-utils": "^1.0.0" + "@nlighten/json-schema-utils": "^1.0.2" }, "devDependencies": { - "@nlighten/json-schema-utils": "^1.0.0", + "@nlighten/json-schema-utils": "^1.0.2", "microbundle": "^0.15.1", "prettier": "3.1.1", "typescript": "^5.3.3", diff --git a/javascript/json-transform/package-lock.json b/javascript/json-transform/package-lock.json index e2d0921..2bb5b3e 100644 --- a/javascript/json-transform/package-lock.json +++ b/javascript/json-transform/package-lock.json @@ -1,15 +1,19 @@ { - "name": "@nlighten/json-transform-ts", + "name": "@nlighten/json-transform", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "@nlighten/json-transform-ts", + "name": "@nlighten/json-transform", "version": "0.1.0", "license": "MIT", + "dependencies": { + "bignumber.js": "^9.1.2", + "sequency": "^0.20.0" + }, "devDependencies": { - "@nlighten/json-schema-utils": "^1.0.0", + "@nlighten/json-schema-utils": "^1.0.2", "@types/jsonpath": "^0.2.4", "@types/uuid": "^10.0.0", "jsonpath": "^1.1.1", @@ -20,7 +24,7 @@ "vitest": "1.2.2" }, "peerDependencies": { - "@nlighten/json-schema-utils": "^1.0.0", + "@nlighten/json-schema-utils": "^1.0.2", "jsonpath": "^1.1.1", "uuid": "^10.0.0" } @@ -2310,9 +2314,9 @@ } }, "node_modules/@nlighten/json-schema-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@nlighten/json-schema-utils/-/json-schema-utils-1.0.1.tgz", - "integrity": "sha512-2rhtw99i625TPr7lj8PR2H+iSg+z2zRaF06KXA7G6aichfpRBlSZZxIjUiZ21VNjluMyQoscpS5zskRl1ZPPNQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@nlighten/json-schema-utils/-/json-schema-utils-1.0.2.tgz", + "integrity": "sha512-2Og9g+IsE9+edk8ahtwSOHDDwh6cW2vT2dQrRt6c8gAxtNaiVJBp7f6ImdgY8CZ5u6RmaV/0MaJRPohvj2g1HA==", "dev": true }, "node_modules/@rollup/plugin-alias": { @@ -3048,6 +3052,14 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, + "node_modules/bignumber.js": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", + "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", + "engines": { + "node": "*" + } + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -7051,6 +7063,14 @@ "semver": "bin/semver.js" } }, + "node_modules/sequency": { + "version": "0.20.0", + "resolved": "https://registry.npmjs.org/sequency/-/sequency-0.20.0.tgz", + "integrity": "sha512-WpqgbxLrvAOhkNHcluXt+TWWdYm4U4vAW5lLuDki+QIZxnnOG6tCbhKdDV6UKpapdYCN7+YKZreMKw79Pg6kBA==", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/serialize-javascript": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", diff --git a/javascript/json-transform/package.json b/javascript/json-transform/package.json index 44cd463..54e18ee 100644 --- a/javascript/json-transform/package.json +++ b/javascript/json-transform/package.json @@ -40,12 +40,12 @@ }, "homepage": "https://github.com/nlighten-oss/json-transform#readme", "peerDependencies": { - "@nlighten/json-schema-utils": "^1.0.0", + "@nlighten/json-schema-utils": "^1.0.2", "jsonpath": "^1.1.1", "uuid": "^10.0.0" }, "devDependencies": { - "@nlighten/json-schema-utils": "^1.0.0", + "@nlighten/json-schema-utils": "^1.0.2", "@types/jsonpath": "^0.2.4", "@types/uuid": "^10.0.0", "jsonpath": "^1.1.1", @@ -54,5 +54,9 @@ "typescript": "^5.3.3", "uuid": "^10.0.0", "vitest": "1.2.2" + }, + "dependencies": { + "bignumber.js": "^9.1.2", + "sequency": "^0.20.0" } } diff --git a/javascript/json-transform/src/JsonElementStreamer.ts b/javascript/json-transform/src/JsonElementStreamer.ts new file mode 100644 index 0000000..d73a1a1 --- /dev/null +++ b/javascript/json-transform/src/JsonElementStreamer.ts @@ -0,0 +1,67 @@ +import FunctionContext from "./functions/common/FunctionContext"; +import Sequence, { asSequence, emptySequence } from "sequency"; + +class JsonElementStreamer { + private readonly context: FunctionContext; + private readonly transformed: boolean; + private readonly value?: any[]; + private readonly _stream?: Sequence; + + private constructor(context: FunctionContext, value: any[] | Sequence, transformed: boolean) { + this.context = context; + this.transformed = transformed; + if (Array.isArray(value)) { + this.value = value; + this._stream = undefined; + } else { + this.value = undefined; + this._stream = value; + } + } + + public knownAsEmpty() { + return this.value && this.value.length == 0; + } + + public stream(skip: number = 0, limit: number = -1) { + if (this._stream != null) { + const skipped = skip > 0 ? this._stream.drop(skip) : this._stream; + return limit > -1 ? skipped.take(limit) : skipped; + } + if (this.value == null) { + return emptySequence(); + } + let valueStream = asSequence(this.value); + if (skip > 0) { + valueStream = valueStream.drop(skip); + } + if (limit > -1) { + valueStream = valueStream.take(limit); + } + if (!this.transformed) { + valueStream = valueStream.map(el => this.context.transform(el)); + } + return valueStream; + } + + public static fromJsonArray(context: FunctionContext, value: any[], transformed: boolean) { + return new JsonElementStreamer(context, value, transformed); + } + + public static fromTransformedStream(context: FunctionContext, stream: Sequence) { + return new JsonElementStreamer(context, stream, true); + } + + public toJsonArray() { + if (this.value) { + return this.value; + } + const ja: any[] = []; + if (this._stream) { + this._stream.forEach(item => ja.push(item)); + } + return ja; + } +} + +export default JsonElementStreamer; \ No newline at end of file diff --git a/javascript/json-transform/src/JsonHelpers.ts b/javascript/json-transform/src/JsonHelpers.ts index 7f8db67..d1dd610 100644 --- a/javascript/json-transform/src/JsonHelpers.ts +++ b/javascript/json-transform/src/JsonHelpers.ts @@ -1,5 +1,7 @@ import { v4 as uuidv4 } from 'uuid'; import DocumentContext from "./DocumentContext"; +import {BigDecimal} from "./functions/common/FunctionHelpers"; +import {areSimilar} from "@nlighten/json-schema-utils"; const JSONPATH_ROOT = "$", JSONPATH_ROOT_ESC = "\\$", @@ -29,6 +31,8 @@ const numberCompare = (a: number, b: number) => { return a < b ? -1 : (a === b ? 0 : 1); } +const numberType = (a: any) => typeof a === 'number' || typeof a === 'bigint' || a instanceof BigDecimal; + const compareTo = (a: any, b: any) => { if (Array.isArray(a) && Array.isArray(b)) { return numberCompare(a.length, b.length); @@ -36,8 +40,8 @@ const compareTo = (a: any, b: any) => { return numberCompare(Object.keys(a).length, Object.keys(b).length); } else if (typeof a === 'string' && typeof b === 'string') { return a.localeCompare(b); - } else if (typeof a === 'number' && typeof b === 'number') { - return numberCompare(a, b); + } else if (numberType(a) && numberType(b)) { + return BigDecimal(a).comparedTo(BigDecimal(b)); } else if (typeof a === 'boolean' && typeof b === 'boolean') { return a === b ? 0 : (a ? 1 : -1); } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) { @@ -143,23 +147,35 @@ const lenientJsonParse = (input: string) => { const BIGINT_ZERO = BigInt(0); const isTruthy = (value: any, javascriptStyle?: boolean) => { - if (Array.isArray(value)) { - return value.length > 0; - } else if (value && typeof value === 'object') { - return Object.keys(value).length > 0; - } if (typeof value === 'boolean') { return value; } else if (typeof value === 'number') { return value != 0; } else if (typeof value === 'bigint') { return value !== BIGINT_ZERO; + } else if (value instanceof BigDecimal) { + return !value.isZero(); } else if (typeof value === 'string') { - return javascriptStyle ? value !== '' : value === 'true'; + return javascriptStyle ? Boolean(value) : value.toLowerCase() === 'true'; + } else if (Array.isArray(value)) { + return value.length > 0; + } else if (value && typeof value === 'object') { + return Object.keys(value).length > 0; } return !isNullOrUndefined(value); } +const isEqual = (value: any, other: any): boolean => { + if (value === other) { + return true; + } + if (numberType(value) && numberType(other)) { + return BigDecimal(value).eq(BigDecimal(other)); + } + return areSimilar(value, other); +} + + export { isNullOrUndefined, createPayloadResolver, @@ -167,5 +183,6 @@ export { compareTo, getDocumentContext, lenientJsonParse, - isTruthy + isTruthy, + isEqual }; \ No newline at end of file diff --git a/javascript/json-transform/src/JsonTransformer.ts b/javascript/json-transform/src/JsonTransformer.ts index cfe070c..6eb8d8b 100644 --- a/javascript/json-transform/src/JsonTransformer.ts +++ b/javascript/json-transform/src/JsonTransformer.ts @@ -3,6 +3,7 @@ import {JsonTransformerFunction} from "./JsonTransformerFunction"; import {ParameterResolver} from "./ParameterResolver"; import {createPayloadResolver, isNullOrUndefined} from "./JsonHelpers"; import transformerFunctions, { TransformerFunctions } from "./transformerFunctions"; +import JsonElementStreamer from "./JsonElementStreamer"; class JsonTransformer implements Transformer { @@ -32,13 +33,16 @@ class JsonTransformer implements Transformer { fromJsonPrimitive(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean) : any { if (typeof definition !== 'string') { - return definition; + return definition ?? null; } try { // test for inline function (e.g. $$function:...) const match = this.transformerFunctions.matchInline(definition, resolver, this.JSON_TRANSFORMER); if (match != null) { - // TODO: add streams support + const matchResult = match.getResult(); + if (matchResult instanceof JsonElementStreamer) { + return allowReturningStreams ? matchResult : matchResult.toJsonArray(); + } return match.getResult(); } // jsonpath / context @@ -52,7 +56,10 @@ class JsonTransformer implements Transformer { fromJsonObject(definition: any, resolver: ParameterResolver, allowReturningStreams: boolean) : any { const match = this.transformerFunctions.matchObject(definition, resolver, this.JSON_TRANSFORMER); if (match != null) { - // TODO: add streams support + const res = match.getResult(); + if (res instanceof JsonElementStreamer) { + return allowReturningStreams ? res : res.toJsonArray(); + } return match.getResult(); } diff --git a/javascript/json-transform/src/__tests__/JsonTransformer.test.ts b/javascript/json-transform/src/__tests__/JsonTransformer.test.ts index cc69084..c31dccd 100644 --- a/javascript/json-transform/src/__tests__/JsonTransformer.test.ts +++ b/javascript/json-transform/src/__tests__/JsonTransformer.test.ts @@ -108,7 +108,8 @@ describe("JsonTransformer", () => { }); }); - test("InputExtractorSpreadDontRemoveByNull", () => { + // skipped since it doesn't work the same in javascript (null are being treated as values) + test.skip("InputExtractorSpreadDontRemoveByNull", () => { assertTransformation({ "a": "A", "b": "B" diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionAnd.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionAnd.test.ts new file mode 100644 index 0000000..bdd366c --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionAnd.test.ts @@ -0,0 +1,22 @@ +import { describe, test } from "vitest"; +import { assertTransformation} from "../BaseTransformationTest"; + +describe("TransformerFunctionAnd", () => { + test("assertAnd", () => { + const twoAndThree = + { + "$$and": [ + {"$$is": "$[0]", "eq": 2}, + {"$$is": "$[1]", "eq": 3} + ] + }; + assertTransformation([2,3], twoAndThree, true); + assertTransformation([2,4], twoAndThree, false); + }); + + test("inline", () => { + assertTransformation([null,0], "$$and:$", false); + assertTransformation([1,0], "$$and:$", false); + assertTransformation([1,true], "$$and:$", true); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionAt.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionAt.test.ts new file mode 100644 index 0000000..aa5e1b0 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionAt.test.ts @@ -0,0 +1,23 @@ +import { describe, test } from "vitest"; +import { assertTransformation} from "../BaseTransformationTest"; +import {BigDecimal} from "../../functions/common/FunctionHelpers"; + +describe("TransformerFunctionAt", () => { + test("inline", () => { + const arr = [4, 2, 13]; + assertTransformation(arr, "$$at(0):$", 4); + assertTransformation(arr, "$$at(1):$", 2); + assertTransformation(arr, "$$at(-1):$", 13); + assertTransformation(arr, "$$at(3):$", null); + assertTransformation(arr, "$$at:$", null); + }); + + test("object", () => { + const arr = [4, 2, BigDecimal(13)]; + assertTransformation(arr, {"$$at": "$", index: 0}, 4); + assertTransformation(arr, {"$$at": "$", index: 1}, 2); + assertTransformation(arr, {"$$at": "$", index: -1}, BigDecimal(13)); + assertTransformation(arr, {"$$at": "$", index: 3}, null); + assertTransformation(arr, {"$$at": "$"}, null); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionAvg.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionAvg.test.ts new file mode 100644 index 0000000..66cb107 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionAvg.test.ts @@ -0,0 +1,29 @@ +import { describe, test } from "vitest"; +import { assertTransformation} from "../BaseTransformationTest"; +import {BigDecimal} from "../../functions/common/FunctionHelpers"; + +class Holder { + readonly value: number|null|bigint; + + constructor(value: number|null|bigint) { + this.value = value; + } +} + +describe("TransformerFunctionAvg", () => { + test("inline", () => { + const arr = [4, BigInt(2), 13.45, null]; + assertTransformation(arr, "$$avg():$", BigDecimal(4.8625)); + assertTransformation(arr, "$$avg(1):$", BigDecimal(5.1125)); + }); + + test("object", () => { + const arr = [ + new Holder(BigInt(4)), + new Holder(BigInt(2)), + new Holder(13.45), + new Holder(null)]; + assertTransformation(arr, {"$$avg": "$", "by": "##current.value"}, BigDecimal(4.8625)); + assertTransformation(arr, {"$$avg": "$", "by": "##current.value", "default": 1}, BigDecimal(5.1125)); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionBoolean.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionBoolean.test.ts new file mode 100644 index 0000000..fcb1934 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionBoolean.test.ts @@ -0,0 +1,46 @@ +import { describe, test } from "vitest"; +import { assertTransformation} from "../BaseTransformationTest"; + +describe("TransformerFunctionBoolean", () => { + test("truthy", () => { + assertTransformation(true, "$$boolean:$", true); + // string + assertTransformation("0", "$$boolean(js):$", true); + assertTransformation("false", "$$boolean(js):$", true); + assertTransformation("true", "$$boolean:$", true); + assertTransformation("True", "$$boolean:$", true); + assertTransformation("true", "$$boolean(JS):$", true); + // number + assertTransformation(1, "$$boolean:$", true); + assertTransformation(-1, "$$boolean:$", true); + assertTransformation(BigInt("1"), "$$boolean:$", true); + // object + assertTransformation({"":0}, "$$boolean:$", true); + // arrays + assertTransformation([0], "$$boolean:$", true); + }); + + test("falsy", () => { + assertTransformation(false, "$$boolean:$", false); + // string + assertTransformation("", "$$boolean:$", false); + assertTransformation("", "$$boolean(js):$", false); + assertTransformation("0", "$$boolean:$", false); + assertTransformation("false", "$$boolean:$", false); + assertTransformation("False", "$$boolean:$", false); + // number + assertTransformation(0, "$$boolean:$", false); + assertTransformation(BigInt(0), "$$boolean:$", false); + // object + assertTransformation(null, "$$boolean:$", false); + assertTransformation({}, "$$boolean:$", false); + // arrays + assertTransformation([], "$$boolean:$", false); + }); + + test("object", () => { + assertTransformation("true", JSON.parse("{\"$$boolean\":\"$\",\"style\":\"JS\"}"), true); + assertTransformation("false", JSON.parse("{\"$$boolean\":\"$\",\"style\":\"js\"}"), true); + assertTransformation("false", JSON.parse("{\"$$boolean\":\"$\"}"), false); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionIs.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionIs.test.ts new file mode 100644 index 0000000..95cb531 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionIs.test.ts @@ -0,0 +1,130 @@ +import { describe, test } from "vitest"; +import { assertTransformation} from "../BaseTransformationTest"; + +describe("TransformerFunctionIs", () => { + test("assertEq_NotEq", () => { + assertTransformation("A", + { "$$is": "$", "eq": "A" }, true); + assertTransformation("A", + { "$$is": "$", "eq": "B" }, false); + assertTransformation(4, + { "$$is": "$", "eq": 4 }, true); + assertTransformation(4.5, + { "$$is": "$", "eq": 4 }, false); + assertTransformation(4.5, + { "$$is": "$", "neq": 4 }, true); + assertTransformation(4.5, + { "$$is": "$", "eq": 4.5, "neq": 4 }, true); + }); + + test("assertGt_Gte_Lt_Lte", () => { + assertTransformation("B", + { "$$is": "$", "gt": "A" }, true); + assertTransformation("B", + { "$$is": "$", "gte": "B" }, true); + assertTransformation(4, + { "$$is": "$", "gt": 3 }, true); + assertTransformation(4, + { "$$is": "$", "gte": 4 }, true); + assertTransformation(4, + { "$$is": "$", "lte": 4 }, true); + assertTransformation(3, + { "$$is": "$", "lt": 4 }, true); + assertTransformation(4, + { "$$is": "$", "lt": 4 }, false); + assertTransformation([1,2,3], + { "$$is": "$", "lt": [true,"a","b","c"] }, true); + assertTransformation([1,2,3], + { "$$is": "$", "gte": ["a","b","c"] }, true); + assertTransformation( + { "a": 1, "b": 2 }, + { "$$is": "$", "gte": { "key1": "a", "key2": "b" } }, + true); + }); + + test("assertIn_NotIn", () => { + assertTransformation("A", + { "$$is": "$", "in": ["A", "B"] }, true); + + assertTransformation(["A", "B"], + { "$$is": "A", "in": "$" }, true); + assertTransformation(["A", "B"], + { "$$is": "B", "in": ["$[0]","$[1]"] }, + true); + assertTransformation(["a", "B"], + { "$$is": "A", "in": "$" }, false); + // other types + assertTransformation([false, true], + { "$$is": true, "in": "$" }, true); + assertTransformation(null, + { "$$is": 30, "in": [10,20,30] }, true); + assertTransformation(null, + { "$$is": 30, "nin": [10,20,30] }, false); + assertTransformation(null, + { "$$is": 30, "in": "$" }, false); + assertTransformation(null, + { "$$is": 30, "nin": "$" }, false); + // even complex + assertTransformation(null, + { "$$is": [{"a": 1}], "in": [[{"a": 4}], [{"a": 1}], [{"a": 3}]] }, + true); + assertTransformation(null, + { "$$is": 30, "in": [10,20,30], "nin": [40,50,60] }, true); + assertTransformation(null, + { "$$is": 30, "in": [40,50,60], "nin": [10,20,30] }, false); + }); + + test("andOrExample", () => { + // check if number is between 1 < x < 3 or 4 <= x <= 6 + var between1And3 = + { "$$is": "$", "gt": 1, "lt": 3 }; + var between4And6 = + { "$$is": "$", "gte": 4, "lte": 6 }; + var definition = + { "$$is": true, "in": [ between1And3 , between4And6 ] }; + + var goodValues = [2, 4, 5, 6]; + var badValues = [1, 3, 7]; + + for (const value of goodValues) { + assertTransformation(value, definition, true); + } + for (const value of badValues) { + assertTransformation(value, definition, false); + } + }); + + test("objectOpThat", () => { + assertTransformation("A", + { "$$is": "$", "op": "EQ", "that": "A" }, true); + assertTransformation("A", + { "$$is": "$", "op": "EQ", "that": "B" }, false); + assertTransformation("A", + { "$$is": "$", "op": "!=", "that": "B" }, true); + assertTransformation(5, + { "$$is": "$", "op": ">", "that": 2 }, true); + }); + + test("inlineOpThat", () => { + assertTransformation("A", "$$is(EQ,A):$", true); + assertTransformation("A", "$$is(=,B):$", false); + assertTransformation("A", "$$is(!=,B):$", true); + // string comparison vs number comparison + assertTransformation("10", "$$is(>,2):$", false); + assertTransformation(10, "$$is(>,2):$", true); + // in / not in + assertTransformation( + ["a","b","A","B"], "$$is(IN,$):A", true); + assertTransformation( + ["a","b","A","B"], "$$is(IN,$):C", false); + assertTransformation( + ["a","b","A","B"], "$$is(NIN,$):C", true); + assertTransformation(null, "$$is(in,$):C", false); + assertTransformation(null, "$$is(Nin,$):C", false); + }); + + test("inlineCompareToNull", () => { + assertTransformation(null, "$$is(!=,#null):$", false); + assertTransformation(null, "$$is(=,#null):$", true); + }); +}); \ No newline at end of file diff --git a/javascript/json-transform/src/functions/TransformerFunctionAnd.ts b/javascript/json-transform/src/functions/TransformerFunctionAnd.ts index f5b03dc..9a994a3 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionAnd.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionAnd.ts @@ -16,11 +16,11 @@ class TransformerFunctionAnd extends TransformerFunction { } override apply(context: FunctionContext): any { - const arr = context.getJsonArray(null); + const arr = context.getJsonElementStreamer(null); if (arr == null) { - return null; + return false; } - return arr.every(item => isTruthy(item)); + return arr.stream().all(item => isTruthy(item)); } } diff --git a/javascript/json-transform/src/functions/TransformerFunctionAt.ts b/javascript/json-transform/src/functions/TransformerFunctionAt.ts index 2e8ade7..5e4e2c6 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionAt.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionAt.ts @@ -22,7 +22,7 @@ class TransformerFunctionAt extends TransformerFunction { } override apply(context: FunctionContext): any { - const value = context.getJsonArray(null); + const value = context.getJsonElementStreamer(null); if (value == null) { return null; } @@ -30,7 +30,15 @@ class TransformerFunctionAt extends TransformerFunction { if (index == null) { return null; } - return value.at(index); + if (index == 0) { + return value.stream().firstOrNull(); + } + if (index > 0) { + return value.stream(index).firstOrNull(); + } + // negative + const arr = value.toJsonArray(); + return arr.at(index) ?? null; } } diff --git a/javascript/json-transform/src/functions/TransformerFunctionAvg.ts b/javascript/json-transform/src/functions/TransformerFunctionAvg.ts index f5c5c6f..f16f6b8 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionAvg.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionAvg.ts @@ -1,8 +1,10 @@ +import BigNumber from "bignumber.js"; import TransformerFunction from "./common/TransformerFunction"; import {ArgType} from "./common/ArgType"; import FunctionContext from "./common/FunctionContext"; import {FunctionDescription} from "./common/FunctionDescription"; import {isNullOrUndefined} from "../JsonHelpers"; +import {BigDecimal} from "./common/FunctionHelpers"; const DESCRIPTION : FunctionDescription = { alias: "avg", @@ -26,17 +28,22 @@ class TransformerFunctionAvg extends TransformerFunction { } override apply(context: FunctionContext): any { - const value = context.getJsonArray(null); - if (value == null) { + const value = context.getJsonElementStreamer(null); + if (value == null || value.knownAsEmpty()) { return null; } const by = context.getJsonElement( "by", false); - const def = context.getBigDecimal("default") ?? 0; - let size = value.length; // TODO: when streams will be used, needs to count during reduce - return Math.round(value.reduce((a, c) => { - let val = !isNullOrUndefined(by) ? context.transformItem(by, c) : c; - return a + (isNullOrUndefined(val) ? def : val); - }, 0) / size); + const _default = context.getBigDecimal("default") ?? BigDecimal(0); + let size = 0; + const result = value.stream() + .map(t => { + size++; + let res = !isNullOrUndefined(by) ? context.transformItem(by, t) : t; + return isNullOrUndefined(res) ? _default : BigDecimal(res); + }) + .reduce((a: BigNumber, c) => a.plus(c)) + .dividedBy(size); + return result; } } diff --git a/javascript/json-transform/src/functions/TransformerFunctionBoolean.ts b/javascript/json-transform/src/functions/TransformerFunctionBoolean.ts new file mode 100644 index 0000000..e0acac1 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionBoolean.ts @@ -0,0 +1,31 @@ +import TransformerFunction from "./common/TransformerFunction"; +import {ArgType} from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import {FunctionDescription} from "./common/FunctionDescription"; +import {isNullOrUndefined, isTruthy} from "../JsonHelpers"; + +const DESCRIPTION : FunctionDescription = { + alias: "boolean", + description: "", + inputType: ArgType.Any, + arguments: { + style: { + type: ArgType.Enum, position: 0, defaultEnum: "JAVA", + enumValues: ["JAVA", "JS"], + description: "Style of considering truthy values (JS only relates to string handling; not objects and arrays)" + } + }, + outputType: ArgType.Boolean +}; +class TransformerFunctionBoolean extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override apply(context: FunctionContext): any { + const jsStyle = context.getEnum("style") === "JS"; + return isTruthy(context.getUnwrapped(null), jsStyle); + } +} + +export default TransformerFunctionBoolean; \ No newline at end of file diff --git a/javascript/json-transform/src/functions/TransformerFunctionIs.ts b/javascript/json-transform/src/functions/TransformerFunctionIs.ts new file mode 100644 index 0000000..ccb74e1 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionIs.ts @@ -0,0 +1,158 @@ +import TransformerFunction from "./common/TransformerFunction"; +import {ArgType} from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import {FunctionDescription} from "./common/FunctionDescription"; +import {isEqual, isNullOrUndefined} from "../JsonHelpers"; +import {BigDecimal} from "./common/FunctionHelpers"; + +const DESCRIPTION : FunctionDescription = { + alias: "is", + description: "", + inputType: ArgType.Any, + arguments: { + op: { + type: ArgType.Enum, position: 0, defaultIsNull: true, + enumValues: ["IN", "NIN", "EQ", "=", "==", "NEQ", "!=", "<>", "GT", ">", "GTE", ">=", "LT", "<", "LTE", "<="], + description: "A type of check to do exclusively (goes together with `that`)" + }, + that: { + type: ArgType.Any, position: 1, defaultIsNull: true, + description: "A transformer to extract a property to sum by (using ##current to refer to the current item)" + }, + in: { + type: ArgType.Array, defaultIsNull: true, + description: "Array of values the input should be part of" + }, + nin: { + type: ArgType.Array, defaultIsNull: true, + description: "Array of values the input should **NOT** be part of" + }, + eq: { + type: ArgType.Any, defaultIsNull: true, + description: "A Value the input should be equal to" + }, + neq: { + type: ArgType.Any, defaultIsNull: true, + description: "A Value the input should **NOT** be equal to" + }, + gt: { + type: ArgType.Any, defaultIsNull: true, + description: "A Value the input should be greater than (input > value)" + }, + gte: { + type: ArgType.Any, defaultIsNull: true, + description: "A Value the input should be greater than or equal (input >= value)" + }, + lt: { + type: ArgType.Any, defaultIsNull: true, + description: "A Value the input should be lower than (input < value)" + }, + lte: { + type: ArgType.Any, defaultIsNull: true, + description: "A Value the input should be lower than or equal (input <= value)" + }, + }, + outputType: ArgType.Boolean +}; +class TransformerFunctionIs extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override apply(context: FunctionContext): any { + const value = context.getJsonElement(null); + if (context.has("op")) { + const op = context.getEnum("op"); + let that: any = null; + // if operator is not in/nin then prepare the "that" argument which is a JsonElement + if (op !== "IN" && op !== "NIN") { + that = context.isJsonNumber(value) + ? context.getBigDecimal("that") + : context.getJsonElement("that"); + } + switch (op) { + case "IN": { + const _in = context.getJsonElementStreamer("that"); + return _in != null && _in.stream().any(item => isEqual(item, value)); + } + case "NIN": { + var nin = context.getJsonElementStreamer("that"); + return nin != null && nin.stream().none(item => isEqual(item, value)); + } + case "GT": + case ">": { + const comparison = context.compareTo(value, that); + return comparison != null && comparison > 0; + } + case "GTE": + case ">=": { + const comparison = context.compareTo(value, that); + return comparison != null && comparison >= 0; + } + case "LT": + case "<": { + const comparison = context.compareTo(value, that); + return comparison != null && comparison < 0; + } + case "LTE": + case "<=": { + const comparison = context.compareTo(value, that); + return comparison != null && comparison <= 0; + } + case "EQ": + case "=": + case "==": { + return value === that; + } + case "NEQ": + case "!=": + case "<>": { + return value !== that; + } + default: { + return false; + } + } + } + let result = true; + if (context.has("in")) { + const _in = context.getJsonElementStreamer("in"); + result = _in != null && _in.stream().any(item => isEqual(item, value)); + } + if (result && context.has("nin")) { + const nin = context.getJsonElementStreamer("nin"); + result = nin != null && nin.stream().none(item => isEqual(item, value)); + } + if (result && context.has("gt")) { + const gt = context.getJsonElement("gt"); + const comparison = context.compareTo(value, gt); + result = comparison != null && comparison > 0; + } + if (result && context.has("gte")) { + const gte = context.getJsonElement("gte"); + const comparison = context.compareTo(value, gte); + result = comparison != null && comparison >= 0; + } + if (result && context.has("lt")) { + const lt = context.getJsonElement("lt"); + const comparison = context.compareTo(value, lt); + result = comparison != null && comparison < 0; + } + if (result && context.has("lte")) { + const lte = context.getJsonElement("lte"); + const comparison = context.compareTo(value, lte); + result = comparison != null && comparison <= 0; + } + if (result && context.has("eq")) { + const eq = context.getJsonElement("eq"); + result = value === eq; + } + if (result && context.has("neq")) { + const neq = context.getJsonElement("neq"); + result = value !== neq; + } + return result; + } +} + +export default TransformerFunctionIs; \ No newline at end of file diff --git a/javascript/json-transform/src/functions/common/ArgumentType.ts b/javascript/json-transform/src/functions/common/ArgumentType.ts index 88d361c..0e6b7f8 100644 --- a/javascript/json-transform/src/functions/common/ArgumentType.ts +++ b/javascript/json-transform/src/functions/common/ArgumentType.ts @@ -1,4 +1,5 @@ import {ArgType} from "./ArgType"; +import {BigDecimal} from "./FunctionHelpers"; export type ArgumentType = { description: string; @@ -13,7 +14,7 @@ export type ArgumentType = { defaultEnum?: string // default "" defaultInteger?: number; // default -1 defaultLong?: number; // default -1L - defaultBigDecimal?: number; // default -1 + defaultBigDecimal?: number | string; // default -1 aliases?: string[]; // default {}; } \ No newline at end of file diff --git a/javascript/json-transform/src/functions/common/FunctionContext.ts b/javascript/json-transform/src/functions/common/FunctionContext.ts index 4275ebb..3252c42 100644 --- a/javascript/json-transform/src/functions/common/FunctionContext.ts +++ b/javascript/json-transform/src/functions/common/FunctionContext.ts @@ -2,6 +2,8 @@ import TransformerFunction from "./TransformerFunction"; import {ParameterResolver} from "../../ParameterResolver"; import {JsonTransformerFunction} from "../../JsonTransformerFunction"; import {compareTo, isNullOrUndefined, getAsString, getDocumentContext} from "../../JsonHelpers"; +import {BigDecimal} from "./FunctionHelpers"; +import JsonElementStreamer from "../../JsonElementStreamer"; class FunctionContext { protected static readonly CONTEXT_KEY= "context"; @@ -66,11 +68,11 @@ class FunctionContext { return this.resolver; } - protected has(name: string): boolean { + public has(name: string): boolean { return false }; - protected get(name: string | null, transform: boolean = true): any { + public get(name: string | null, transform: boolean = true): any { return null; } @@ -86,13 +88,23 @@ class FunctionContext { return typeof value === 'boolean'; } + public getUnwrapped(name: string | null, reduceBigDecimals?: boolean) { + const value = this.get(name, true); + if (value instanceof JsonElementStreamer) { + return value.toJsonArray(); + } + return value; + } + public compareTo(a: any, b: any) { return compareTo(a, b); } public getJsonElement(name: string | null, transform: boolean = true) { const value = this.get(name, transform); - // TODO: add stream support + if (value instanceof JsonElementStreamer) { + return value.toJsonArray(); + } return value; } @@ -126,14 +138,20 @@ class FunctionContext { return value.trim().toUpperCase(); } - private getNumber(name: string | null, transform: boolean = true) { + public getInteger(name: string | null, transform: boolean = true) { const value = this.get(name, transform); if (value == null) { return null; } + if (value instanceof BigDecimal) { + return Math.floor(value.toNumber()); + } if (typeof value === 'number') { return Math.floor(value); } + if (typeof value === 'bigint') { + return Number(value); + } let str = getAsString(value); if (str == null) return null; str = str.trim(); @@ -141,26 +159,80 @@ class FunctionContext { return parseInt(value); } - public getInteger(name: string | null, transform: boolean = true) { - return this.getNumber(name, transform) - } - public getLong(name: string | null, transform: boolean = true) { - return this.getNumber(name, transform); + const value = this.get(name, transform); + if (value == null) { + return null; + } + if (typeof value === 'bigint') { + return value; + } + if (typeof value === 'number') { + return BigInt(value); + } + if (value instanceof BigDecimal) { + return BigInt(value.toFixed(0)); + } + let str = getAsString(value); + if (str == null) return null; + str = str.trim(); + if (str === "") return null; + return BigInt(value); } public getBigDecimal(name: string | null, transform: boolean = true) { - return this.getNumber(name, transform); + const value = this.get(name, transform); + if (value == null) { + return null; + } + if (value instanceof BigDecimal) { + return value; + } + if (typeof value === 'number') { + return new BigDecimal(value); + } + let str = getAsString(value); + if (str == null) return null; + str = str.trim(); + if (str === "") return null; + return new BigDecimal(value); } public getJsonArray(name: string | null, transform: boolean = true) { const value = this.get(name, transform); - // TODO: add stream support + if (value instanceof JsonElementStreamer) { + return value.toJsonArray(); + } return Array.isArray(value) ? value : null; } + /** + * Use this method instead of getJsonArray if you plan on iterating over the array + * The pros of using this method are + * - That it will not transform all the array to a single (possibly huge) array in memory + * - It lazy transforms the array elements, so if there is short-circuiting, some transformations might be prevented + * @return JsonElementStreamer + */ public getJsonElementStreamer(name: string | null) { - // TODO: add stream support + let transformed = false; + let value = this.get(name, false); + if (value instanceof JsonElementStreamer) { + return value; + } + // in case val is already an array we don't transform it to prevent evaluation of its result values + // so if is not an array, we must transform it and check after-wards (not lazy anymore) + if (!Array.isArray(value)) { + value = this.extractor.transform(value, this.resolver, true); + if (value instanceof JsonElementStreamer) { + return value; + } + transformed = true; + } + // check if initially or after transformation we got an array + if (Array.isArray(value)) { + return JsonElementStreamer.fromJsonArray(this, value, transformed); + } + return null; } public transform(definition: any, allowReturningStreams: boolean = false) { diff --git a/javascript/json-transform/src/functions/common/FunctionHelpers.ts b/javascript/json-transform/src/functions/common/FunctionHelpers.ts new file mode 100644 index 0000000..1de635d --- /dev/null +++ b/javascript/json-transform/src/functions/common/FunctionHelpers.ts @@ -0,0 +1,8 @@ +import BigNumber from "bignumber.js"; + +export const BigDecimal = BigNumber.clone({ + // We try to fit into Decimal128 which supports 34 decimal digits of precision + // so in principle, we allow 19 digits for the whole number and 15 for the fraction + ROUNDING_MODE: BigNumber.ROUND_HALF_UP, + DECIMAL_PLACES: 15 +}); diff --git a/javascript/json-transform/src/functions/common/TransformerFunction.ts b/javascript/json-transform/src/functions/common/TransformerFunction.ts index 80d0648..5b27ba0 100644 --- a/javascript/json-transform/src/functions/common/TransformerFunction.ts +++ b/javascript/json-transform/src/functions/common/TransformerFunction.ts @@ -3,6 +3,7 @@ import {ArgType} from "./ArgType"; import FunctionContext from "./FunctionContext"; import {isNullOrUndefined} from "../../JsonHelpers"; import {FunctionDescription} from "./FunctionDescription"; +import {BigDecimal} from "./FunctionHelpers"; /** * Base class for all transformer functions. @@ -30,12 +31,12 @@ class TransformerFunction { private static getDefaultValue(a: ArgumentType) { if (a == null || a.defaultIsNull) return null; switch (a.type) { - case ArgType.Boolean: return a.defaultBoolean; - case ArgType.String: return a.defaultString; - case ArgType.Enum: return a.defaultEnum; - case ArgType.Integer: return a.defaultInteger; - case ArgType.Long: return a.defaultLong; - case ArgType.BigDecimal: return a.defaultBigDecimal; + case ArgType.Boolean: return a.defaultBoolean ?? null; + case ArgType.String: return a.defaultString ?? null; + case ArgType.Enum: return a.defaultEnum ?? null; + case ArgType.Integer: return a.defaultInteger ?? null; + case ArgType.Long: return a.defaultLong ?? null; + case ArgType.BigDecimal: return a.defaultBigDecimal ? new BigDecimal(a.defaultBigDecimal) : null; } return null; } diff --git a/javascript/json-transform/src/transformerFunctions.ts b/javascript/json-transform/src/transformerFunctions.ts index f01cbaf..2b2e11f 100644 --- a/javascript/json-transform/src/transformerFunctions.ts +++ b/javascript/json-transform/src/transformerFunctions.ts @@ -8,8 +8,11 @@ import TransformerFunctionAnd from "./functions/TransformerFunctionAnd"; import TransformerFunctionAt from "./functions/TransformerFunctionAt"; import TransformerFunctionAvg from "./functions/TransformerFunctionAvg"; import TransformerFunctionBase64 from "./functions/TransformerFunctionBase64"; +import TransformerFunctionBoolean from "./functions/TransformerFunctionBoolean"; + import TransformerFunctionLower from "./functions/TransformerFunctionLower"; import TransformerFunctionUpper from "./functions/TransformerFunctionUpper"; +import TransformerFunctionIs from "./functions/TransformerFunctionIs"; class FunctionMatchResult { private result; @@ -43,7 +46,7 @@ export class TransformerFunctions { "at": new TransformerFunctionAt(), "avg": new TransformerFunctionAvg(), "base64": new TransformerFunctionBase64(), - "boolean": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionBoolean(), + "boolean": new TransformerFunctionBoolean(), "coalesce": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionCoalesce(), "concat": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionConcat(), "contains": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionContains(), @@ -64,7 +67,7 @@ export class TransformerFunctions { "formparse": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionFormParse(), "group": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionGroup(), "if": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionIf(), - "is": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionIs(), + "is": new TransformerFunctionIs(), "isnull": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionIsNull(), "join": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionJoin(), "json": new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionJsonParse(),