diff --git a/javascript/json-transform/src/JsonHelpers.ts b/javascript/json-transform/src/JsonHelpers.ts index 69c3ab9..978abff 100644 --- a/javascript/json-transform/src/JsonHelpers.ts +++ b/javascript/json-transform/src/JsonHelpers.ts @@ -2,6 +2,8 @@ import { v4 as uuidv4 } from "uuid"; import DocumentContext from "./DocumentContext"; import { BigDecimal } from "./functions/common/FunctionHelpers"; import { areSimilar } from "@nlighten/json-schema-utils"; +import { Comparator, ComparatorFactory } from "@wortise/sequency"; +import CompareBy from "./functions/common/CompareBy"; const JSONPATH_ROOT = "$", JSONPATH_ROOT_ESC = "\\$", @@ -34,17 +36,17 @@ const numberCompare = (a: number, b: number) => { return a < b ? -1 : a === b ? 0 : 1; }; -const numberType = (a: any) => typeof a === "number" || typeof a === "bigint" || a instanceof BigDecimal; +const isNumberType = (a: any) => typeof a === "number" || typeof a === "bigint" || a instanceof BigDecimal; const compareTo = (a: any, b: any) => { if (Array.isArray(a) && Array.isArray(b)) { return numberCompare(a.length, b.length); + } else if (isNumberType(a) && isNumberType(b)) { + return BigDecimal(a).comparedTo(BigDecimal(b)); } else if (a && b && typeof a === "object" && typeof b === "object") { return numberCompare(Object.keys(a).length, Object.keys(b).length); } else if (typeof a === "string" && typeof b === "string") { return a.localeCompare(b); - } else if (numberType(a) && numberType(b)) { - return BigDecimal(a).comparedTo(BigDecimal(b)); } else if (typeof a === "boolean" && typeof b === "boolean") { return a === b ? 0 : a ? 1 : -1; } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) { @@ -184,7 +186,7 @@ const isEqual = (value: any, other: any): boolean => { if (value === other) { return true; } - if (numberType(value) && numberType(other)) { + if (isNumberType(value) && isNumberType(other)) { return BigDecimal(value).eq(BigDecimal(other)); } return areSimilar(value, other); @@ -301,12 +303,58 @@ function mergeInto(rootEl: Record, value: any, path: string | null) return root; } +const factory = new ComparatorFactory(); + +function createComparator(type: string | null) { + let comparator: Comparator; + if (isNullOrUndefined(type) || "AUTO" === type.toUpperCase()) { + comparator = factory.compare((a, b) => compareTo(a, b) ?? 0); + } else { + switch (type.toUpperCase()) { + case "NUMBER": { + comparator = factory.compare((a, b) => { + if ((isNumberType(a) || typeof a === "string") && (isNumberType(b) || typeof b === "string")) { + return BigDecimal(a).comparedTo(BigDecimal(b)); + } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) { + return -1; + } else if (!isNullOrUndefined(a) && isNullOrUndefined(b)) { + return 1; + } + return 0; + }); + break; + } + case "BOOLEAN": { + comparator = factory.compare((a, b) => { + if (typeof a === "boolean" && typeof b === "boolean") { + return a === b ? 0 : a ? 1 : -1; + } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) { + return -1; + } else if (!isNullOrUndefined(a) && isNullOrUndefined(b)) { + return 1; + } + return 0; + }); + break; + } + //case "STRING" + default: { + comparator = factory.compareBy(getAsString); + break; + } + } + } + return comparator; +} + export { isNullOrUndefined, isMap, getAsString, createPayloadResolver, + isNumberType, compareTo, + createComparator, getDocumentContext, lenientJsonParse, isTruthy, diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionMap.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMap.test.ts new file mode 100644 index 0000000..508cd48 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMap.test.ts @@ -0,0 +1,83 @@ +import { describe, test } from "vitest"; +import { assertTransformation } from "../BaseTransformationTest"; + +describe("TransformerFunctionMap", () => { + test("testObjectFunctionMap", async () => { + const source = { + value: 5, + item: { + foo: "aaa", + id: "bbb", + }, + items: [ + { foo: "bar", id: "aaa" }, + { foo: "bar2", id: "bbb" }, + ], + }; + await assertTransformation( + source, + { + $$map: [ + "$.items", + { + id: "##current.id", + map_foo: "##current.foo", + idx: "##index", + value: "$.value", + }, + ], + }, + [ + { id: "aaa", map_foo: "bar", idx: 0, value: 5 }, + { id: "bbb", map_foo: "bar2", idx: 1, value: 5 }, + ], + ); + + await assertTransformation( + source, + { + $$map: "$.items", + to: { + id: "##current.id", + map_foo: "##current.foo", + idx: "##index", + value: "$.value", + }, + }, + [ + { id: "aaa", map_foo: "bar", idx: 0, value: 5 }, + { id: "bbb", map_foo: "bar2", idx: 1, value: 5 }, + ], + ); + + var valueLookup = { + $$map: [ + "$.item", + { + id: "##current.id", + map_foo: "##current.foo", + idx: "##index", + value: "$.value", + }, + ], + }; + await assertTransformation(source, valueLookup, null); + }); + + test("objectNonTransformed", async () => { + await assertTransformation( + { + a: [1, 2], + b: [2, 4], + }, + { + $$map: [["$.a", "$.b"], "##current[1]"], + }, + [2, 4], + ); + }); + + test("inline", async () => { + await assertTransformation([{ a: 10 }, { a: 11 }, { a: 12 }], "$$map(##current.a):$", [10, 11, 12]); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionMatch.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMatch.test.ts new file mode 100644 index 0000000..b5d0f11 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMatch.test.ts @@ -0,0 +1,17 @@ +import { describe, test } from "vitest"; +import { assertTransformation } from "../BaseTransformationTest"; + +describe("TransformerFunctionMatch", () => { + test("inline", async () => { + const input = "hello"; + await assertTransformation(input, "$$match([le]):$", "e"); + await assertTransformation(input, "$$match([le]+):$", "ell"); + await assertTransformation(input, "$$match(hell):$", "hell"); + await assertTransformation(input, "$$match(hello$):$", "hello"); + await assertTransformation(input, "$$match(hell$):$", null); + }); + + test("inlineGroup", async () => { + await assertTransformation("world", "$$match('w(\\\\w+)d',1):$", "orl"); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionMatchAll.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMatchAll.test.ts new file mode 100644 index 0000000..6b6fb96 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMatchAll.test.ts @@ -0,0 +1,17 @@ +import { describe, test } from "vitest"; +import { assertTransformation } from "../BaseTransformationTest"; + +describe("TransformerFunctionMatchAll", () => { + test("inline", async () => { + const input = "hello my helloKitty"; + await assertTransformation(input, "$$matchall([el]):$", ["e", "l", "l", "e", "l", "l"]); + await assertTransformation(input, "$$matchall([le]+):$", ["ell", "ell"]); + await assertTransformation(input, "$$matchall(hell):$", ["hell", "hell"]); + await assertTransformation(input, "$$matchall(^hello):$", ["hello"]); + await assertTransformation(input, "$$matchall(hello$):$", null); + }); + + test("inlineGroup", async () => { + await assertTransformation("world to waterWorld", "$$matchall('w(\\\\w+)d',1):$", ["orl", "aterWorl"]); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionMath.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMath.test.ts new file mode 100644 index 0000000..2c47a8f --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMath.test.ts @@ -0,0 +1,214 @@ +import { describe, test } from "vitest"; +import { assertTransformation } from "../BaseTransformationTest"; +import { BigDecimal } from "../../functions/common/FunctionHelpers"; + +describe("TransformerFunctionMath", () => { + test("inline", async () => { + const arr = [BigDecimal(4), BigDecimal(2)]; + await assertTransformation(arr, "$$math(+,$[0],$[1])", BigDecimal(6)); + await assertTransformation(arr, "$$math($[0],+,$[1])", BigDecimal(6)); + await assertTransformation(arr, "$$math($[0],+)", arr[0]); + await assertTransformation(arr, "$$math($[0],-,$[1])", BigDecimal(2)); + await assertTransformation(arr, "$$math($[0],*,$[1])", BigDecimal(8)); + await assertTransformation(arr, "$$math($[0],/,$[1])", BigDecimal(2)); + await assertTransformation(arr, "$$math($[0],//,3)", BigDecimal(1)); + await assertTransformation(arr, "$$math($[0],%,3)", BigDecimal(1)); + await assertTransformation(arr, "$$math($[1],^,3)", BigDecimal(8)); + await assertTransformation(arr, "$$math(3,&,1)", BigDecimal(1)); + await assertTransformation(arr, "$$math(6,AND,3)", BigDecimal(2)); + await assertTransformation(arr, "$$math(6,|,3)", BigDecimal(7)); + await assertTransformation(arr, "$$math(6,~,3)", BigDecimal(5)); + await assertTransformation(arr, "$$math(6,>>,1)", BigDecimal(3)); + await assertTransformation(arr, "$$math(6,<<,3)", BigDecimal(48)); + await assertTransformation(arr, "$$math(MIN,$[0],$[1])", BigDecimal(2)); + await assertTransformation(arr, "$$math(MAX,$[0],$[1])", BigDecimal(4)); + await assertTransformation(arr, "$$math(SQRT,81)", BigDecimal(9)); + await assertTransformation(arr, "$$math(SQRT):81", BigDecimal(9)); + await assertTransformation(arr, "$$math(SQRT):$$math($[0],^,2)", arr[0]); + await assertTransformation(arr, "$$math(ROUND,4.6)", BigDecimal(5)); + await assertTransformation(arr, "$$math(ROUND):4.6", BigDecimal(5)); + await assertTransformation(arr, "$$math(ROUND,1):4.66", BigDecimal(4.7)); + await assertTransformation(arr, "$$math(ROUND,4.66,1)", BigDecimal(4.7)); + await assertTransformation(arr, "$$math(4.66,ROUND,1)", BigDecimal(4.7)); + await assertTransformation(arr, "$$math(FLOOR,4.6)", BigDecimal(4)); + await assertTransformation(arr, "$$math(FLOOR):4.6", BigDecimal(4)); + await assertTransformation(arr, "$$math(FLOOR,1):4.66", BigDecimal(4.6)); + await assertTransformation(arr, "$$math(CEIL,4.2)", BigDecimal(5)); + await assertTransformation(arr, "$$math(CEIL):4.2", BigDecimal(5)); + await assertTransformation(arr, "$$math(CEIL,1):4.22", BigDecimal(4.3)); + await assertTransformation(arr, "$$math(ABS,-10)", BigDecimal(10)); + await assertTransformation(arr, "$$math(ABS):-10", BigDecimal(10)); + await assertTransformation(arr, "$$math(NEG,$[0])", BigDecimal(-4)); + await assertTransformation(arr, "$$math(NEG):$[0]", BigDecimal(-4)); + await assertTransformation(arr, "$$math(SIG):42", BigDecimal(1)); + await assertTransformation(arr, "$$math(SIGNUM):-42", BigDecimal(-1)); + await assertTransformation(arr, "$$math(SIG):0", BigDecimal(0)); + }); + + test("object", async () => { + const arr = [BigDecimal(4), new BigDecimal(2)]; + await assertTransformation( + arr, + { + $$math: ["+", "$[0]", "$[1]"], + }, + BigDecimal(6), + ); + await assertTransformation( + arr, + { + $$math: ["$[0]", "+", "$[1]"], + }, + BigDecimal(6), + ); + await assertTransformation( + arr, + { + $$math: ["-", "$[0]", "$[1]"], + }, + BigDecimal(2), + ); + await assertTransformation( + arr, + { + $$math: ["*", "$[0]", "$[1]"], + }, + BigDecimal(8), + ); + await assertTransformation( + arr, + { + $$math: ["/", "$[0]", "$[1]"], + }, + BigDecimal(2), + ); + await assertTransformation( + arr, + { + $$math: ["//", "$[0]", "3"], + }, + BigDecimal(1), + ); + await assertTransformation( + arr, + { + $$math: ["$[0]", "//", 3], + }, + BigDecimal(1), + ); + await assertTransformation( + arr, + { + $$math: ["%", "$[0]", "3"], + }, + BigDecimal(1), + ); + await assertTransformation( + arr, + { + $$math: ["^", "$[1]", "3"], + }, + BigDecimal(8), + ); + + await assertTransformation(arr, { $$math: [3, "&", 1] }, BigDecimal(1)); + await assertTransformation(arr, { $$math: [6, "&", 3] }, BigDecimal(2)); + await assertTransformation(arr, { $$math: [6, "|", 3] }, BigDecimal(7)); + await assertTransformation(arr, { $$math: [6, "XOR", 3] }, BigDecimal(5)); + await assertTransformation(arr, { $$math: [6, ">>", 1] }, BigDecimal(3)); + await assertTransformation(arr, { $$math: [6, "<<", 3] }, BigDecimal(48)); + await assertTransformation( + arr, + { + $$math: ["sqrt", 81], + }, + BigDecimal(9), + ); + await assertTransformation( + arr, + { + $$math: ["min", "$[0]", "$[1]"], + }, + BigDecimal(2), + ); + await assertTransformation( + arr, + { + $$math: ["max", "$[0]", "$[1]"], + }, + BigDecimal(4), + ); + await assertTransformation( + arr, + { + $$math: ["round", 4.6], + }, + BigDecimal(5), + ); + await assertTransformation( + arr, + { + $$math: ["round", 4.66, 1], + }, + BigDecimal(4.7), + ); + await assertTransformation( + arr, + { + $$math: ["floor", 4.6], + }, + BigDecimal(4), + ); + await assertTransformation( + arr, + { + $$math: ["floor", 4.66, 1], + }, + BigDecimal(4.6), + ); + await assertTransformation( + arr, + { + $$math: ["ceil", 4.2], + }, + BigDecimal(5), + ); + await assertTransformation( + arr, + { + $$math: ["ceil", 4.22, 1], + }, + BigDecimal(4.3), + ); + await assertTransformation( + arr, + { + $$math: ["abs", -10], + }, + BigDecimal(10), + ); + await assertTransformation( + arr, + { + $$math: ["neg", "$[0]"], + }, + BigDecimal(-4), + ); + await assertTransformation( + ["abs", -10], + { + $$math: "$", + }, + BigDecimal(10), + ); + + await assertTransformation(arr, { $$math: ["SIG", 42] }, BigDecimal(1)); + await assertTransformation(arr, { $$math: ["SIGNUM", -42] }, BigDecimal(-1)); + await assertTransformation(arr, { $$math: ["SIG", 0] }, BigDecimal(0)); + }); + + test("combineScaling", async () => { + await assertTransformation(null, "$$decimal(2):$$math(1,*,0.987654321)", BigDecimal(0.99)); + await assertTransformation(null, "$$decimal(2,FLOOR):$$math(1,*,0.987654321)", BigDecimal(0.98)); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionMax.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMax.test.ts new file mode 100644 index 0000000..b5a53d5 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMax.test.ts @@ -0,0 +1,49 @@ +import type BigNumber from "bignumber.js"; +import { describe, test } from "vitest"; +import { assertTransformation } from "../BaseTransformationTest"; +import { BigDecimal } from "../../functions/common/FunctionHelpers"; + +describe("TransformerFunctionMax", () => { + test("inline", async () => { + const arr = [4, -2, 13.45, null]; + await assertTransformation(arr, "$$max($$long:40):$", BigInt(40)); + await assertTransformation(arr, "$$max(-8,NUMBER):$", 13.45); + await assertTransformation(arr, "$$max():$", 13.45); + await assertTransformation(arr, "$$max(z,STRING):$", "z"); + }); + + class Holder { + public value: BigNumber | null; + constructor(value: BigNumber | null) { + this.value = value; + } + } + + test("object", async () => { + const arr = [ + new Holder(BigDecimal(4)), + new Holder(BigDecimal(2)), + new Holder(BigDecimal("13.45")), + new Holder(null), + ]; + await assertTransformation( + arr, + { + $$max: "$", + by: "##current.value", + default: "zz", + type: "STRING", + }, + "zz", + ); + + await assertTransformation( + arr, + { + $$max: "$", + by: "##current.value", + }, + BigDecimal("13.45"), + ); + }); +}); diff --git a/javascript/json-transform/src/__tests__/functions/TransformerFunctionMin.test.ts b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMin.test.ts new file mode 100644 index 0000000..ff4eb24 --- /dev/null +++ b/javascript/json-transform/src/__tests__/functions/TransformerFunctionMin.test.ts @@ -0,0 +1,48 @@ +import type BigNumber from "bignumber.js"; +import { describe, test } from "vitest"; +import { assertTransformation } from "../BaseTransformationTest"; +import { BigDecimal } from "../../functions/common/FunctionHelpers"; + +describe("TransformerFunctionMin", () => { + test("inline", async () => { + const arr = [4, -2, 13.45, null]; + await assertTransformation(arr, "$$min($$long:-4):$", BigInt(-4)); + await assertTransformation(arr, "$$min(-8,NUMBER):$", "-8"); + await assertTransformation(arr, "$$min():$", null); + await assertTransformation(arr, "$$min(z,STRING):$", -2); + }); + + class Holder { + public value: BigNumber | null; + constructor(value: BigNumber | null) { + this.value = value; + } + } + + test("object", async () => { + const arr = [ + new Holder(BigDecimal(4)), + new Holder(BigDecimal(2)), + new Holder(BigDecimal("13.45")), + new Holder(null), + ]; + await assertTransformation( + arr, + { + $$min: "$", + by: "##current.value", + }, + null, + ); + + await assertTransformation( + arr, + { + $$min: "$", + by: "##current.value", + default: 1, + }, + 1, + ); + }); +}); diff --git a/javascript/json-transform/src/functions/TransformerFunctionLength.ts b/javascript/json-transform/src/functions/TransformerFunctionLength.ts index 8048c5a..c897b24 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionLength.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionLength.ts @@ -2,9 +2,7 @@ import TransformerFunction from "./common/TransformerFunction"; import { ArgType } from "./common/ArgType"; import { FunctionDescription } from "./common/FunctionDescription"; import FunctionContext from "./common/FunctionContext"; -import Base64 from "./utils/Base64"; -import TextEncoding from "./common/TextEncoding"; -import { getAsString, isMap } from "../JsonHelpers"; +import { isMap } from "../JsonHelpers"; import JsonElementStreamer from "../JsonElementStreamer"; const DESCRIPTION: FunctionDescription = { diff --git a/javascript/json-transform/src/functions/TransformerFunctionLookup.ts b/javascript/json-transform/src/functions/TransformerFunctionLookup.ts index 8b59ce3..7454ec4 100644 --- a/javascript/json-transform/src/functions/TransformerFunctionLookup.ts +++ b/javascript/json-transform/src/functions/TransformerFunctionLookup.ts @@ -14,8 +14,22 @@ type UsingEntry = { const DESCRIPTION: FunctionDescription = { aliases: ["lookup"], description: "", - inputType: ArgType.Any, - outputType: ArgType.Long, + inputType: ArgType.Array, + arguments: { + using: { + type: ArgType.Array, + position: 0, + required: true, + description: "Array of definitions of how to match other arrays to the main one", + }, + to: { + type: ArgType.Transformer, + position: 1, + defaultIsNull: true, + description: "Transformer to map each pair of elements to its value in the result array", + }, + }, + outputType: ArgType.Array, }; class TransformerFunctionLookup extends TransformerFunction { constructor() { diff --git a/javascript/json-transform/src/functions/TransformerFunctionMap.ts b/javascript/json-transform/src/functions/TransformerFunctionMap.ts new file mode 100644 index 0000000..3850334 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionMap.ts @@ -0,0 +1,55 @@ +import { AsyncSequence, asAsyncSequence } from "@wortise/sequency"; +import TransformerFunction from "./common/TransformerFunction"; +import { ArgType } from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import { FunctionDescription } from "./common/FunctionDescription"; +import JsonElementStreamer from "../JsonElementStreamer"; + +const DESCRIPTION: FunctionDescription = { + aliases: ["map"], + description: "", + inputType: ArgType.Array, + arguments: { + to: { + type: ArgType.Transformer, + position: 0, + defaultIsNull: true, + description: "Transformer to map each element to its value in the result array (inputs: ##current, ##index)", + }, + }, + outputType: ArgType.Array, +}; +class TransformerFunctionMap extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override async apply(context: FunctionContext): Promise { + let inputStream: AsyncSequence; + let to: any; + if (context.has("to")) { + const streamer = await context.getJsonElementStreamer(null); + if (streamer == null) return null; + inputStream = streamer.stream(); + to = await context.getJsonElement("to", false); // we don't transform definitions to prevent premature evaluation + } else { + // [ input, to ] + const arr = await context.getJsonArray(null, false); // we don't transform definitions to prevent premature evaluation + if (arr == null) return null; + const inputEl = await context.transform(arr[0]); + if (!Array.isArray(inputEl)) { + console.warn(`${context.getAlias()} was not specified with an array of items`); + return null; + } + inputStream = asAsyncSequence(inputEl); + to = arr[1]; + } + let i = 0; + return JsonElementStreamer.fromTransformedStream( + context, + inputStream.map(x => context.transformItem(to, x, i++)), + ); + } +} + +export default TransformerFunctionMap; diff --git a/javascript/json-transform/src/functions/TransformerFunctionMatch.ts b/javascript/json-transform/src/functions/TransformerFunctionMatch.ts new file mode 100644 index 0000000..f874951 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionMatch.ts @@ -0,0 +1,43 @@ +import TransformerFunction from "./common/TransformerFunction"; +import { ArgType } from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import { FunctionDescription } from "./common/FunctionDescription"; + +const DESCRIPTION: FunctionDescription = { + aliases: ["match"], + description: "", + inputType: ArgType.Object, + arguments: { + pattern: { + type: ArgType.String, + position: 0, + required: true, + description: "Regular expression to match and extract from input string", + }, + group: { type: ArgType.Integer, position: 1, defaultInteger: 0, description: "The group id to get" }, + }, + outputType: ArgType.String, +}; +class TransformerFunctionMatch extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override async apply(context: FunctionContext): Promise { + const str = await context.getString(null); + if (str == null) { + return null; + } + const patternString = await context.getString("pattern"); + if (patternString == null) { + return null; + } + const matcher = new RegExp(patternString); + const result = str.match(matcher); + if (!result) return null; // not found + const group = await context.getInteger("group"); + return result[group ?? 0]; + } +} + +export default TransformerFunctionMatch; diff --git a/javascript/json-transform/src/functions/TransformerFunctionMatchAll.ts b/javascript/json-transform/src/functions/TransformerFunctionMatchAll.ts new file mode 100644 index 0000000..d15c3b2 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionMatchAll.ts @@ -0,0 +1,46 @@ +import TransformerFunction from "./common/TransformerFunction"; +import { ArgType } from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import { FunctionDescription } from "./common/FunctionDescription"; + +const DESCRIPTION: FunctionDescription = { + aliases: ["matchall"], + description: "", + inputType: ArgType.Object, + arguments: { + pattern: { + type: ArgType.String, + position: 0, + required: true, + description: "Regular expression to match and extract from input string", + }, + group: { type: ArgType.Integer, position: 1, defaultInteger: 0, description: "The group id to get" }, + }, + outputType: ArgType.ArrayOfString, +}; +class TransformerFunctionMatchAll extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override async apply(context: FunctionContext): Promise { + const str = await context.getString(null); + if (str == null) { + return null; + } + const patternString = await context.getString("pattern"); + if (patternString == null) { + return null; + } + const matcher = new RegExp(patternString, "g"); + + const allMatches: string[] = []; + const group = await context.getInteger("group"); + for (const match of str.matchAll(matcher)) { + allMatches.push(match[group ?? 0]); + } + return allMatches.length == 0 ? null : allMatches; + } +} + +export default TransformerFunctionMatchAll; diff --git a/javascript/json-transform/src/functions/TransformerFunctionMath.ts b/javascript/json-transform/src/functions/TransformerFunctionMath.ts new file mode 100644 index 0000000..745a050 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionMath.ts @@ -0,0 +1,282 @@ +import TransformerFunction from "./common/TransformerFunction"; +import { ArgType } from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import { FunctionDescription } from "./common/FunctionDescription"; +import { getAsString, isNullOrUndefined } from "../JsonHelpers"; +import { BigDecimal, MAX_SCALE, MAX_SCALE_ROUNDING } from "./common/FunctionHelpers"; +import BigNumber from "bignumber.js"; + +enum MathOp { + ADDITION = 1, + SUBTRACTION, + MULTIPLICATION, + DIVISION, + INTEGER_DIVISION, + MODULU, + POWER, + SQUARE_ROOT, + MIN, + MAX, + ROUND, + FLOOR, + CEIL, + ABSOLUTE, + NEGATION, + SIGNUM, + BITAND, + BITOR, + BITXOR, + SHIFT_LEFT, + SHIFT_RIGHT, + UNKNOWN, +} + +const BigZero = BigDecimal(0); + +const toBigInt = (value: BigNumber) => BigInt(BigDecimal(value).toFixed(0, BigNumber.ROUND_DOWN)); + +const DESCRIPTION: FunctionDescription = { + aliases: ["math"], + description: "", + inputType: ArgType.Any, + arguments: { + op1: { + type: ArgType.BigDecimal, + position: 0 /* or 1 */, + defaultBigDecimal: 0, + required: true, + description: "First operand", + }, + op: { + type: ArgType.Enum, + position: 1 /* or 0 */, + defaultEnum: "0", + required: true, + enumValues: [ + "+", + "-", + "*", + "/", + "//", + "%", + "^", + "&", + "|", + "~", + "<<", + ">>", + "MIN", + "MAX", + "SQRT", + "ROUND", + "FLOOR", + "CEIL", + "ABS", + "NEG", + "SIG", + ], + description: "", + }, + op2: { + type: ArgType.BigDecimal, + position: 2, + defaultBigDecimal: 0, + description: "Second operand or scale for ROUND/FLOOR/CEIL", + }, + }, + outputType: ArgType.BigDecimal, +}; +class TransformerFunctionMath extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override async apply(context: FunctionContext): Promise { + var value = await context.getJsonArray(null); + let parsedOp: string | null = null; + let op: MathOp; + let op1: BigNumber | null = null; + let op2: BigNumber | null = null; + if (value != null) { + const size = value.length; + if (size <= 1) return null; // invalid input + const arg0 = value[0]; + const arg1 = value[1]; + parsedOp = getAsString(arg0); + op = TransformerFunctionMath.parseMathOp(parsedOp); + if (size > 2 && op == MathOp.UNKNOWN) { + parsedOp = getAsString(arg1); + op = TransformerFunctionMath.parseMathOp(parsedOp); + op1 = BigDecimal(arg0); + } else { + op1 = BigDecimal(arg1); + } + op2 = size < 3 ? BigDecimal(0) : BigDecimal(value[2]); + } else { + // order of arguments ( op1, op, op2 ) + parsedOp = await context.getEnum("op1"); + op = TransformerFunctionMath.parseMathOp(parsedOp); + if (op == MathOp.UNKNOWN) { + // op was not detected as the first argument, so we assume it is in the second argument + // -> op1, op, [op2] + parsedOp = await context.getEnum("op"); + op = TransformerFunctionMath.parseMathOp(parsedOp); + op1 = await context.getBigDecimal("op1"); + op2 = await context.getBigDecimal("op2"); + } else { + var mainArgValue = await context.getUnwrapped(null); + if (mainArgValue != null) { + // we set operand 1 as main argument value for the sake of functions with only one operand + // -> op, [op2] : op1 + op1 = BigDecimal(mainArgValue.toString()); + op2 = await context.getBigDecimal("op"); + } else { + // -> op, op1, op2 + op1 = await context.getBigDecimal("op"); + op2 = await context.getBigDecimal("op2"); + } + } + } + + if (op == MathOp.UNKNOWN) { + console.warn("{} was specified with an unknown op ({})", context.getAlias(), parsedOp); + return null; + } + var result = TransformerFunctionMath.eval(op, op1 ?? BigZero, op2 ?? BigZero); + + if (result == null) { + return null; + } + // cap scale at max + if ((result.decimalPlaces() ?? 0) > MAX_SCALE) { + result = result.decimalPlaces(MAX_SCALE, MAX_SCALE_ROUNDING); + } + return result; + } + + static parseMathOp(value: string | null): MathOp { + switch (value?.toUpperCase()) { + case "+": + case "ADD": + return MathOp.ADDITION; + case "-": + case "SUB": + case "SUBTRACT": + return MathOp.SUBTRACTION; + case "*": + case "MUL": + case "MULTIPLY": + return MathOp.MULTIPLICATION; + case "/": + case "DIV": + case "DIVIDE": + return MathOp.DIVISION; + case "//": + case "INTDIV": + return MathOp.INTEGER_DIVISION; + case "%": + case "MOD": + case "REMAINDER": + return MathOp.MODULU; + case "^": + case "**": + case "POW": + case "POWER": + return MathOp.POWER; + case "&": + case "AND": + return MathOp.BITAND; + case "|": + case "OR": + return MathOp.BITOR; + case "~": + case "XOR": + return MathOp.BITXOR; + case "<<": + case "SHL": + return MathOp.SHIFT_LEFT; + case ">>": + case "SHR": + return MathOp.SHIFT_RIGHT; + case "MIN": + return MathOp.MIN; + case "MAX": + return MathOp.MAX; + case "SQRT": + return MathOp.SQUARE_ROOT; + case "ROUND": + return MathOp.ROUND; + case "FLOOR": + return MathOp.FLOOR; + case "CEIL": + return MathOp.CEIL; + case "ABS": + return MathOp.ABSOLUTE; + case "NEG": + case "NEGATE": + return MathOp.NEGATION; + case "SIG": + case "SIGNUM": + return MathOp.SIGNUM; + default: + return MathOp.UNKNOWN; + } + } + + static eval(op: MathOp, op1: BigNumber, op2: BigNumber): BigNumber | null { + switch (op) { + // 2 operands + case MathOp.ADDITION: + return op1.plus(op2); + case MathOp.SUBTRACTION: + return op1.minus(op2); + case MathOp.MULTIPLICATION: + return op1.multipliedBy(op2); + case MathOp.DIVISION: + return op1.dividedBy(op2); + case MathOp.INTEGER_DIVISION: + return op1.dividedToIntegerBy(op2); + case MathOp.MODULU: + return op1.mod(op2); + case MathOp.POWER: + return op1.pow(op2); + case MathOp.MIN: + return BigNumber.min(op1, op2); + case MathOp.MAX: + return BigNumber.max(op1, op2); + // only one operand + case MathOp.SQUARE_ROOT: + return op1.sqrt(); + case MathOp.ROUND: + return op1.decimalPlaces(op2.integerValue().toNumber(), BigNumber.ROUND_HALF_UP); + case MathOp.FLOOR: + return op1.decimalPlaces(op2.integerValue().toNumber(), BigNumber.ROUND_FLOOR); + case MathOp.CEIL: + return op1.decimalPlaces(op2.integerValue().toNumber(), BigNumber.ROUND_CEIL); + case MathOp.ABSOLUTE: + return op1.abs(); + case MathOp.NEGATION: + return op1.negated(); + case MathOp.SIGNUM: + return BigDecimal(op1.isZero() ? 0 : op1.isPositive() ? 1 : -1); + // bitwise + case MathOp.BITAND: + return BigDecimal((toBigInt(op1) & toBigInt(op2)).toString()); + case MathOp.BITOR: + return BigDecimal((toBigInt(op1) | toBigInt(op2)).toString()); + // special case where only 1 op (~x) acts as NOT (op2 acts like ~0) + case MathOp.BITXOR: + return isNullOrUndefined(op2) + ? BigDecimal(~toBigInt(op1).toString()) + : BigDecimal((toBigInt(op1) ^ toBigInt(op2)).toString()); + case MathOp.SHIFT_LEFT: + return BigDecimal((toBigInt(op1) << toBigInt(op2)).toString()); + case MathOp.SHIFT_RIGHT: + return BigDecimal((toBigInt(op1) >> toBigInt(op2)).toString()); + default: + return null; + } + } +} + +export default TransformerFunctionMath; diff --git a/javascript/json-transform/src/functions/TransformerFunctionMax.ts b/javascript/json-transform/src/functions/TransformerFunctionMax.ts new file mode 100644 index 0000000..b85d339 --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionMax.ts @@ -0,0 +1,53 @@ +import TransformerFunction from "./common/TransformerFunction"; +import { ArgType } from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import { FunctionDescription } from "./common/FunctionDescription"; +import { createComparator, isNullOrUndefined } from "../JsonHelpers"; + +const DESCRIPTION: FunctionDescription = { + aliases: ["max"], + description: "", + inputType: ArgType.Array, + arguments: { + default: { type: ArgType.Object, position: 0, description: "The default value to use for empty values" }, + by: { + type: ArgType.Transformer, + position: 2, + defaultString: "##current", + description: "A transformer to extract a property to max by (using ##current to refer to the current item)", + }, + type: { + type: ArgType.Enum, + position: 1, + defaultEnum: "AUTO", + enumValues: ["AUTO", "STRING", "NUMBER", "BOOLEAN"], + description: "Type of values to expect when ordering the input array", + }, + }, + outputType: ArgType.BigDecimal, +}; +class TransformerFunctionMax extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override async apply(context: FunctionContext): Promise { + const streamer = await context.getJsonElementStreamer(null); + if (streamer == null || streamer.knownAsEmpty()) return null; + const by = await context.getJsonElement("by", true); + + const type = await context.getEnum("type"); + + const def = await context.getJsonElement("default", true); + const comparator = createComparator(type); + return streamer + .stream() + .map(async t => { + const res = !isNullOrUndefined(by) ? await context.transformItem(by, t) : t; + return isNullOrUndefined(res) ? def : res; + }) + .maxWith(comparator.compare); + } +} + +export default TransformerFunctionMax; diff --git a/javascript/json-transform/src/functions/TransformerFunctionMin.ts b/javascript/json-transform/src/functions/TransformerFunctionMin.ts new file mode 100644 index 0000000..11334cc --- /dev/null +++ b/javascript/json-transform/src/functions/TransformerFunctionMin.ts @@ -0,0 +1,53 @@ +import TransformerFunction from "./common/TransformerFunction"; +import { ArgType } from "./common/ArgType"; +import FunctionContext from "./common/FunctionContext"; +import { FunctionDescription } from "./common/FunctionDescription"; +import { createComparator, isNullOrUndefined } from "../JsonHelpers"; + +const DESCRIPTION: FunctionDescription = { + aliases: ["min"], + description: "", + inputType: ArgType.Array, + arguments: { + default: { type: ArgType.Object, position: 0, description: "The default value to use for empty values" }, + by: { + type: ArgType.Transformer, + position: 2, + defaultString: "##current", + description: "A transformer to extract a property to min by (using ##current to refer to the current item)", + }, + type: { + type: ArgType.Enum, + position: 1, + defaultEnum: "AUTO", + enumValues: ["AUTO", "STRING", "NUMBER", "BOOLEAN"], + description: "Type of values to expect when ordering the input array", + }, + }, + outputType: ArgType.BigDecimal, +}; +class TransformerFunctionMin extends TransformerFunction { + constructor() { + super(DESCRIPTION); + } + + override async apply(context: FunctionContext): Promise { + const streamer = await context.getJsonElementStreamer(null); + if (streamer == null || streamer.knownAsEmpty()) return null; + const by = await context.getJsonElement("by", true); + + const type = await context.getEnum("type"); + + const def = await context.getJsonElement("default", true); + const comparator = createComparator(type); + return streamer + .stream() + .map(async t => { + const res = !isNullOrUndefined(by) ? await context.transformItem(by, t) : t; + return isNullOrUndefined(res) ? def : res; + }) + .minWith(comparator.compare); + } +} + +export default TransformerFunctionMin; diff --git a/javascript/json-transform/src/functions/common/CompareBy.ts b/javascript/json-transform/src/functions/common/CompareBy.ts index 07ab8ad..76c4ca9 100644 --- a/javascript/json-transform/src/functions/common/CompareBy.ts +++ b/javascript/json-transform/src/functions/common/CompareBy.ts @@ -1,5 +1,5 @@ import { Comparator, ComparatorFactory } from "@wortise/sequency"; -import { compareTo, getAsString } from "../../JsonHelpers"; +import { compareTo, getAsString, isNullOrUndefined, isNumberType } from "../../JsonHelpers"; import { BigDecimal } from "./FunctionHelpers"; const factory = new ComparatorFactory(); @@ -15,19 +15,41 @@ class CompareBy { public static createByComparator(index: number, type: string | null): Comparator { let comparator: Comparator; - if (type == null || "AUTO" === type.toUpperCase()) { - comparator = factory.compare((a: CompareBy, b: CompareBy) => { + if (isNullOrUndefined(type) || "AUTO" === type.toUpperCase()) { + comparator = factory.compare((a, b) => { const compareResult = compareTo(a.by?.[index], b.by?.[index]); return compareResult ?? 0; }); } else { switch (type.toUpperCase()) { case "NUMBER": { - comparator = factory.compareBy(tup => BigDecimal(tup.by?.[index])); + comparator = factory.compare((_a, _b) => { + const a = _a.by?.[index], + b = _b.by?.[index]; + if ((isNumberType(a) || typeof a === "string") && (isNumberType(b) || typeof b === "string")) { + return BigDecimal(a).comparedTo(BigDecimal(b)); + } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) { + return -1; + } else if (!isNullOrUndefined(a) && isNullOrUndefined(b)) { + return 1; + } + return 0; + }); break; } case "BOOLEAN": { - comparator = factory.compareBy(tup => Boolean(tup.by?.[index])); + comparator = factory.compare((_a, _b) => { + const a = _a.by?.[index], + b = _b.by?.[index]; + if (typeof a === "boolean" && typeof b === "boolean") { + return a === b ? 0 : a ? 1 : -1; + } else if (isNullOrUndefined(a) && !isNullOrUndefined(b)) { + return -1; + } else if (!isNullOrUndefined(a) && isNullOrUndefined(b)) { + return 1; + } + return 0; + }); break; } //case "STRING" diff --git a/javascript/json-transform/src/transformerFunctions.ts b/javascript/json-transform/src/transformerFunctions.ts index 7c32bcb..4b08182 100644 --- a/javascript/json-transform/src/transformerFunctions.ts +++ b/javascript/json-transform/src/transformerFunctions.ts @@ -42,6 +42,12 @@ import TransformerFunctionUpper from "./functions/TransformerFunctionUpper"; import TransformerFunctionLength from "./functions/TransformerFunctionLength"; import TransformerFunctionLong from "./functions/TransformerFunctionLong"; import TransformerFunctionLookup from "./functions/TransformerFunctionLookup"; +import TransformerFunctionMap from "./functions/TransformerFunctionMap"; +import TransformerFunctionMatch from "./functions/TransformerFunctionMatch"; +import TransformerFunctionMatchAll from "./functions/TransformerFunctionMatchAll"; +import TransformerFunctionMax from "./functions/TransformerFunctionMax"; +import TransformerFunctionMin from "./functions/TransformerFunctionMin"; +import TransformerFunctionMath from "./functions/TransformerFunctionMath"; class FunctionMatchResult { private readonly result; @@ -108,12 +114,12 @@ export class TransformerFunctions { long: new TransformerFunctionLong(), lookup: new TransformerFunctionLookup(), lower: new TransformerFunctionLower(), - map: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionMap(), - match: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionMatch(), - matchall: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionMatchAll(), - math: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionMath(), - max: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionMax(), - min: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionMin(), + map: new TransformerFunctionMap(), + match: new TransformerFunctionMatch(), + matchall: new TransformerFunctionMatchAll(), + math: new TransformerFunctionMath(), + max: new TransformerFunctionMax(), + min: new TransformerFunctionMin(), normalize: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionNormalize(), not: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionNot(), numberformat: new TransformerFunction(UNIMPLEMENTED), // TODO: new TransformerFunctionNumberFormat(),