From 678a65ced7cc05591150773965e266ec51802f1d Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 9 Sep 2024 22:37:46 -0700 Subject: [PATCH 1/5] fix: use files trailing arguments in Node --- .vscode/settings.json | 3 ++ Readme.md | 3 +- src/native/cli.d | 6 +-- src/native/lib.d | 35 ++++++++---------- src/node/lib.ts | 86 +++++++++++++++++++++---------------------- 5 files changed, 63 insertions(+), 70 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..cf459d9 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "cSpell.words": ["jsonminify", "minijson"] +} diff --git a/Readme.md b/Readme.md index e5fba89..0085008 100644 --- a/Readme.md +++ b/Readme.md @@ -24,8 +24,7 @@ dub build --config=executable --build=release-nobounds --compiler=ldc2 ``` - Download Native Binaries from -https://github.com/aminya/minijson/releases/latest - + https://github.com/aminya/minijson/releases/latest ### CLI Usage diff --git a/src/native/cli.d b/src/native/cli.d index 54ff9a3..6513420 100644 --- a/src/native/cli.d +++ b/src/native/cli.d @@ -3,8 +3,7 @@ module minijson.cli; import minijson.lib : minifyFiles, minifyStrings; import argparse; -@(Command("minijson") - .Description(`minijson: minify json files with support for comments +@(Command("minijson").Description(`minijson: minify json files with support for comments # Minify the specified files minijson ./dist/**/*.json ./build/a.json @@ -19,8 +18,7 @@ import argparse; minijson --comment --str '{"some_json": "string_here"} //comment' More information at https://github.com/aminya/minijson - `) -) + `)) struct Options { bool comment = false; diff --git a/src/native/lib.d b/src/native/lib.d index 249aa67..02ffe7e 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -202,30 +202,27 @@ void minifyFiles(in string[] paths, in bool hasComment = false) @trusted import std.stdio : writeln; // get the files from the given paths (resolve glob patterns) - auto files = paths - .map!((path) { - if (path.exists) + auto files = paths.map!((path) { + if (path.exists) + { + if (path.isFile) { - if (path.isFile) - { - return [path]; - } - else if (path.isDir) - { - return glob(path ~ "/**/*.json"); - } - else - { - throw new Exception("The given path is not a file or a directory: " ~ path); - } + return [path]; + } + else if (path.isDir) + { + return glob(path ~ "/**/*.json"); } else { - return glob(path); + throw new Exception("The given path is not a file or a directory: " ~ path); } - }) - .joiner() - .array(); + } + else + { + return glob(path); + } + }).joiner().array(); foreach (file; files.parallel()) { diff --git a/src/node/lib.ts b/src/node/lib.ts index c8553d6..c113f0c 100644 --- a/src/node/lib.ts +++ b/src/node/lib.ts @@ -12,35 +12,52 @@ import { join } from "path" * @throws {Promise} The promise is rejected with the reason for failure */ export async function minifyFiles(files: string[], hasComment = false): Promise { - if (process.platform === "darwin" && process.arch === "arm64") { - // fallback to jasonminify due to missing ARM64 binaries - // eslint-disable-next-line @typescript-eslint/no-var-requires - const jsonminify = require("jsonminify") - await Promise.all(files.map(async (file) => { - const jsonString = await readFile(file, "utf8") - const minifiedJsonString = jsonminify(jsonString) as string - await writeFile(file, minifiedJsonString) - })) - return - } + try { + const filesNum = files.length + if (filesNum === 0) { + return Promise.resolve() + } - const filesNum = files.length - if (filesNum === 0) { - return Promise.resolve() - } - - const args = [...files] - const spliceUpper = 2 * filesNum - 2 + const minijsonArgs = hasComment ? ["--comment", ...files] : files - for (let iSplice = 0; iSplice <= spliceUpper; iSplice += 2) { - args.splice(iSplice, 0, "--file") + await spawnMinijson(minijsonArgs) + } catch (e) { + console.error(e, "Falling back to jsonminify") + await minifyFilesFallback(files) } +} - if (hasComment) { - args.push("--comment") - } +/** + * Spawn minijson with the given arguments + * + * @param args An array of arguments + * @returns {Promise} Returns a promise that resolves to stdout output string when the operation finishes + * @throws {Promise} The promise is rejected with the reason for failure + */ +export function spawnMinijson(args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile(minijsonBin, args, (err, stdout, stderr) => { + if (err) { + reject(err) + } + if (stderr !== "") { + reject(stderr) + } + resolve(stdout) + }) + }) +} - await spawnMinijson(args) +async function minifyFilesFallback(files: string[]) { + const jsonminify = require("jsonminify") + await Promise.all( + files.map(async (file) => { + const jsonString = await readFile(file, "utf8") + const minifiedJsonString = jsonminify(jsonString) as string + await writeFile(file, minifiedJsonString) + }), + ) + return } /** @@ -69,24 +86,3 @@ const minijsonBin = join(__dirname, `${process.platform}-${process.arch}`, binNa if (process.platform !== "win32") { chmodSync(minijsonBin, 0o755) } - -/** - * Spawn minijson with the given arguments - * - * @param args An array of arguments - * @returns {Promise} Returns a promise that resolves to stdout output string when the operation finishes - * @throws {Promise} The promise is rejected with the reason for failure - */ -export function spawnMinijson(args: string[]): Promise { - return new Promise((resolve, reject) => { - execFile(minijsonBin, args, (err, stdout, stderr) => { - if (err) { - reject(err) - } - if (stderr !== "") { - reject(stderr) - } - resolve(stdout) - }) - }) -} From 597fcccf55c854a2556ac686fa59c1c0353b7e29 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 9 Sep 2024 23:44:23 -0700 Subject: [PATCH 2/5] fix: confirm minification of files with unknown extension --- src/native/lib.d | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/native/lib.d b/src/native/lib.d index 02ffe7e..27799fa 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -224,8 +224,42 @@ void minifyFiles(in string[] paths, in bool hasComment = false) @trusted } }).joiner().array(); + if (!confirmExtension(files)) + { + return; + } + foreach (file; files.parallel()) { write(file, minifyString(readText(file), hasComment)); } } + +bool confirmExtension(string[] files) @trusted +{ + auto confirmExtension = false; + import std.path : extension; + + foreach (file; files) + { + // if the file extension is not json, jsonc, or json5, confirm before minifying + auto fileExtension = file.extension(); + if (fileExtension != ".json" && fileExtension != ".jsonc" && fileExtension != ".json5") + { + if (!confirmExtension) + { + import std.stdio : readln, writeln; + + writeln("The file ", file, " doesn't have a json extension. Do you want to minify it? (y/n)"); + auto input = readln(); + confirmExtension = input == "y"; + if (!confirmExtension) + { + return false; + } + } + } + } + + return true; +} From 961c7d78ca51ee9d3896445160686b6587d1ef9d Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Mon, 9 Sep 2024 23:46:38 -0700 Subject: [PATCH 3/5] fix: warn on not finding any files --- src/native/lib.d | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/native/lib.d b/src/native/lib.d index 27799fa..5f869a0 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -224,6 +224,12 @@ void minifyFiles(in string[] paths, in bool hasComment = false) @trusted } }).joiner().array(); + if (files.empty) + { + writeln("No files found."); + return; + } + if (!confirmExtension(files)) { return; From 034b4e552097cd42e89c0daf2c4f3a6aa4625a4e Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 10 Sep 2024 00:20:27 -0700 Subject: [PATCH 4/5] perf: separate no-comment minifier for better performance --- package.json | 1 + src/native/lib.d | 67 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 0c3b531..e5a2173 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "format.d": "dub run --build=release --quiet dfmt -- --soft_max_line_length 110 --indent_size 2 --inplace ./src ./benchmark", "lint": "eslint . --fix", "prepublishOnly": "shx rm -rf ./dist/tsconfig.tsbuildinfo ./dist/*.zip ./dist/build.* && chmod +x ./dist/*/minijson", + "start.benchmark.js": "node ./benchmark/js-benchmark.mjs", "start.benchmark.node": "node ./benchmark/native-benchmark.mjs", "start.browser": "servor ./dist/ --browse --reload", "start.node": "node ./dist/node/cli.js", diff --git a/src/native/lib.d b/src/native/lib.d index 5f869a0..2b0939c 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -18,6 +18,60 @@ const tokenizerNoComment = ctRegex!(`[\n\r"[]]`, "g"); the minified json string */ string minifyString(in string jsonString, in bool hasComment = false) @trusted +{ + return hasComment ? minifyStringWithComments(jsonString) : minifyStringNoComments(jsonString); +} + +string minifyStringNoComments(in string jsonString) @trusted +{ + auto in_string = false; + string result; + size_t from = 0; + auto rightContext = ""; + + auto match = jsonString.matchAll(tokenizerNoComment); + + while (!match.empty()) + { + const matchFrontHit = match.front().hit(); + + rightContext = match.post(); + + // update from for the next iteration + const prevFrom = from; + from = jsonString.length - rightContext.length; // lastIndex + + auto leftContextSubstr = match.pre()[prevFrom .. $]; + const noLeftContext = leftContextSubstr.length == 0; + if (!noLeftContext) + { + if (!in_string) + { + leftContextSubstr = remove_spaces(leftContextSubstr); + } + result ~= leftContextSubstr; + } + if (matchFrontHit == "\"") + { + if (!in_string || noLeftContext || hasNoSlashOrEvenNumberOfSlashes(leftContextSubstr)) + { + // start of string with ", or unescaped " character found to end string + in_string = !in_string; + } + --from; // include " character in next catch + rightContext = jsonString[from .. $]; + } + else if (notSlashAndNoSpaceOrBreak(matchFrontHit)) + { + result ~= matchFrontHit; + } + match.popFront(); + } + result ~= rightContext; + return result; +} + +string minifyStringWithComments(in string jsonString) @trusted { auto in_string = false; auto in_multiline_comment = false; @@ -26,9 +80,7 @@ string minifyString(in string jsonString, in bool hasComment = false) @trusted size_t from = 0; auto rightContext = ""; - const tokenizer = !hasComment ? tokenizerNoComment : tokenizerWithComment; - - auto match = jsonString.matchAll(tokenizer); + auto match = jsonString.matchAll(tokenizerWithComment); while (!match.empty()) { @@ -40,10 +92,9 @@ string minifyString(in string jsonString, in bool hasComment = false) @trusted const prevFrom = from; from = jsonString.length - rightContext.length; // lastIndex - const notInComment = (!in_multiline_comment && !in_singleline_comment); - const noCommentOrNotInComment = !hasComment || notInComment; + const notInComment = !in_multiline_comment && !in_singleline_comment; - if (noCommentOrNotInComment) + if (notInComment) { auto leftContextSubstr = match.pre()[prevFrom .. $]; const noLeftContext = leftContextSubstr.length == 0; @@ -67,7 +118,7 @@ string minifyString(in string jsonString, in bool hasComment = false) @trusted } } // comments - if (hasComment && !in_string) + if (!in_string) { if (notInComment) { @@ -93,7 +144,7 @@ string minifyString(in string jsonString, in bool hasComment = false) @trusted in_singleline_comment = false; } } - else if (!hasComment && notSlashAndNoSpaceOrBreak(matchFrontHit)) + else if (notSlashAndNoSpaceOrBreak(matchFrontHit)) { result ~= matchFrontHit; } From b5f68d26ceaeb2318d66044f54e0c349e121ca95 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 10 Sep 2024 00:33:32 -0700 Subject: [PATCH 5/5] feat: enable comment support by default --- src/native/cli.d | 11 +++++------ src/native/lib.d | 8 ++++---- src/native/libc.d | 4 ++-- src/node/lib.ts | 14 ++++++-------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/native/cli.d b/src/native/cli.d index 6513420..0faffde 100644 --- a/src/native/cli.d +++ b/src/native/cli.d @@ -7,21 +7,20 @@ import argparse; # Minify the specified files minijson ./dist/**/*.json ./build/a.json + minijson file1_with_comment.json file2_with_comment.json - # Minify the specified files (supports comments) - minijson --comment file1_with_comment.json file2_with_comment.json + # Minify the specified files and disable comment support for faster minification + minijson --comment=false file1_no_comment.json file2_no_comment.json # Minify the specified json string minijson --str '{"some_json": "string_here"}' - - # Minify the specified json string (supports comments) - minijson --comment --str '{"some_json": "string_here"} //comment' + minijson --str '{"some_json": "string_here"} //comment' More information at https://github.com/aminya/minijson `)) struct Options { - bool comment = false; + bool comment = true; string[] str; // (Deprecated) A list of files to minify (for backwards compatiblitity with getopt) string[] file; diff --git a/src/native/lib.d b/src/native/lib.d index 2b0939c..bb512df 100644 --- a/src/native/lib.d +++ b/src/native/lib.d @@ -12,12 +12,12 @@ const tokenizerNoComment = ctRegex!(`[\n\r"[]]`, "g"); Params: jsonString = the json string you want to minify - hasComment = a boolean to support comments in json. Default: `false`. + hasComment = a boolean to support comments in json. Default: `true`. Return: the minified json string */ -string minifyString(in string jsonString, in bool hasComment = false) @trusted +string minifyString(in string jsonString, in bool hasComment = true) @trusted { return hasComment ? minifyStringWithComments(jsonString) : minifyStringNoComments(jsonString); } @@ -227,7 +227,7 @@ private bool hasNoSpace(const ref string str) @trusted Return: the minified json strings */ -string[] minifyStrings(in string[] jsonStrings, in bool hasComment = false) @trusted +string[] minifyStrings(in string[] jsonStrings, in bool hasComment = true) @trusted { import std.algorithm : map; import std.array : array; @@ -242,7 +242,7 @@ string[] minifyStrings(in string[] jsonStrings, in bool hasComment = false) @tru paths = the paths to the files. It could be glob patterns. hasComment = a boolean to support comments in json. Default: `false`. */ -void minifyFiles(in string[] paths, in bool hasComment = false) @trusted +void minifyFiles(in string[] paths, in bool hasComment = true) @trusted { import std.parallelism : parallel; import std.algorithm; diff --git a/src/native/libc.d b/src/native/libc.d index c8b19eb..78e6dcb 100644 --- a/src/native/libc.d +++ b/src/native/libc.d @@ -7,12 +7,12 @@ import minijson.lib : minifyString; Params: jsonString = the json string you want to minify - hasComment = a boolean to support comments in json. Default: `false`. + hasComment = a boolean to support comments in json. Default: `true`. Return: the minified json string */ -extern (C) auto c_minifyString(char* jsonCString, bool hasComment = false) +extern (C) auto c_minifyString(char* jsonCString, bool hasComment = true) { import std : fromStringz, toStringz; diff --git a/src/node/lib.ts b/src/node/lib.ts index c113f0c..2ad32c3 100644 --- a/src/node/lib.ts +++ b/src/node/lib.ts @@ -7,20 +7,18 @@ import { join } from "path" * Minify all the given JSON files in place. It minifies the files in parallel. * * @param files An array of paths to the files - * @param hasComment A boolean to support comments in json. Default `false`. + * @param hasComment A boolean to support comments in json. Default `true`. * @returns {Promise} Returns a void promise that resolves when all the files are minified * @throws {Promise} The promise is rejected with the reason for failure */ -export async function minifyFiles(files: string[], hasComment = false): Promise { +export async function minifyFiles(files: readonly string[], hasComment = true): Promise { try { const filesNum = files.length if (filesNum === 0) { return Promise.resolve() } - const minijsonArgs = hasComment ? ["--comment", ...files] : files - - await spawnMinijson(minijsonArgs) + await spawnMinijson([...files, hasComment ? "--comment=true" : "--comment=false"]) } catch (e) { console.error(e, "Falling back to jsonminify") await minifyFilesFallback(files) @@ -48,7 +46,7 @@ export function spawnMinijson(args: string[]): Promise { }) } -async function minifyFilesFallback(files: string[]) { +async function minifyFilesFallback(files: readonly string[]) { const jsonminify = require("jsonminify") await Promise.all( files.map(async (file) => { @@ -64,11 +62,11 @@ async function minifyFilesFallback(files: string[]) { * Minify the given JSON string * * @param jsonString The json string you want to minify - * @param hasComment A boolean to support comments in json. Default `false`. + * @param hasComment A boolean to support comments in json. Default `true`. * @returns {Promise} The minified json string * @throws {Promise} The promise is rejected with the reason for failure */ -export async function minifyString(jsonString: string, hasComment = false): Promise { +export async function minifyString(jsonString: string, hasComment = true): Promise { const args = ["--str", jsonString] if (hasComment) { args.push("--comment")