From 22ea2aa71bc7a400c5a458d9dc1054f74c78ad35 Mon Sep 17 00:00:00 2001 From: George Fu Date: Wed, 27 Mar 2024 17:06:55 +0000 Subject: [PATCH] chore: update compression algorithm for objects --- .../CompressionAlgorithm.js | 28 +++++--- .../PatternDetection.js | 9 ++- scripts/smithy-model/compress.js | 65 +++++++++++++++++++ scripts/smithy-model/getPrunedModelObject.js | 27 ++++++++ 4 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 scripts/smithy-model/compress.js create mode 100644 scripts/smithy-model/getPrunedModelObject.js diff --git a/scripts/endpoints-ruleset/compression-algorithms/CompressionAlgorithm.js b/scripts/endpoints-ruleset/compression-algorithms/CompressionAlgorithm.js index 96ad2f82cfc9..4e46b53322ff 100644 --- a/scripts/endpoints-ruleset/compression-algorithms/CompressionAlgorithm.js +++ b/scripts/endpoints-ruleset/compression-algorithms/CompressionAlgorithm.js @@ -31,19 +31,26 @@ module.exports = class CompressionAlgorithm { */ verifyImplementation() { const tempFile = path.join(__dirname, "..", "temp", `__${uuid.v4()}tmp.js`); // defeat require-cache. - let original = 0; - let compressed = 1; + const original = this.data; + let compressed = { x: "compressed code string read failure" }; try { fs.writeFileSync(tempFile, this.toCodeString("module.exports = $"), "utf-8"); - original = this.toJsonString(this.data); - compressed = this.toJsonString(require(tempFile)); - } catch (e) {} + compressed = require(tempFile); + } catch (e) { + console.error(e); + } fs.rmSync(tempFile); - if (original !== compressed) { - fs.writeFileSync(path.join(__dirname, "..", "temp", "__original.json"), original); - fs.writeFileSync(path.join(__dirname, "..", "temp", "__compressed.json"), compressed); + try { + assert.deepStrictEqual( + original, + compressed, + `Compression implementation is not correct for ${this.constructor.name}.` + ); + } catch (e) { + fs.writeFileSync(path.join(__dirname, "..", "temp", "__original.json"), JSON.stringify(original, null, 2)); + fs.writeFileSync(path.join(__dirname, "..", "temp", "__compressed.json"), JSON.stringify(original, null, 2)); + throw e; } - assert(original === compressed, `Compression implementation is not correct for ${this.constructor.name}.`); } /** @@ -55,6 +62,9 @@ module.exports = class CompressionAlgorithm { case "undefined": break; case "object": + if (null === data) { + break; + } if (Array.isArray(data)) { buffer += `\n` + indent + `[\n`; for (let i = 0; i < data.length; ++i) { diff --git a/scripts/endpoints-ruleset/compression-algorithms/PatternDetection.js b/scripts/endpoints-ruleset/compression-algorithms/PatternDetection.js index f41925859720..a445d0fedb76 100644 --- a/scripts/endpoints-ruleset/compression-algorithms/PatternDetection.js +++ b/scripts/endpoints-ruleset/compression-algorithms/PatternDetection.js @@ -102,6 +102,11 @@ module.exports = class PatternDetection extends CompressionAlgorithm { if (carry) { this.varName.push(0); } + + if (["in", "do", "if", "else", "try", "catch", "return"].includes(out)) { + return this.nextVariableName(); + } + return out; } @@ -238,6 +243,7 @@ module.exports = class PatternDetection extends CompressionAlgorithm { } } + return aStartChar < bStartChar ? -1 : 1; throw new Error(`unexpected start char: ${aStartChar}, ${bStartChar}`); }); @@ -285,7 +291,8 @@ module.exports = class PatternDetection extends CompressionAlgorithm { if (found.length > 1 && key.length * found.length > 8) { const symbol = this.nextVariableName(); keyVarBuffer.push(`${symbol}="${key}"`); - buffer = buffer.replaceAll(new RegExp(`"?${key}"?:([^ ])`, "g"), `[${symbol}]:$1`); + buffer = buffer.replaceAll(new RegExp(`"${key}":([^ ])`, "g"), `[${symbol}]:$1`); + buffer = buffer.replaceAll(new RegExp(`([,{\n])${key}:([^ ])`, "g"), `$1[${symbol}]:$2`); } } if (keyVarBuffer.length > 0) { diff --git a/scripts/smithy-model/compress.js b/scripts/smithy-model/compress.js new file mode 100644 index 000000000000..de5fd1f4253b --- /dev/null +++ b/scripts/smithy-model/compress.js @@ -0,0 +1,65 @@ +const fs = require("fs"); +const path = require("path"); + +const RemoveWhitespace = require("../endpoints-ruleset/compression-algorithms/RemoveWhitespace"); +const PatternDetection = require("../endpoints-ruleset/compression-algorithms/PatternDetection"); +const { getPrunedModelObject } = require("./getPrunedModelObject"); + +/** + * Run compression on model objects for SDK clients. + */ +const main = (singleModel = "lambda") => { + const root = path.join(__dirname, "..", ".."); + const modelsFolder = path.join(root, "codegen", "sdk-codegen", "aws-models"); + const modelsList = singleModel ? [`${singleModel}.json`] : fs.readdirSync(modelsFolder); + + /** + * The first algorithm which passes self-verification will be used. + */ + const compressionAlgorithms = [(data) => new PatternDetection(data), (data) => new RemoveWhitespace(data)]; + + for (const modelJsonFilename of modelsList) { + const client = modelJsonFilename.replace(".json", ""); + const serviceJson = path.join(modelsFolder, modelJsonFilename); + + const service = require(serviceJson); + + const data = getPrunedModelObject(service); + + let selectedAlgorithm = null; + for (const factory of compressionAlgorithms) { + const algo = factory(data); + try { + algo.verifyImplementation(); + selectedAlgorithm = factory(data); + break; + } catch (e) { + const sample = factory(data).toCodeString("module.exports = $;"); + fs.writeFileSync( + path.join(path.join(__dirname, "temp", client + `-failed-${factory(data).constructor.name}.js`)), + sample, + "utf-8" + ); + console.warn(`WARN: Algorithm ${algo.constructor.name} failed for ${client}.`); + } + } + + if (!selectedAlgorithm) { + throw new Error(`No viable algorithm for ${client}`); + } + + const modifiedSource = `// @ts-nocheck +// generated code, do not edit +/* This file is a compressed version of ./codegen/sdk-codegen/aws-models/${client}.json */ + +${selectedAlgorithm.toCodeString("module.exports = $;")} +`; + + fs.writeFileSync(path.join(__dirname, "temp", `${client}.min.js`), modifiedSource, "utf-8"); + console.log(client, `OK - ${selectedAlgorithm.constructor.name}`); + } + + return 0; +}; + +main(); diff --git a/scripts/smithy-model/getPrunedModelObject.js b/scripts/smithy-model/getPrunedModelObject.js new file mode 100644 index 000000000000..ad4f240085ed --- /dev/null +++ b/scripts/smithy-model/getPrunedModelObject.js @@ -0,0 +1,27 @@ +/** + * Remove entries from model not required for runtime: + */ +const getPrunedModelObject = (data) => { + const prunedData = data; + if (!data || typeof data !== "object") { + return prunedData; + } + for (const key in prunedData) { + if (key === "documentation" || key === "smithy.api#documentation") { + delete prunedData[key]; + continue; + } + if ( + key === "smithy.api#pattern" || + key === "smithy.rules#endpointRuleSet" || + key === "smithy.rules#endpointTests" + ) { + delete prunedData[key]; + continue; + } + prunedData[key] = getPrunedModelObject(prunedData[key]); + } + return prunedData; +}; + +module.exports = { getPrunedModelObject };