From 89f567b3b3574d9dba770d8e6464b1e3a4afa2c6 Mon Sep 17 00:00:00 2001 From: hai-x <98948357+hai-x@users.noreply.github.com> Date: Tue, 24 Dec 2024 12:48:02 +0800 Subject: [PATCH] perf: improve FlagDependencyExportsPlugin for large JSON by depth (#8802) --- crates/node_binding/binding.d.ts | 7 +++- .../src/raw_options/raw_module/mod.rs | 31 +++++++++++++--- crates/rspack_core/src/options/module.rs | 8 +++++ .../src/json_exports_dependency.rs | 36 +++++++++++++------ crates/rspack_plugin_json/src/lib.rs | 19 ++++++++-- .../tests/__snapshots__/Defaults.test.js.snap | 3 ++ .../cache/cache-filesystem-dev.js | 3 ++ .../tests/defaultsCases/mode/development.js | 3 ++ packages/rspack/etc/core.api.md | 8 +++++ packages/rspack/src/config/adapter.ts | 18 ++++++++++ packages/rspack/src/config/defaults.ts | 17 +++++++-- packages/rspack/src/config/types.ts | 10 ++++++ .../bailout-flag-dep-export-perf/data.json | 27 ++++++++++++++ .../bailout-flag-dep-export-perf/index.js | 11 ++++++ .../webpack.config.js | 11 ++++++ .../json/flag-dep-export-perf/data.json | 27 ++++++++++++++ .../json/flag-dep-export-perf/index.js | 11 ++++++ .../flag-dep-export-perf/webpack.config.js | 4 +++ 18 files changed, 231 insertions(+), 23 deletions(-) create mode 100644 tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json create mode 100644 tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js create mode 100644 tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js create mode 100644 tests/webpack-test/configCases/json/flag-dep-export-perf/data.json create mode 100644 tests/webpack-test/configCases/json/flag-dep-export-perf/index.js create mode 100644 tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts index d6cc6d819448..5156de84c696 100644 --- a/crates/node_binding/binding.d.ts +++ b/crates/node_binding/binding.d.ts @@ -1573,6 +1573,10 @@ export interface RawJavascriptParserOptions { importDynamic?: boolean } +export interface RawJsonParserOptions { + exportsDepth?: number +} + export interface RawLazyCompilationOption { module: ((err: Error | null, arg: RawModuleArg) => RawModuleInfo) test?: RawLazyCompilationTest @@ -1808,12 +1812,13 @@ export interface RawOutputOptions { } export interface RawParserOptions { - type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" + type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json" asset?: RawAssetParserOptions css?: RawCssParserOptions cssAuto?: RawCssAutoParserOptions cssModule?: RawCssModuleParserOptions javascript?: RawJavascriptParserOptions + json?: RawJsonParserOptions } export interface RawPathData { diff --git a/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs b/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs index da924f94b081..f63faa257142 100644 --- a/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs +++ b/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs @@ -12,10 +12,10 @@ use rspack_core::{ CssAutoGeneratorOptions, CssAutoParserOptions, CssGeneratorOptions, CssModuleGeneratorOptions, CssModuleParserOptions, CssParserOptions, DescriptionData, DynamicImportFetchPriority, DynamicImportMode, ExportPresenceMode, FuncUseCtx, GeneratorOptions, GeneratorOptionsMap, - JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, ModuleNoParseRule, - ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule, ModuleRuleEffect, - ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict, ParserOptions, - ParserOptionsMap, + JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, JsonParserOptions, + ModuleNoParseRule, ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule, + ModuleRuleEffect, ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict, + ParserOptions, ParserOptionsMap, }; use rspack_error::error; use rspack_napi::threadsafe_function::ThreadsafeFunction; @@ -183,7 +183,7 @@ pub struct RawModuleRule { #[napi(object)] pub struct RawParserOptions { #[napi( - ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm""# + ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json""# )] pub r#type: String, pub asset: Option, @@ -191,6 +191,7 @@ pub struct RawParserOptions { pub css_auto: Option, pub css_module: Option, pub javascript: Option, + pub json: Option, } impl From for ParserOptions { @@ -246,6 +247,12 @@ impl From for ParserOptions { .expect("should have an \"css_module\" when RawParserOptions.type is \"css/module\"") .into(), ), + "json" => Self::Json( + value + .json + .expect("should have an \"json\" when RawParserOptions.type is \"json\"") + .into(), + ), _ => panic!( "Failed to resolve the RawParserOptions.type {}.", value.r#type @@ -425,6 +432,20 @@ impl From for CssModuleParserOptions { } } +#[derive(Debug, Default)] +#[napi(object)] +pub struct RawJsonParserOptions { + pub exports_depth: Option, +} + +impl From for JsonParserOptions { + fn from(value: RawJsonParserOptions) -> Self { + Self { + exports_depth: value.exports_depth, + } + } +} + #[derive(Debug, Default)] #[napi(object, object_to_js = false)] pub struct RawGeneratorOptions { diff --git a/crates/rspack_core/src/options/module.rs b/crates/rspack_core/src/options/module.rs index 9579dc6c5b1d..05137ca4b1d9 100644 --- a/crates/rspack_core/src/options/module.rs +++ b/crates/rspack_core/src/options/module.rs @@ -41,6 +41,7 @@ pub enum ParserOptions { JavascriptAuto(JavascriptParserOptions), JavascriptEsm(JavascriptParserOptions), JavascriptDynamic(JavascriptParserOptions), + Json(JsonParserOptions), Unknown, } @@ -68,6 +69,7 @@ impl ParserOptions { JavascriptDynamic, JavascriptParserOptions ); + get_variant!(get_json, Json, JsonParserOptions); } #[cacheable] @@ -287,6 +289,12 @@ pub struct CssModuleParserOptions { pub named_exports: Option, } +#[cacheable] +#[derive(Debug, Clone, MergeFrom)] +pub struct JsonParserOptions { + pub exports_depth: Option, +} + #[derive(Debug)] pub struct GeneratorOptionsMap(HashMap); diff --git a/crates/rspack_plugin_json/src/json_exports_dependency.rs b/crates/rspack_plugin_json/src/json_exports_dependency.rs index 629b3d6899dc..69130b2aed7e 100644 --- a/crates/rspack_plugin_json/src/json_exports_dependency.rs +++ b/crates/rspack_plugin_json/src/json_exports_dependency.rs @@ -13,13 +13,15 @@ pub struct JsonExportsDependency { id: DependencyId, #[cacheable(with=AsPreset)] data: JsonValue, + exports_depth: u32, } impl JsonExportsDependency { - pub fn new(data: JsonValue) -> Self { + pub fn new(data: JsonValue, exports_depth: u32) -> Self { Self { data, id: DependencyId::new(), + exports_depth, } } } @@ -32,7 +34,8 @@ impl Dependency for JsonExportsDependency { fn get_exports(&self, _mg: &ModuleGraph) -> Option { Some(ExportsSpec { - exports: get_exports_from_data(&self.data).unwrap_or(ExportsOfExportsSpec::Null), + exports: get_exports_from_data(&self.data, self.exports_depth, 1) + .unwrap_or(ExportsOfExportsSpec::Null), ..Default::default() }) } @@ -68,7 +71,14 @@ impl DependencyTemplate for JsonExportsDependency { } } -fn get_exports_from_data(data: &JsonValue) -> Option { +fn get_exports_from_data( + data: &JsonValue, + exports_depth: u32, + cur_depth: u32, +) -> Option { + if cur_depth > exports_depth { + return None; + } let ret = match data { JsonValue::Null | JsonValue::Short(_) @@ -84,11 +94,13 @@ fn get_exports_from_data(data: &JsonValue) -> Option { ExportNameOrSpec::ExportSpec(ExportSpec { name: k.into(), can_mangle: Some(true), - exports: get_exports_from_data(v).map(|item| match item { - ExportsOfExportsSpec::True => unreachable!(), - ExportsOfExportsSpec::Null => unreachable!(), - ExportsOfExportsSpec::Array(arr) => arr, - }), + exports: get_exports_from_data(v, exports_depth, cur_depth + 1).map( + |item| match item { + ExportsOfExportsSpec::True => unreachable!(), + ExportsOfExportsSpec::Null => unreachable!(), + ExportsOfExportsSpec::Array(arr) => arr, + }, + ), ..Default::default() }) }) @@ -106,9 +118,11 @@ fn get_exports_from_data(data: &JsonValue) -> Option { ExportNameOrSpec::ExportSpec(ExportSpec { name: itoa!(i).into(), can_mangle: Some(true), - exports: get_exports_from_data(item).map(|item| match item { - ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(), - ExportsOfExportsSpec::Array(arr) => arr, + exports: get_exports_from_data(item, exports_depth, cur_depth + 1).map(|item| { + match item { + ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(), + ExportsOfExportsSpec::Array(arr) => arr, + } }), ..Default::default() }) diff --git a/crates/rspack_plugin_json/src/lib.rs b/crates/rspack_plugin_json/src/lib.rs index f9abb2bef1e1..3b2408b2a4cf 100644 --- a/crates/rspack_plugin_json/src/lib.rs +++ b/crates/rspack_plugin_json/src/lib.rs @@ -32,7 +32,9 @@ mod utils; #[cacheable] #[derive(Debug)] -struct JsonParserAndGenerator; +struct JsonParserAndGenerator { + pub exports_depth: u32, +} #[cacheable_dyn] impl ParserAndGenerator for JsonParserAndGenerator { @@ -119,7 +121,10 @@ impl ParserAndGenerator for JsonParserAndGenerator { rspack_core::ParseResult { presentational_dependencies: vec![], dependencies: if let Some(data) = data { - vec![Box::new(JsonExportsDependency::new(data))] + vec![Box::new(JsonExportsDependency::new( + data, + self.exports_depth, + ))] } else { vec![] }, @@ -224,7 +229,15 @@ impl Plugin for JsonPlugin { ) -> Result<()> { ctx.context.register_parser_and_generator_builder( rspack_core::ModuleType::Json, - Box::new(|_, _| Box::new(JsonParserAndGenerator {})), + Box::new(|p, _| { + let p = p + .and_then(|p| p.get_json()) + .expect("should have JsonParserOptions"); + + Box::new(JsonParserAndGenerator { + exports_depth: p.exports_depth.expect("should have exports_depth"), + }) + }), ); Ok(()) diff --git a/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap b/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap index 608114f77a58..ef0e2e2c85c8 100644 --- a/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap +++ b/packages/rspack-test-tools/tests/__snapshots__/Defaults.test.js.snap @@ -192,6 +192,9 @@ Object { wrappedContextCritical: false, wrappedContextRegExp: /\\.\\*/, }, + json: Object { + exportsDepth: 9007199254740991, + }, }, rules: Array [], }, diff --git a/packages/rspack-test-tools/tests/defaultsCases/cache/cache-filesystem-dev.js b/packages/rspack-test-tools/tests/defaultsCases/cache/cache-filesystem-dev.js index d24cb95fb8d4..27093d67420f 100644 --- a/packages/rspack-test-tools/tests/defaultsCases/cache/cache-filesystem-dev.js +++ b/packages/rspack-test-tools/tests/defaultsCases/cache/cache-filesystem-dev.js @@ -22,6 +22,9 @@ module.exports = { - "mode": "none", + "mode": "development", @@ ... @@ + - "exportsDepth": 9007199254740991, + + "exportsDepth": 1, + @@ ... @@ - "chunkIds": "natural", + "chunkIds": "named", @@ ... @@ diff --git a/packages/rspack-test-tools/tests/defaultsCases/mode/development.js b/packages/rspack-test-tools/tests/defaultsCases/mode/development.js index 9760dde289f6..7095b8a29133 100644 --- a/packages/rspack-test-tools/tests/defaultsCases/mode/development.js +++ b/packages/rspack-test-tools/tests/defaultsCases/mode/development.js @@ -20,6 +20,9 @@ module.exports = { - "mode": "none", + "mode": "development", @@ ... @@ + - "exportsDepth": 9007199254740991, + + "exportsDepth": 1, + @@ ... @@ - "chunkIds": "natural", + "chunkIds": "named", @@ ... @@ diff --git a/packages/rspack/etc/core.api.md b/packages/rspack/etc/core.api.md index 2da3b562f1a2..e7fd87611822 100644 --- a/packages/rspack/etc/core.api.md +++ b/packages/rspack/etc/core.api.md @@ -2748,6 +2748,11 @@ type JsonObject_2 = { [Key in string]?: JsonValue_2 | undefined; }; +// @public (undocumented) +export type JsonParserOptions = { + exportsDepth?: number; +}; + // @public (undocumented) type JsonPrimitive = string | number | boolean | null; @@ -4479,6 +4484,7 @@ export type ParserOptionsByModuleTypeKnown = { "javascript/auto"?: JavascriptParserOptions; "javascript/dynamic"?: JavascriptParserOptions; "javascript/esm"?: JavascriptParserOptions; + json?: JsonParserOptions; }; // @public @@ -5327,6 +5333,7 @@ declare namespace rspackExports { CssAutoParserOptions, CssModuleParserOptions, JavascriptParserOptions, + JsonParserOptions, ParserOptionsByModuleTypeKnown, ParserOptionsByModuleTypeUnknown, ParserOptionsByModuleType, @@ -10472,6 +10479,7 @@ declare namespace t { CssAutoParserOptions, CssModuleParserOptions, JavascriptParserOptions, + JsonParserOptions, ParserOptionsByModuleTypeKnown, ParserOptionsByModuleTypeUnknown, ParserOptionsByModuleType, diff --git a/packages/rspack/src/config/adapter.ts b/packages/rspack/src/config/adapter.ts index 72c1c88683b6..2e65dde63b89 100644 --- a/packages/rspack/src/config/adapter.ts +++ b/packages/rspack/src/config/adapter.ts @@ -15,6 +15,7 @@ import { type RawFuncUseCtx, type RawGeneratorOptions, type RawJavascriptParserOptions, + type RawJsonParserOptions, type RawModuleRule, type RawModuleRuleUse, type RawOptions, @@ -54,6 +55,7 @@ import type { CssParserOptions, GeneratorOptionsByModuleType, JavascriptParserOptions, + JsonParserOptions, Node, Optimization, ParserOptionsByModuleType, @@ -490,6 +492,14 @@ function getRawParserOptions( cssModule: getRawCssParserOptions(parser) }; } + + if (type === "json") { + return { + type: "json", + json: getRawJsonParserOptions(parser) + }; + } + // FIXME: shouldn't depend on module type, for example: `rules: [{ test: /\.css/, generator: {..} }]` will error throw new Error(`unreachable: unknow module type: ${type}`); } @@ -566,6 +576,14 @@ function getRawCssParserOptions( }; } +function getRawJsonParserOptions( + parser: JsonParserOptions +): RawJsonParserOptions { + return { + exportsDepth: parser.exportsDepth + }; +} + function getRawGeneratorOptions( generator: { [k: string]: any }, type: string diff --git a/packages/rspack/src/config/defaults.ts b/packages/rspack/src/config/defaults.ts index af221c0653ab..dcb46752c18c 100644 --- a/packages/rspack/src/config/defaults.ts +++ b/packages/rspack/src/config/defaults.ts @@ -12,7 +12,7 @@ import assert from "node:assert"; import fs from "node:fs"; import path from "node:path"; -import { ASSET_MODULE_TYPE } from "../ModuleTypeConstants"; +import { ASSET_MODULE_TYPE, JSON_MODULE_TYPE } from "../ModuleTypeConstants"; import { Template } from "../Template"; import { LightningCssMinimizerRspackPlugin, @@ -102,7 +102,8 @@ export const applyRspackOptionsDefaults = ( applyModuleDefaults(options.module, { asyncWebAssembly: options.experiments.asyncWebAssembly!, css: options.experiments.css, - targetProperties + targetProperties, + mode: options.mode }); applyOutputDefaults(options.output, { @@ -285,11 +286,13 @@ const applyModuleDefaults = ( { asyncWebAssembly, css, - targetProperties + targetProperties, + mode }: { asyncWebAssembly: boolean; css?: boolean; targetProperties: any; + mode?: Mode; } ) => { assertNotNill(module.parser); @@ -307,6 +310,14 @@ const applyModuleDefaults = ( assertNotNill(module.parser.javascript); applyJavascriptParserOptionsDefaults(module.parser.javascript); + F(module.parser, JSON_MODULE_TYPE, () => ({})); + assertNotNill(module.parser[JSON_MODULE_TYPE]); + D( + module.parser[JSON_MODULE_TYPE], + "exportsDepth", + mode === "development" ? 1 : Number.MAX_SAFE_INTEGER + ); + if (css) { F(module.parser, "css", () => ({})); assertNotNill(module.parser.css); diff --git a/packages/rspack/src/config/types.ts b/packages/rspack/src/config/types.ts index 7dcaf48c9258..0a1c3c4bb6ef 100644 --- a/packages/rspack/src/config/types.ts +++ b/packages/rspack/src/config/types.ts @@ -1068,6 +1068,13 @@ export type JavascriptParserOptions = { importDynamic?: boolean; }; +export type JsonParserOptions = { + /** + * The depth of json dependency flagged as `exportInfo`. + */ + exportsDepth?: number; +}; + /** Configure all parsers' options in one place with module.parser. */ export type ParserOptionsByModuleTypeKnown = { /** Parser options for `asset` modules. */ @@ -1093,6 +1100,9 @@ export type ParserOptionsByModuleTypeKnown = { /** Parser options for `javascript/esm` modules. */ "javascript/esm"?: JavascriptParserOptions; + + /** Parser options for `json` modules. */ + json?: JsonParserOptions; }; /** Configure all parsers' options in one place with module.parser. */ diff --git a/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json new file mode 100644 index 000000000000..bee9f9b8d889 --- /dev/null +++ b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json @@ -0,0 +1,27 @@ +{ + "depth_1": { + "depth_2": { + "depth_3": { + "depth_4": { + "depth_5": { + "depth_6": "depth_6" + } + } + } + } + }, + "_depth_1": { + "_depth_2": { + "_depth_3": { + "_depth_4": { + "_depth_5": { + "_depth_6": "_depth_6" + } + } + } + } + }, + "__depth_1": [ + { "__depth_3": [{ "__depth_5": [{ "__depth_7": ["__depth_8"] }] }] } + ] +} diff --git a/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js new file mode 100644 index 000000000000..df6ffa8072bc --- /dev/null +++ b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js @@ -0,0 +1,11 @@ +export * from './data.json'; + +it("should compile and run", () => { + expect(__webpack_exports_info__.depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__._depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__.__depth_1.provideInfo).toBe(true) + + expect(__webpack_exports_info__.depth_1.depth_2.provideInfo).toBe(true) + expect(__webpack_exports_info__._depth_1._depth_2._depth_3._depth_4.provideInfo).toBe(true) + expect(__webpack_exports_info__.__depth_1[0].__depth_3[0].__depth_5.provideInfo).toBe(true) +}); diff --git a/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js new file mode 100644 index 000000000000..5b899783606f --- /dev/null +++ b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js @@ -0,0 +1,11 @@ +/** @type {import("@rspack/core").Configuration} */ +module.exports = { + mode: "development", + module: { + parser: { + json: { + exportsDepth: Number.MAX_SAFE_INTEGER + } + } + } +}; diff --git a/tests/webpack-test/configCases/json/flag-dep-export-perf/data.json b/tests/webpack-test/configCases/json/flag-dep-export-perf/data.json new file mode 100644 index 000000000000..bee9f9b8d889 --- /dev/null +++ b/tests/webpack-test/configCases/json/flag-dep-export-perf/data.json @@ -0,0 +1,27 @@ +{ + "depth_1": { + "depth_2": { + "depth_3": { + "depth_4": { + "depth_5": { + "depth_6": "depth_6" + } + } + } + } + }, + "_depth_1": { + "_depth_2": { + "_depth_3": { + "_depth_4": { + "_depth_5": { + "_depth_6": "_depth_6" + } + } + } + } + }, + "__depth_1": [ + { "__depth_3": [{ "__depth_5": [{ "__depth_7": ["__depth_8"] }] }] } + ] +} diff --git a/tests/webpack-test/configCases/json/flag-dep-export-perf/index.js b/tests/webpack-test/configCases/json/flag-dep-export-perf/index.js new file mode 100644 index 000000000000..6ecc00cfec07 --- /dev/null +++ b/tests/webpack-test/configCases/json/flag-dep-export-perf/index.js @@ -0,0 +1,11 @@ +export * from './data.json'; + +it("should compile and run", () => { + expect(__webpack_exports_info__.depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__._depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__.__depth_1.provideInfo).toBe(true) + + expect(__webpack_exports_info__.depth_1.depth_2.provideInfo).toBe(undefined) + expect(__webpack_exports_info__._depth_1._depth_2._depth_3._depth_4.provideInfo).toBe(undefined) + expect(__webpack_exports_info__.__depth_1[0].__depth_3[0].__depth_5.provideInfo).toBe(undefined) +}); diff --git a/tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js b/tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js new file mode 100644 index 000000000000..45f6191a8d4c --- /dev/null +++ b/tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js @@ -0,0 +1,4 @@ +/** @type {import("@rspack/core").Configuration} */ +module.exports = { + mode: "development" +};