Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

perf: improve FlagDependencyExportsPlugin for large JSON by depth #8802

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion crates/node_binding/binding.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1569,6 +1569,10 @@ export interface RawJavascriptParserOptions {
importDynamic?: boolean
}

export interface RawJsonParserOptions {
exportsDepth?: number
}

export interface RawLazyCompilationOption {
module: ((err: Error | null, arg: RawModuleArg) => RawModuleInfo)
test?: RawLazyCompilationTest
Expand Down Expand Up @@ -1804,12 +1808,13 @@ export interface RawOutputOptions {
}

export interface RawParserOptions {
type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm"
type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json"
asset?: RawAssetParserOptions
css?: RawCssParserOptions
cssAuto?: RawCssAutoParserOptions
cssModule?: RawCssModuleParserOptions
javascript?: RawJavascriptParserOptions
json?: RawJsonParserOptions
}

export interface RawPathData {
Expand Down
31 changes: 26 additions & 5 deletions crates/rspack_binding_values/src/raw_options/raw_module/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ use rspack_core::{
CssAutoGeneratorOptions, CssAutoParserOptions, CssGeneratorOptions, CssModuleGeneratorOptions,
CssModuleParserOptions, CssParserOptions, DescriptionData, DynamicImportFetchPriority,
DynamicImportMode, ExportPresenceMode, FuncUseCtx, GeneratorOptions, GeneratorOptionsMap,
JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, ModuleNoParseRule,
ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule, ModuleRuleEffect,
ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict, ParserOptions,
ParserOptionsMap,
JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, JsonParserOptions,
ModuleNoParseRule, ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule,
ModuleRuleEffect, ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict,
ParserOptions, ParserOptionsMap,
};
use rspack_error::error;
use rspack_napi::threadsafe_function::ThreadsafeFunction;
Expand Down Expand Up @@ -183,14 +183,15 @@ pub struct RawModuleRule {
#[napi(object)]
pub struct RawParserOptions {
#[napi(
ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm""#
ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json""#
)]
pub r#type: String,
pub asset: Option<RawAssetParserOptions>,
pub css: Option<RawCssParserOptions>,
pub css_auto: Option<RawCssAutoParserOptions>,
pub css_module: Option<RawCssModuleParserOptions>,
pub javascript: Option<RawJavascriptParserOptions>,
pub json: Option<RawJsonParserOptions>,
}

impl From<RawParserOptions> for ParserOptions {
Expand Down Expand Up @@ -246,6 +247,12 @@ impl From<RawParserOptions> for ParserOptions {
.expect("should have an \"css_module\" when RawParserOptions.type is \"css/module\"")
.into(),
),
"json" => Self::Json(
value
.json
.expect("should have an \"json\" when RawParserOptions.type is \"json\"")
.into(),
),
_ => panic!(
"Failed to resolve the RawParserOptions.type {}.",
value.r#type
Expand Down Expand Up @@ -425,6 +432,20 @@ impl From<RawCssModuleParserOptions> for CssModuleParserOptions {
}
}

#[derive(Debug, Default)]
#[napi(object)]
pub struct RawJsonParserOptions {
pub exports_depth: Option<f64>,
}

impl From<RawJsonParserOptions> for JsonParserOptions {
fn from(value: RawJsonParserOptions) -> Self {
Self {
exports_depth: value.exports_depth,
}
}
}

#[derive(Debug, Default)]
#[napi(object, object_to_js = false)]
pub struct RawGeneratorOptions {
Expand Down
8 changes: 8 additions & 0 deletions crates/rspack_core/src/options/module.rs
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ pub enum ParserOptions {
JavascriptAuto(JavascriptParserOptions),
JavascriptEsm(JavascriptParserOptions),
JavascriptDynamic(JavascriptParserOptions),
Json(JsonParserOptions),
Unknown,
}

Expand Down Expand Up @@ -68,6 +69,7 @@ impl ParserOptions {
JavascriptDynamic,
JavascriptParserOptions
);
get_variant!(get_json, Json, JsonParserOptions);
}

#[cacheable]
Expand Down Expand Up @@ -287,6 +289,12 @@ pub struct CssModuleParserOptions {
pub named_exports: Option<bool>,
}

#[cacheable]
#[derive(Debug, Clone, MergeFrom)]
pub struct JsonParserOptions {
pub exports_depth: Option<f64>,
}

#[derive(Debug)]
pub struct GeneratorOptionsMap(HashMap<String, GeneratorOptions>);

Expand Down
37 changes: 26 additions & 11 deletions crates/rspack_plugin_json/src/json_exports_dependency.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ pub struct JsonExportsDependency {
id: DependencyId,
#[cacheable(with=AsPreset)]
data: JsonValue,
exports_depth: f64,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
exports_depth: f64,
exports_depth: u32,

u32 should be enough

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make sense.

}

impl JsonExportsDependency {
pub fn new(data: JsonValue) -> Self {
pub fn new(data: JsonValue, exports_depth: f64) -> Self {
Self {
data,
id: DependencyId::new(),
exports_depth,
}
}
}
Expand All @@ -31,8 +33,10 @@ impl Dependency for JsonExportsDependency {
}

fn get_exports(&self, _mg: &ModuleGraph) -> Option<ExportsSpec> {
let exports_depth: u64 = self.exports_depth as u64;
Some(ExportsSpec {
exports: get_exports_from_data(&self.data).unwrap_or(ExportsOfExportsSpec::Null),
exports: get_exports_from_data(&self.data, exports_depth, 1)
.unwrap_or(ExportsOfExportsSpec::Null),
..Default::default()
})
}
Expand Down Expand Up @@ -68,7 +72,14 @@ impl DependencyTemplate for JsonExportsDependency {
}
}

fn get_exports_from_data(data: &JsonValue) -> Option<ExportsOfExportsSpec> {
fn get_exports_from_data(
data: &JsonValue,
exports_depth: u64,
cur_depth: u64,
) -> Option<ExportsOfExportsSpec> {
if cur_depth > exports_depth {
return None;
}
let ret = match data {
JsonValue::Null
| JsonValue::Short(_)
Expand All @@ -84,11 +95,13 @@ fn get_exports_from_data(data: &JsonValue) -> Option<ExportsOfExportsSpec> {
ExportNameOrSpec::ExportSpec(ExportSpec {
name: k.into(),
can_mangle: Some(true),
exports: get_exports_from_data(v).map(|item| match item {
ExportsOfExportsSpec::True => unreachable!(),
ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
}),
exports: get_exports_from_data(v, exports_depth, cur_depth + 1).map(
|item| match item {
ExportsOfExportsSpec::True => unreachable!(),
ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
},
),
..Default::default()
})
})
Expand All @@ -106,9 +119,11 @@ fn get_exports_from_data(data: &JsonValue) -> Option<ExportsOfExportsSpec> {
ExportNameOrSpec::ExportSpec(ExportSpec {
name: itoa!(i).into(),
can_mangle: Some(true),
exports: get_exports_from_data(item).map(|item| match item {
ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
exports: get_exports_from_data(item, exports_depth, cur_depth + 1).map(|item| {
match item {
ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
}
}),
..Default::default()
})
Expand Down
19 changes: 16 additions & 3 deletions crates/rspack_plugin_json/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ mod utils;

#[cacheable]
#[derive(Debug)]
struct JsonParserAndGenerator;
struct JsonParserAndGenerator {
pub exports_depth: f64,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
pub exports_depth: f64,
pub exports_depth: u32,

}

#[cacheable_dyn]
impl ParserAndGenerator for JsonParserAndGenerator {
Expand Down Expand Up @@ -119,7 +121,10 @@ impl ParserAndGenerator for JsonParserAndGenerator {
rspack_core::ParseResult {
presentational_dependencies: vec![],
dependencies: if let Some(data) = data {
vec![Box::new(JsonExportsDependency::new(data))]
vec![Box::new(JsonExportsDependency::new(
data,
self.exports_depth,
))]
} else {
vec![]
},
Expand Down Expand Up @@ -224,7 +229,15 @@ impl Plugin for JsonPlugin {
) -> Result<()> {
ctx.context.register_parser_and_generator_builder(
rspack_core::ModuleType::Json,
Box::new(|_, _| Box::new(JsonParserAndGenerator {})),
Box::new(|p, _| {
let p = p
.and_then(|p| p.get_json())
.expect("should have JsonParserOptions");

Box::new(JsonParserAndGenerator {
exports_depth: p.exports_depth.expect("should have exports_depth"),
})
}),
);

Ok(())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,9 @@ Object {
wrappedContextCritical: false,
wrappedContextRegExp: /\\.\\*/,
},
json: Object {
exportsDepth: Infinity,
},
},
rules: Array [],
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ module.exports = {
- "mode": "none",
+ "mode": "development",
@@ ... @@
- "exportsDepth": Infinity,
+ "exportsDepth": 1,
@@ ... @@
- "chunkIds": "natural",
+ "chunkIds": "named",
@@ ... @@
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ module.exports = {
- "mode": "none",
+ "mode": "development",
@@ ... @@
- "exportsDepth": Infinity,
+ "exportsDepth": 1,
@@ ... @@
- "chunkIds": "natural",
+ "chunkIds": "named",
@@ ... @@
Expand Down
8 changes: 8 additions & 0 deletions packages/rspack/etc/core.api.md
Original file line number Diff line number Diff line change
Expand Up @@ -2747,6 +2747,11 @@ type JsonObject_2 = {
[Key in string]?: JsonValue_2 | undefined;
};

// @public (undocumented)
export type JsonParserOptions = {
exportsDepth?: number;
};

// @public (undocumented)
type JsonPrimitive = string | number | boolean | null;

Expand Down Expand Up @@ -4460,6 +4465,7 @@ export type ParserOptionsByModuleTypeKnown = {
"javascript/auto"?: JavascriptParserOptions;
"javascript/dynamic"?: JavascriptParserOptions;
"javascript/esm"?: JavascriptParserOptions;
json?: JsonParserOptions;
};

// @public
Expand Down Expand Up @@ -5308,6 +5314,7 @@ declare namespace rspackExports {
CssAutoParserOptions,
CssModuleParserOptions,
JavascriptParserOptions,
JsonParserOptions,
ParserOptionsByModuleTypeKnown,
ParserOptionsByModuleTypeUnknown,
ParserOptionsByModuleType,
Expand Down Expand Up @@ -10453,6 +10460,7 @@ declare namespace t {
CssAutoParserOptions,
CssModuleParserOptions,
JavascriptParserOptions,
JsonParserOptions,
ParserOptionsByModuleTypeKnown,
ParserOptionsByModuleTypeUnknown,
ParserOptionsByModuleType,
Expand Down
18 changes: 18 additions & 0 deletions packages/rspack/src/config/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
type RawFuncUseCtx,
type RawGeneratorOptions,
type RawJavascriptParserOptions,
type RawJsonParserOptions,
type RawModuleRule,
type RawModuleRuleUse,
type RawOptions,
Expand Down Expand Up @@ -54,6 +55,7 @@ import type {
CssParserOptions,
GeneratorOptionsByModuleType,
JavascriptParserOptions,
JsonParserOptions,
Node,
Optimization,
ParserOptionsByModuleType,
Expand Down Expand Up @@ -490,6 +492,14 @@ function getRawParserOptions(
cssModule: getRawCssParserOptions(parser)
};
}

if (type === "json") {
return {
type: "json",
json: getRawJsonParserOptions(parser)
};
}

// FIXME: shouldn't depend on module type, for example: `rules: [{ test: /\.css/, generator: {..} }]` will error
throw new Error(`unreachable: unknow module type: ${type}`);
}
Expand Down Expand Up @@ -566,6 +576,14 @@ function getRawCssParserOptions(
};
}

function getRawJsonParserOptions(
parser: JsonParserOptions
): RawJsonParserOptions {
return {
exportsDepth: parser.exportsDepth
};
}

function getRawGeneratorOptions(
generator: { [k: string]: any },
type: string
Expand Down
Loading
Loading