Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support custom parser for json type #8947

Merged
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/node_binding/binding.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1581,6 +1581,7 @@ export interface RawJavascriptParserOptions {

export interface RawJsonParserOptions {
exportsDepth?: number
parse?: (source: string) => string
cbbfcd marked this conversation as resolved.
Show resolved Hide resolved
}

export interface RawLazyCompilationOption {
Expand Down
7 changes: 4 additions & 3 deletions crates/rspack/src/options/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ use rspack_core::{
Filename, FilenameTemplate, GeneratorOptions, GeneratorOptionsMap, JavascriptParserOptions,
JavascriptParserOrder, JavascriptParserUrl, JsonParserOptions, LibraryName, LibraryNonUmdObject,
LibraryOptions, LibraryType, Mode, ModuleNoParseRules, ModuleOptions, ModuleRule,
ModuleRuleEffect, OutputOptions, ParserOptions, ParserOptionsMap, PathInfo, PublicPath, Resolve,
RspackFuture, RuleSetCondition, RuleSetLogicalConditions, TrustedTypes, WasmLoading,
WasmLoadingType,
ModuleRuleEffect, OutputOptions, ParseOption, ParserOptions, ParserOptionsMap, PathInfo,
PublicPath, Resolve, RspackFuture, RuleSetCondition, RuleSetLogicalConditions, TrustedTypes,
WasmLoading, WasmLoadingType,
};
use rspack_hash::{HashDigest, HashFunction, HashSalt};
use rspack_paths::{AssertUtf8, Utf8PathBuf};
Expand Down Expand Up @@ -642,6 +642,7 @@ impl ModuleOptionsBuilder {
} else {
Some(u32::MAX)
},
parse: ParseOption::None,
}),
);
}
Expand Down
14 changes: 11 additions & 3 deletions crates/rspack_binding_values/src/raw_options/raw_module/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use rspack_core::{
JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, JsonParserOptions,
ModuleNoParseRule, ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule,
ModuleRuleEffect, ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict,
ParserOptions, ParserOptionsMap,
ParseOption, ParserOptions, ParserOptionsMap,
};
use rspack_error::error;
use rspack_napi::threadsafe_function::ThreadsafeFunction;
Expand Down Expand Up @@ -188,7 +188,7 @@ pub struct RawModuleRule {
}

#[derive(Debug, Default)]
#[napi(object)]
#[napi(object, object_to_js = false)]
pub struct RawParserOptions {
#[napi(
ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json""#
Expand Down Expand Up @@ -441,15 +441,23 @@ impl From<RawCssModuleParserOptions> for CssModuleParserOptions {
}

#[derive(Debug, Default)]
#[napi(object)]
#[napi(object, object_to_js = false)]
pub struct RawJsonParserOptions {
pub exports_depth: Option<u32>,
#[napi(ts_type = "(source: string) => string")]
pub parse: Option<ThreadsafeFunction<String, String>>,
}

impl From<RawJsonParserOptions> for JsonParserOptions {
fn from(value: RawJsonParserOptions) -> Self {
let parse = match value.parse {
Some(f) => ParseOption::Func(Arc::new(move |s: String| f.blocking_call_with_sync(s))),
_ => ParseOption::None,
};

Self {
exports_depth: value.exports_depth,
parse,
}
}
}
Expand Down
1 change: 1 addition & 0 deletions crates/rspack_core/src/normal_module_factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ impl NormalModuleFactory {
| ParserOptions::JavascriptDynamic(b)
| ParserOptions::JavascriptEsm(b),
) => ParserOptions::Javascript(a.merge_from(b)),
(ParserOptions::Json(a), ParserOptions::Json(b)) => ParserOptions::Json(a.merge_from(b)),
(global, _) => global,
},
);
Expand Down
33 changes: 33 additions & 0 deletions crates/rspack_core/src/options/module.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,10 +306,43 @@ pub struct CssModuleParserOptions {
pub named_exports: Option<bool>,
}

pub type JsonParseFn = Arc<dyn Fn(String) -> Result<String> + Sync + Send>;

#[cacheable]
pub enum ParseOption {
Func(#[cacheable(with=Unsupported)] JsonParseFn),
None,
}

impl Debug for ParseOption {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Func(_) => write!(f, "ParseOption::Func(...)"),
_ => write!(f, "ParseOption::None"),
}
}
}

impl Clone for ParseOption {
fn clone(&self) -> Self {
match self {
Self::Func(f) => Self::Func(f.clone()),
Self::None => Self::None,
}
}
}

impl MergeFrom for ParseOption {
fn merge_from(self, other: &Self) -> Self {
other.clone()
}
}

#[cacheable]
#[derive(Debug, Clone, MergeFrom)]
pub struct JsonParserOptions {
pub exports_depth: Option<u32>,
pub parse: ParseOption,
}

#[derive(Debug, Default)]
Expand Down
100 changes: 58 additions & 42 deletions crates/rspack_plugin_json/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ use rspack_core::{
diagnostics::ModuleParseError,
rspack_sources::{BoxSource, RawStringSource, Source, SourceExt},
BuildMetaDefaultObject, BuildMetaExportsType, ChunkGraph, CompilerOptions, ExportsInfo,
GenerateContext, Module, ModuleGraph, ParserAndGenerator, Plugin, RuntimeGlobals, RuntimeSpec,
SourceType, UsageState, NAMESPACE_OBJECT_EXPORT,
GenerateContext, Module, ModuleGraph, ParseOption, ParserAndGenerator, Plugin, RuntimeGlobals,
RuntimeSpec, SourceType, UsageState, NAMESPACE_OBJECT_EXPORT,
};
use rspack_error::{
miette::diagnostic, DiagnosticExt, DiagnosticKind, IntoTWithDiagnosticArray, Result,
Expand All @@ -34,6 +34,7 @@ mod utils;
#[derive(Debug)]
struct JsonParserAndGenerator {
pub exports_depth: u32,
pub parse: ParseOption,
}

#[cacheable_dyn]
Expand All @@ -55,54 +56,68 @@ impl ParserAndGenerator for JsonParserAndGenerator {
build_info,
build_meta,
loaders,
module_parser_options,
..
} = parse_context;
let source = box_source.source();
let strip_bom_source = source.strip_prefix('\u{feff}');
let need_strip_bom = strip_bom_source.is_some();
let strip_bom_source = strip_bom_source.unwrap_or(&source);

let parse_result = json::parse(strip_bom_source.unwrap_or(&source)).map_err(|e| {
match e {
UnexpectedCharacter { ch, line, column } => {
let rope = ropey::Rope::from_str(&source);
let line_offset = rope.try_line_to_byte(line - 1).expect("TODO:");
let start_offset = source[line_offset..]
.chars()
.take(column)
.fold(line_offset, |acc, cur| acc + cur.len_utf8());
let start_offset = if need_strip_bom {
start_offset + 1
} else {
start_offset
};
TraceableError::from_file(
source.into_owned(),
// one character offset
start_offset,
start_offset + 1,
"Json parsing error".to_string(),
format!("Unexpected character {ch}"),
)
.with_kind(DiagnosticKind::Json)
.boxed()
// If there is a custom parse, execute it to obtain the returned string.
let parse_result_str = module_parser_options
.and_then(|p| p.get_json())
.and_then(|p| match &p.parse {
ParseOption::Func(p) => {
let parse_result = p(strip_bom_source.to_string());
parse_result.ok()
}
ExceededDepthLimit | WrongType(_) | FailedUtf8Parsing => diagnostic!("{e}").boxed(),
UnexpectedEndOfJson => {
// End offset of json file
let length = source.len();
let offset = if length > 0 { length - 1 } else { length };
TraceableError::from_file(
source.into_owned(),
offset,
offset,
"Json parsing error".to_string(),
format!("{e}"),
)
.with_kind(DiagnosticKind::Json)
.boxed()
_ => None,
});

let parse_result = json::parse(parse_result_str.as_deref().unwrap_or(strip_bom_source))
.map_err(|e| {
match e {
UnexpectedCharacter { ch, line, column } => {
let rope = ropey::Rope::from_str(&source);
let line_offset = rope.try_line_to_byte(line - 1).expect("TODO:");
let start_offset = source[line_offset..]
.chars()
.take(column)
.fold(line_offset, |acc, cur| acc + cur.len_utf8());
let start_offset = if need_strip_bom {
start_offset + 1
} else {
start_offset
};
TraceableError::from_file(
source.into_owned(),
// one character offset
start_offset,
start_offset + 1,
"Json parsing error".to_string(),
format!("Unexpected character {ch}"),
)
.with_kind(DiagnosticKind::Json)
.boxed()
}
ExceededDepthLimit | WrongType(_) | FailedUtf8Parsing => diagnostic!("{e}").boxed(),
UnexpectedEndOfJson => {
// End offset of json file
let length = source.len();
let offset = if length > 0 { length - 1 } else { length };
TraceableError::from_file(
source.into_owned(),
offset,
offset,
"Json parsing error".to_string(),
format!("{e}"),
)
.with_kind(DiagnosticKind::Json)
.boxed()
}
}
}
});
});

let (diagnostics, data) = match parse_result {
Ok(data) => (vec![], Some(data)),
Expand Down Expand Up @@ -236,6 +251,7 @@ impl Plugin for JsonPlugin {

Box::new(JsonParserAndGenerator {
exports_depth: p.exports_depth.expect("should have exports_depth"),
parse: p.parse.clone(),
})
}),
);
Expand Down
Loading
Loading