From 2e52ab4f566503adf78c5eb40177f755424d7703 Mon Sep 17 00:00:00 2001 From: hardfist Date: Fri, 9 Aug 2024 17:50:19 +0800 Subject: [PATCH 1/4] chore: increase publish limitation (#7515) --- Cargo.toml | 3 +++ justfile | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 justfile diff --git a/Cargo.toml b/Cargo.toml index 22bdc31b537..e783c9652ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -78,6 +78,9 @@ swc_html = { version = "=0.145.0" } swc_html_minifier = { version = "=0.142.2" } swc_node_comments = { version = "=0.23.0" } + +[workspace.metadata.release] +rate-limit = { existing-packages = 70, new-packages = 70 } [profile.dev] codegen-units = 16 debug = 2 # debug build will cause runtime panic if codegen-unints is default diff --git a/justfile b/justfile new file mode 100644 index 00000000000..f6773de8b92 --- /dev/null +++ b/justfile @@ -0,0 +1,6 @@ +# Setup the tools needed to develop +setup-tools: + cargo install cargo-release insta +# publish rust crates +release-crates: + cargo release publish --no-verify --execute --no-confirm From ba260c5a6c6269dbca00c3d6be6f31b67a92fdc0 Mon Sep 17 00:00:00 2001 From: Cong-Cong Pan Date: Fri, 9 Aug 2024 17:55:29 +0800 Subject: [PATCH 2/4] feat: support module.size function in cacheGroups.[i].test (#7482) * ModuleDTOSingleton * feat: use ModuleDTO in JsCacheGroupTestCtx * feat: JsCompilationSingleton * refactor: rename to JsCompilationWrapper * feat: clean COMPILATION_INSTANCE_REFS and MODULE_INSTANCE_REFS * rename: ModuleDTOWrapper * fix: ts type for ModuleDTOWrapper * fix: use add_env_cleanup_hook * test: add case * chore: update snapshot * fix * refactor: code in Rspack * chore: add SAFETY comment * fix: cargo lint * fix: problem in review * feat: cleanup_last_compilation * fix: remove unused files * chore: rmove dispose method * fix: cargo lint error --- crates/node_binding/binding.d.ts | 9 +- crates/node_binding/src/lib.rs | 9 +- .../node_binding/src/plugins/interceptor.rs | 88 ++++---------- .../raw_split_chunk_cache_group_test.rs | 31 +++-- .../src/compilation/mod.rs | 112 ++++++++++++++---- crates/rspack_binding_values/src/module.rs | 106 ++++++++++++++++- crates/rspack_binding_values/src/stats.rs | 4 +- .../src/options/cache_group_test.rs | 3 +- .../src/plugin/module_group.rs | 2 +- .../split-chunks/cache-group-test/index.js | 1 + .../cache-group-test/rspack.config.js | 21 ++++ packages/rspack/etc/api.md | 2 + packages/rspack/src/Module.ts | 7 ++ 13 files changed, 283 insertions(+), 112 deletions(-) create mode 100644 packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/index.js create mode 100644 packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/rspack.config.js diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts index 04eb6568e3a..193d3ad1b52 100644 --- a/crates/node_binding/binding.d.ts +++ b/crates/node_binding/binding.d.ts @@ -146,6 +146,7 @@ export class ModuleDto { get type(): string get layer(): string | undefined get blocks(): Array + size(ty?: string | undefined | null): number } export type ModuleDTO = ModuleDto @@ -351,6 +352,10 @@ export interface JsBuildTimeExecutionOption { baseUri?: string } +export interface JsCacheGroupTestCtx { + module: ModuleDTO +} + export interface JsChunk { __inner_ukey: number __inner_groups: Array @@ -993,10 +998,6 @@ export interface RawCacheGroupOptions { usedExports?: boolean } -export interface RawCacheGroupTestCtx { - module: JsModule -} - export interface RawCacheOptions { type: string maxGenerations: number diff --git a/crates/node_binding/src/lib.rs b/crates/node_binding/src/lib.rs index 599aca4615a..f78ba6ac59e 100644 --- a/crates/node_binding/src/lib.rs +++ b/crates/node_binding/src/lib.rs @@ -11,7 +11,7 @@ use std::sync::Mutex; use compiler::{Compiler, CompilerState, CompilerStateGuard}; use napi::bindgen_prelude::*; use rspack_binding_options::BuiltinPlugin; -use rspack_core::PluginExt; +use rspack_core::{Compilation, PluginExt}; use rspack_error::Diagnostic; use rspack_fs_node::{AsyncNodeWritableFileSystem, ThreadsafeNodeFS}; @@ -164,6 +164,9 @@ impl Rspack { // SAFETY: The mutable reference to `Compiler` is exclusive. It's guaranteed by the running state guard. Ok(unsafe { s.compiler.as_mut().get_unchecked_mut() }) })?; + + self.cleanup_last_compilation(&compiler.compilation); + // SAFETY: // 1. `Compiler` is pinned and stored on the heap. // 2. `JsReference` (NAPI internal mechanism) keeps `Compiler` alive until its instance getting garbage collected. @@ -172,6 +175,10 @@ impl Rspack { _guard, ) } + + fn cleanup_last_compilation(&self, compilation: &Compilation) { + JsCompilationWrapper::cleanup(compilation.id()); + } } fn concurrent_compiler_error() -> Error { diff --git a/crates/node_binding/src/plugins/interceptor.rs b/crates/node_binding/src/plugins/interceptor.rs index 76d92a35f0a..4c4044592b2 100644 --- a/crates/node_binding/src/plugins/interceptor.rs +++ b/crates/node_binding/src/plugins/interceptor.rs @@ -13,7 +13,7 @@ use napi::{ use rspack_binding_values::{ CompatSource, JsAdditionalTreeRuntimeRequirementsArg, JsAdditionalTreeRuntimeRequirementsResult, JsAfterResolveData, JsAfterResolveOutput, JsAssetEmittedArgs, JsBeforeResolveArgs, - JsBeforeResolveOutput, JsChunk, JsChunkAssetArgs, JsCompilation, + JsBeforeResolveOutput, JsChunk, JsChunkAssetArgs, JsCompilationWrapper, JsContextModuleFactoryAfterResolveData, JsContextModuleFactoryAfterResolveResult, JsContextModuleFactoryBeforeResolveData, JsContextModuleFactoryBeforeResolveResult, JsCreateData, JsExecuteModuleArg, JsFactorizeArgs, JsFactorizeOutput, JsModule, @@ -350,23 +350,23 @@ pub struct RegisterJsTaps { #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => void); stage: number; }>" )] - pub register_compiler_this_compilation_taps: RegisterFunction, + pub register_compiler_this_compilation_taps: RegisterFunction, #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => void); stage: number; }>" )] - pub register_compiler_compilation_taps: RegisterFunction, + pub register_compiler_compilation_taps: RegisterFunction, #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => Promise); stage: number; }>" )] - pub register_compiler_make_taps: RegisterFunction>, + pub register_compiler_make_taps: RegisterFunction>, #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => void); stage: number; }>" )] - pub register_compiler_finish_make_taps: RegisterFunction>, + pub register_compiler_finish_make_taps: RegisterFunction>, #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => boolean | undefined); stage: number; }>" )] - pub register_compiler_should_emit_taps: RegisterFunction>, + pub register_compiler_should_emit_taps: RegisterFunction>, #[napi( ts_type = "(stages: Array) => Array<{ function: (() => Promise); stage: number; }>" )] @@ -410,7 +410,7 @@ pub struct RegisterJsTaps { #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => Promise); stage: number; }>" )] - pub register_compilation_finish_modules_taps: RegisterFunction>, + pub register_compilation_finish_modules_taps: RegisterFunction>, #[napi( ts_type = "(stages: Array) => Array<{ function: (() => boolean | undefined); stage: number; }>" )] @@ -436,11 +436,11 @@ pub struct RegisterJsTaps { #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => Promise); stage: number; }>" )] - pub register_compilation_process_assets_taps: RegisterFunction>, + pub register_compilation_process_assets_taps: RegisterFunction>, #[napi( ts_type = "(stages: Array) => Array<{ function: ((arg: JsCompilation) => void); stage: number; }>" )] - pub register_compilation_after_process_assets_taps: RegisterFunction, + pub register_compilation_after_process_assets_taps: RegisterFunction, #[napi(ts_type = "(stages: Array) => Array<{ function: (() => void); stage: number; }>")] pub register_compilation_seal_taps: RegisterFunction<(), ()>, #[napi( @@ -500,7 +500,7 @@ pub struct RegisterJsTaps { /* Compiler Hooks */ define_register!( RegisterCompilerThisCompilationTaps, - tap = CompilerThisCompilationTap @ CompilerThisCompilationHook, + tap = CompilerThisCompilationTap @ CompilerThisCompilationHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilerThisCompilation, @@ -508,7 +508,7 @@ define_register!( ); define_register!( RegisterCompilerCompilationTaps, - tap = CompilerCompilationTap @ CompilerCompilationHook, + tap = CompilerCompilationTap @ CompilerCompilationHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilerCompilation, @@ -516,7 +516,7 @@ define_register!( ); define_register!( RegisterCompilerMakeTaps, - tap = CompilerMakeTap> @ CompilerMakeHook, + tap = CompilerMakeTap> @ CompilerMakeHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilerMake, @@ -524,7 +524,7 @@ define_register!( ); define_register!( RegisterCompilerFinishMakeTaps, - tap = CompilerFinishMakeTap> @ CompilerFinishMakeHook, + tap = CompilerFinishMakeTap> @ CompilerFinishMakeHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilerFinishMake, @@ -532,7 +532,7 @@ define_register!( ); define_register!( RegisterCompilerShouldEmitTaps, - tap = CompilerShouldEmitTap> @ CompilerShouldEmitHook, + tap = CompilerShouldEmitTap> @ CompilerShouldEmitHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilerShouldEmit, @@ -598,7 +598,7 @@ define_register!( ); define_register!( RegisterCompilationFinishModulesTaps, - tap = CompilationFinishModulesTap> @ CompilationFinishModulesHook, + tap = CompilationFinishModulesTap> @ CompilationFinishModulesHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilationFinishModules, @@ -670,7 +670,7 @@ define_register!( ); define_register!( RegisterCompilationProcessAssetsTaps, - tap = CompilationProcessAssetsTap> @ CompilationProcessAssetsHook, + tap = CompilationProcessAssetsTap> @ CompilationProcessAssetsHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilationProcessAssets, @@ -678,7 +678,7 @@ define_register!( ); define_register!( RegisterCompilationAfterProcessAssetsTaps, - tap = CompilationAfterProcessAssetsTap @ CompilationAfterProcessAssetsHook, + tap = CompilationAfterProcessAssetsTap @ CompilationAfterProcessAssetsHook, cache = false, sync = false, kind = RegisterJsTapKind::CompilationAfterProcessAssets, @@ -786,11 +786,7 @@ impl CompilerThisCompilation for CompilerThisCompilationTap { compilation: &mut Compilation, _: &mut CompilationParams, ) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_sync(compilation).await } @@ -806,11 +802,7 @@ impl CompilerCompilation for CompilerCompilationTap { compilation: &mut Compilation, _: &mut CompilationParams, ) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_sync(compilation).await } @@ -822,12 +814,7 @@ impl CompilerCompilation for CompilerCompilationTap { #[async_trait] impl CompilerMake for CompilerMakeTap { async fn run(&self, compilation: &mut Compilation) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; - + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_promise(compilation).await } @@ -839,12 +826,7 @@ impl CompilerMake for CompilerMakeTap { #[async_trait] impl CompilerFinishMake for CompilerFinishMakeTap { async fn run(&self, compilation: &mut Compilation) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; - + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_promise(compilation).await } @@ -856,12 +838,7 @@ impl CompilerFinishMake for CompilerFinishMakeTap { #[async_trait] impl CompilerShouldEmit for CompilerShouldEmitTap { async fn run(&self, compilation: &mut Compilation) -> rspack_error::Result> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; - + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_sync(compilation).await } @@ -982,12 +959,7 @@ impl CompilationExecuteModule for CompilationExecuteModuleTap { #[async_trait] impl CompilationFinishModules for CompilationFinishModulesTap { async fn run(&self, compilation: &mut Compilation) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; - + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_promise(compilation).await } @@ -1149,12 +1121,7 @@ impl CompilationChunkAsset for CompilationChunkAssetTap { #[async_trait] impl CompilationProcessAssets for CompilationProcessAssetsTap { async fn run(&self, compilation: &mut Compilation) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; - + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_promise(compilation).await } @@ -1166,12 +1133,7 @@ impl CompilationProcessAssets for CompilationProcessAssetsTap { #[async_trait] impl CompilationAfterProcessAssets for CompilationAfterProcessAssetsTap { async fn run(&self, compilation: &mut Compilation) -> rspack_error::Result<()> { - // SAFETY: - // 1. `Compiler` is stored on the heap and pinned in binding crate. - // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. - // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. - let compilation = unsafe { JsCompilation::from_compilation(compilation) }; - + let compilation = JsCompilationWrapper::new(compilation); self.function.call_with_sync(compilation).await } diff --git a/crates/rspack_binding_options/src/options/raw_split_chunks/raw_split_chunk_cache_group_test.rs b/crates/rspack_binding_options/src/options/raw_split_chunks/raw_split_chunk_cache_group_test.rs index b057f974cca..19e7d67572c 100644 --- a/crates/rspack_binding_options/src/options/raw_split_chunks/raw_split_chunk_cache_group_test.rs +++ b/crates/rspack_binding_options/src/options/raw_split_chunks/raw_split_chunk_cache_group_test.rs @@ -1,28 +1,35 @@ use std::sync::Arc; -use napi::bindgen_prelude::Either3; +use napi::bindgen_prelude::{Either3, FromNapiValue}; use napi_derive::napi; -use rspack_binding_values::{JsModule, ToJsModule}; +use rspack_binding_values::ModuleDTOWrapper; use rspack_napi::regexp::{JsRegExp, JsRegExpExt}; use rspack_napi::threadsafe_function::ThreadsafeFunction; use rspack_plugin_split_chunks::{CacheGroupTest, CacheGroupTestFnCtx}; use tokio::runtime::Handle; pub(super) type RawCacheGroupTest = - Either3>>; + Either3>>; -#[napi(object)] -pub struct RawCacheGroupTestCtx { - pub module: JsModule, +#[napi(object, object_from_js = false)] +pub struct JsCacheGroupTestCtx { + #[napi(ts_type = "ModuleDTO")] + pub module: ModuleDTOWrapper, } -impl<'a> From> for RawCacheGroupTestCtx { +impl FromNapiValue for JsCacheGroupTestCtx { + unsafe fn from_napi_value( + _env: napi::sys::napi_env, + _napi_val: napi::sys::napi_value, + ) -> napi::Result { + unreachable!() + } +} + +impl<'a> From> for JsCacheGroupTestCtx { fn from(value: CacheGroupTestFnCtx<'a>) -> Self { - RawCacheGroupTestCtx { - module: value - .module - .to_js_module() - .expect("should convert js module success"), + JsCacheGroupTestCtx { + module: ModuleDTOWrapper::new(value.module.identifier(), value.compilation), } } } diff --git a/crates/rspack_binding_values/src/compilation/mod.rs b/crates/rspack_binding_values/src/compilation/mod.rs index 6d097a398ab..6628d79d83c 100644 --- a/crates/rspack_binding_values/src/compilation/mod.rs +++ b/crates/rspack_binding_values/src/compilation/mod.rs @@ -1,6 +1,7 @@ mod dependency; mod entries; +use std::cell::RefCell; use std::collections::HashMap; use std::ops::Deref; use std::ops::DerefMut; @@ -15,46 +16,29 @@ use rspack_core::get_chunk_group_from_ukey; use rspack_core::rspack_sources::BoxSource; use rspack_core::rspack_sources::SourceExt; use rspack_core::AssetInfo; +use rspack_core::CompilationId; use rspack_core::ModuleIdentifier; use rspack_error::Diagnostic; use rspack_napi::napi::bindgen_prelude::*; use rspack_napi::NapiResultExt; +use rspack_napi::Ref; +use sys::napi_env; use super::module::ToJsModule; use super::{JsFilename, PathWithInfo}; use crate::utils::callbackify; use crate::JsStatsOptimizationBailout; use crate::LocalJsFilename; +use crate::ModuleDTOWrapper; use crate::{ chunk::JsChunk, CompatSource, JsAsset, JsAssetInfo, JsChunkGroup, JsCompatSource, JsPathData, - JsStats, ModuleDTO, ToJsCompatSource, + JsStats, ToJsCompatSource, }; use crate::{JsDiagnostic, JsRspackError}; #[napi] pub struct JsCompilation(pub(crate) &'static mut rspack_core::Compilation); -impl JsCompilation { - /// Convert Rust `Compilation` to `JsCompilation`. - /// - /// ## JS Interoperable - /// `JsCompilation` implements [napi::bindgen_prelude::ToNapiValue]. - /// It can be send to JavaScript. - /// - /// ## Safety - /// Safety is guaranteed by the following contracts: - /// 1. `Compiler` should not be moved. For example: store it on the heap. - /// 2. The pointer should be valid for the entire lifetime of `JsCompilation`. - /// 3. Caching old `Compilation` will result the program to undefined behavior and it's likely to crash. - pub unsafe fn from_compilation(inner: &mut rspack_core::Compilation) -> Self { - Self(unsafe { - std::mem::transmute::<&'_ mut rspack_core::Compilation, &'static mut rspack_core::Compilation>( - inner, - ) - }) - } -} - impl Deref for JsCompilation { type Target = rspack_core::Compilation; @@ -151,15 +135,15 @@ impl JsCompilation { .transpose() } - #[napi(getter)] - pub fn modules(&'static self) -> Vec { + #[napi(getter, ts_return_type = "Array")] + pub fn modules(&'static self) -> Vec { self .0 .get_module_graph() .modules() .keys() .cloned() - .map(|module_id| ModuleDTO::new(module_id, self.0)) + .map(|module_id| ModuleDTOWrapper::new(module_id, self.0)) .collect::>() } @@ -578,6 +562,84 @@ impl JsCompilation { } } +#[derive(Default)] +struct CompilationInstanceRefs(RefCell>); + +impl Drop for CompilationInstanceRefs { + fn drop(&mut self) { + // cleanup references to be executed in cases of panic or unexpected termination + let mut refs = self.0.borrow_mut(); + for (_, (mut r, env)) in refs.drain() { + let _ = r.unref(env); + } + } +} + +thread_local! { + static COMPILATION_INSTANCE_REFS: CompilationInstanceRefs = Default::default(); +} + +// The difference between JsCompilationWrapper and JsCompilation is: +// JsCompilationWrapper maintains a cache to ensure that the corresponding instance of the same Compilation is unique on the JS side. +// +// This means that when transferring a JsCompilation from Rust to JS, you must use JsCompilationWrapper instead. +pub struct JsCompilationWrapper(pub(crate) &'static mut rspack_core::Compilation); + +impl JsCompilationWrapper { + pub fn new(compilation: &mut rspack_core::Compilation) -> Self { + // SAFETY: + // 1. `Compiler` is stored on the heap and pinned in binding crate. + // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. + // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. + Self(unsafe { + std::mem::transmute::<&'_ mut rspack_core::Compilation, &'static mut rspack_core::Compilation>( + compilation, + ) + }) + } + + pub fn cleanup(compilation_id: CompilationId) { + COMPILATION_INSTANCE_REFS.with(|ref_cell| { + let mut refs = ref_cell.0.borrow_mut(); + if let Some((mut r, env)) = refs.remove(&compilation_id) { + let _ = r.unref(env); + } + }); + ModuleDTOWrapper::cleanup(compilation_id); + } +} + +impl ToNapiValue for JsCompilationWrapper { + unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result { + COMPILATION_INSTANCE_REFS.with(|ref_cell| { + let mut env_wrapper = Env::from_raw(env); + let mut refs = ref_cell.0.borrow_mut(); + let compilation_id = val.0.id(); + let mut vacant = false; + let napi_value = match refs.entry(compilation_id) { + std::collections::hash_map::Entry::Occupied(entry) => { + let r = entry.get(); + ToNapiValue::to_napi_value(env, &r.0) + } + std::collections::hash_map::Entry::Vacant(entry) => { + vacant = true; + let instance = JsCompilation(val.0).into_instance(env_wrapper)?; + let napi_value = ToNapiValue::to_napi_value(env, instance)?; + let r = Ref::new(env, napi_value, 1)?; + let r = entry.insert((r, env)); + ToNapiValue::to_napi_value(env, &r.0) + } + }; + if vacant { + // cleanup references to be executed when the JS thread exits normally + let _ = env_wrapper + .add_env_cleanup_hook((), move |_| JsCompilationWrapper::cleanup(compilation_id)); + } + napi_value + }) + } +} + #[napi(object)] pub struct JsExecuteModuleResult { pub file_dependencies: Vec, diff --git a/crates/rspack_binding_values/src/module.rs b/crates/rspack_binding_values/src/module.rs index 96942016233..887927b6c30 100644 --- a/crates/rspack_binding_values/src/module.rs +++ b/crates/rspack_binding_values/src/module.rs @@ -1,9 +1,14 @@ +use std::cell::RefCell; + use napi_derive::napi; +use rspack_collections::Identifier; use rspack_core::{ - AsyncDependenciesBlock, AsyncDependenciesBlockIdentifier, Compilation, CompilerModuleContext, - DependenciesBlock, Module, ModuleGraph, ModuleIdentifier, + AsyncDependenciesBlock, AsyncDependenciesBlockIdentifier, Compilation, CompilationId, + CompilerModuleContext, DependenciesBlock, Module, ModuleGraph, ModuleIdentifier, SourceType, }; -use rspack_napi::napi::bindgen_prelude::*; +use rspack_napi::{napi::bindgen_prelude::*, Ref}; +use rustc_hash::FxHashMap as HashMap; +use sys::napi_env; use super::{JsCompatSource, ToJsCompatSource}; use crate::{DependencyDTO, JsChunk, JsCodegenerationResults}; @@ -209,6 +214,101 @@ impl ModuleDTO { .map(|block_id| DependenciesBlockDTO::new(block_id, self.compilation)) .collect::>() } + + #[napi] + pub fn size(&self, ty: Option) -> f64 { + let module = self.module(); + let ty = ty.map(|s| SourceType::from(s.as_str())); + module.size(ty.as_ref(), self.compilation) + } +} + +type ModuleInstanceRefs = HashMap; + +#[derive(Default)] +struct ModuleInstanceRefsByCompilationId(RefCell>); + +impl Drop for ModuleInstanceRefsByCompilationId { + fn drop(&mut self) { + let mut refs_by_compilation_id = self.0.borrow_mut(); + for (_, mut refs) in refs_by_compilation_id.drain() { + for (_, (mut r, env)) in refs.drain() { + let _ = r.unref(env); + } + } + } +} + +thread_local! { + static MODULE_INSTANCE_REFS: ModuleInstanceRefsByCompilationId = Default::default(); +} + +// The difference between ModuleDTOWrapper and ModuleDTO is: +// ModuleDTOWrapper maintains a cache to ensure that the corresponding instance of the same Module is unique on the JS side. +// +// This means that when transferring a ModuleDTO from Rust to JS, you must use ModuleDTOWrapper instead. +pub struct ModuleDTOWrapper { + pub module_id: ModuleIdentifier, + pub compilation: &'static Compilation, +} + +impl ModuleDTOWrapper { + pub fn new(module_id: ModuleIdentifier, compilation: &Compilation) -> Self { + // SAFETY: + // 1. `Compiler` is stored on the heap and pinned in binding crate. + // 2. `Compilation` outlives `JsCompilation` and `Compiler` outlives `Compilation`. + // 3. `JsCompilation` was replaced everytime a new `Compilation` was created before getting accessed. + let compilation = unsafe { + std::mem::transmute::<&rspack_core::Compilation, &'static rspack_core::Compilation>( + compilation, + ) + }; + Self { + module_id, + compilation, + } + } + + pub fn cleanup(compilation_id: CompilationId) { + MODULE_INSTANCE_REFS.with(|refs| { + let mut refs_by_compilation_id = refs.0.borrow_mut(); + if let Some(mut refs) = refs_by_compilation_id.remove(&compilation_id) { + for (_, (mut r, env)) in refs.drain() { + let _ = r.unref(env); + } + } + }); + } +} + +impl ToNapiValue for ModuleDTOWrapper { + unsafe fn to_napi_value(env: sys::napi_env, val: Self) -> Result { + MODULE_INSTANCE_REFS.with(|refs| { + let mut refs_by_compilation_id = refs.0.borrow_mut(); + let entry = refs_by_compilation_id.entry(val.compilation.id()); + let refs = match entry { + std::collections::hash_map::Entry::Occupied(entry) => entry.into_mut(), + std::collections::hash_map::Entry::Vacant(entry) => { + let refs = HashMap::default(); + entry.insert(refs) + } + }; + match refs.entry(val.module_id) { + std::collections::hash_map::Entry::Occupied(entry) => { + let r = entry.get(); + ToNapiValue::to_napi_value(env, &r.0) + } + std::collections::hash_map::Entry::Vacant(entry) => { + let instance = + ModuleDTO::new(val.module_id, val.compilation).into_instance(Env::from_raw(env))?; + let napi_value = ToNapiValue::to_napi_value(env, instance)?; + let r = Ref::new(env, napi_value, 1)?; + let r = entry.insert((r, env)); + ToNapiValue::to_napi_value(env, &r.0) + } + } + }) + } } #[derive(Default)] diff --git a/crates/rspack_binding_values/src/stats.rs b/crates/rspack_binding_values/src/stats.rs index 66e1204048f..a4a6226ad86 100644 --- a/crates/rspack_binding_values/src/stats.rs +++ b/crates/rspack_binding_values/src/stats.rs @@ -14,8 +14,8 @@ use rspack_napi::{ }; use rustc_hash::FxHashMap as HashMap; -use super::{JsCompilation, ToJsCompatSource}; -use crate::identifier::JsIdentifier; +use super::ToJsCompatSource; +use crate::{identifier::JsIdentifier, JsCompilation}; thread_local! { static MODULE_DESCRIPTOR_REFS: RefCell> = Default::default(); diff --git a/crates/rspack_plugin_split_chunks/src/options/cache_group_test.rs b/crates/rspack_plugin_split_chunks/src/options/cache_group_test.rs index f105d1aac26..47ffeefb969 100644 --- a/crates/rspack_plugin_split_chunks/src/options/cache_group_test.rs +++ b/crates/rspack_plugin_split_chunks/src/options/cache_group_test.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use rspack_core::Module; +use rspack_core::{Compilation, Module}; use rspack_error::Result; pub struct CacheGroupTestFnCtx<'a> { + pub compilation: &'a Compilation, pub module: &'a dyn Module, } diff --git a/crates/rspack_plugin_split_chunks/src/plugin/module_group.rs b/crates/rspack_plugin_split_chunks/src/plugin/module_group.rs index d7ba5b5bea3..707c2b99f4d 100644 --- a/crates/rspack_plugin_split_chunks/src/plugin/module_group.rs +++ b/crates/rspack_plugin_split_chunks/src/plugin/module_group.rs @@ -334,7 +334,7 @@ impl SplitChunksPlugin { .name_for_condition() .map_or(false, |name| regexp.test(&name)), CacheGroupTest::Fn(f) => { - let ctx = CacheGroupTestFnCtx { module }; + let ctx = CacheGroupTestFnCtx { compilation, module }; f(ctx)?.unwrap_or_default() } CacheGroupTest::Enabled => true, diff --git a/packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/index.js b/packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/index.js new file mode 100644 index 00000000000..7e9668e7834 --- /dev/null +++ b/packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/index.js @@ -0,0 +1 @@ +"foo" \ No newline at end of file diff --git a/packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/rspack.config.js b/packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/rspack.config.js new file mode 100644 index 00000000000..781158e3e90 --- /dev/null +++ b/packages/rspack-test-tools/tests/configCases/split-chunks/cache-group-test/rspack.config.js @@ -0,0 +1,21 @@ +/** @type {import("@rspack/core").Configuration} */ +module.exports = { + entry: "./index.js", + target: "node", + output: { + filename: "[name].js" + }, + optimization: { + splitChunks: { + cacheGroups: { + common: { + test(module) { + expect(module.size()).toBe(5); + return true; + } + } + } + } + } + +}; diff --git a/packages/rspack/etc/api.md b/packages/rspack/etc/api.md index f40af894ed1..b3880c90368 100644 --- a/packages/rspack/etc/api.md +++ b/packages/rspack/etc/api.md @@ -6077,6 +6077,8 @@ export class Module { // (undocumented) resource?: Readonly; // (undocumented) + size(type?: string): number; + // (undocumented) type: string; // (undocumented) userRequest?: Readonly; diff --git a/packages/rspack/src/Module.ts b/packages/rspack/src/Module.ts index c987ea25a61..5c474a7c9f4 100644 --- a/packages/rspack/src/Module.ts +++ b/packages/rspack/src/Module.ts @@ -131,6 +131,13 @@ export class Module { } return []; } + + size(type?: string): number { + if ("size" in this.#inner) { + return this.#inner.size(type); + } + return 0; + } } export class CodeGenerationResult { From d49b34f77c34c4a1eecdf6788a4059ddf24ad38b Mon Sep 17 00:00:00 2001 From: neverland Date: Fri, 9 Aug 2024 21:15:01 +0800 Subject: [PATCH 3/4] docs: add MarsCode to online examples (#7525) * docs: add MarsCode to online examples * docs: fix * chore: fix --- website/docs/en/guide/start/quick-start.mdx | 2 +- website/docs/zh/guide/start/quick-start.mdx | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/website/docs/en/guide/start/quick-start.mdx b/website/docs/en/guide/start/quick-start.mdx index a233dbf1803..bcf34aebdc6 100644 --- a/website/docs/en/guide/start/quick-start.mdx +++ b/website/docs/en/guide/start/quick-start.mdx @@ -55,7 +55,7 @@ Then follow the prompts in your terminal. We provide an online example based on Rsbuild. The example gives an intuitive feel for the build performance of Rspack and the development experience of Rsbuild: -- [Rsbuild codesandbox example](https://codesandbox.io/p/github/rspack-contrib/rsbuild-codesandbox-example) +- [Rsbuild CodeSandbox example](https://codesandbox.io/p/github/rspack-contrib/rsbuild-codesandbox-example) ## Manual installation diff --git a/website/docs/zh/guide/start/quick-start.mdx b/website/docs/zh/guide/start/quick-start.mdx index cc3f547d31d..3f932ff81b5 100644 --- a/website/docs/zh/guide/start/quick-start.mdx +++ b/website/docs/zh/guide/start/quick-start.mdx @@ -55,7 +55,12 @@ Rspack CLI 是对标 Webpack CLI 的工具，提供基础的 `serve` 和 `build` 我们提供了基于 Rsbuild 的在线示例，通过示例项目，你可以直观感受 Rspack 的构建性能和 Rsbuild 的开发体验： -- [Rsbuild codesandbox 示例](https://codesandbox.io/p/github/rspack-contrib/rsbuild-codesandbox-example) +- [Rsbuild CodeSandbox 示例](https://codesandbox.io/p/github/rspack-contrib/rsbuild-codesandbox-example) + +你也可以使用 [豆包 MarsCode](https://www.marscode.cn/) 快速创建一个在线的 Rsbuild 项目： + +1. 访问 [MarsCode - Rsbuild 模板](https://www.marscode.cn/dashboard/template-detail?id=8e8qr0k282d6wn)。 +2. 登录，点击右上角的「使用模板」按钮即可。 ## 手动安装 From 3867fe0279d0e2950ce8650ae56f3fd12fff1b04 Mon Sep 17 00:00:00 2001 From: jinrui Date: Sat, 10 Aug 2024 00:30:31 +0800 Subject: [PATCH 4/4] feat: expose added/removed compilation.*_dependencies to js side (#7522) --- crates/node_binding/binding.d.ts | 29 ++-- .../src/compilation/dependencies.rs | 128 ++++++++++++++++++ .../src/compilation/entries.rs | 3 +- .../src/compilation/mod.rs | 39 +----- .../src/{compilation => }/dependency.rs | 0 crates/rspack_binding_values/src/lib.rs | 2 + .../rspack_core/src/compiler/compilation.rs | 126 ++++++++++------- crates/rspack_core/src/compiler/make/mod.rs | 14 +- .../src/compiler/module_executor/mod.rs | 18 +++ .../utils/file_counter/incremental_info.rs | 69 ++++++++++ .../file_counter/mod.rs} | 55 ++++++++ crates/rspack_core/src/utils/mod.rs | 2 + packages/rspack/src/Compilation.ts | 8 +- 13 files changed, 393 insertions(+), 100 deletions(-) create mode 100644 crates/rspack_binding_values/src/compilation/dependencies.rs rename crates/rspack_binding_values/src/{compilation => }/dependency.rs (100%) create mode 100644 crates/rspack_core/src/utils/file_counter/incremental_info.rs rename crates/rspack_core/src/{compiler/make/file_counter.rs => utils/file_counter/mod.rs} (61%) diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts index 193d3ad1b52..5b6482aef81 100644 --- a/crates/node_binding/binding.d.ts +++ b/crates/node_binding/binding.d.ts @@ -25,6 +25,22 @@ export class DependenciesBlockDto { } export type DependenciesBlockDTO = DependenciesBlockDto +export class DependenciesDto { + get fileDependencies(): Array + get addedFileDependencies(): Array + get removedFileDependencies(): Array + get contextDependencies(): Array + get addedContextDependencies(): Array + get removedContextDependencies(): Array + get missingDependencies(): Array + get addedMissingDependencies(): Array + get removedMissingDependencies(): Array + get buildDependencies(): Array + get addedBuildDependencies(): Array + get removedBuildDependencies(): Array +} +export type DependenciesDTO = DependenciesDto + export class DependencyDto { get type(): string get category(): string @@ -33,8 +49,8 @@ export class DependencyDto { export type DependencyDTO = DependencyDto export class EntryDataDto { - get dependencies(): Array - get includeDependencies(): Array + get dependencies(): Array + get includeDependencies(): Array get options(): EntryOptionsDto } export type EntryDataDTO = EntryDataDto @@ -82,10 +98,7 @@ export class JsCompilation { get entrypoints(): Record get chunkGroups(): Array get hash(): string | null - getFileDependencies(): Array - getContextDependencies(): Array - getMissingDependencies(): Array - getBuildDependencies(): Array + dependencies(): DependenciesDto pushDiagnostic(diagnostic: JsDiagnostic): void spliceDiagnostic(start: number, end: number, replaceWith: Array): void pushNativeDiagnostics(diagnostics: ExternalObject<'Diagnostic[]'>): void @@ -436,8 +449,8 @@ export interface JsDiagnostic { } export interface JsEntryData { - dependencies: Array - includeDependencies: Array + dependencies: Array + includeDependencies: Array options: JsEntryOptions } diff --git a/crates/rspack_binding_values/src/compilation/dependencies.rs b/crates/rspack_binding_values/src/compilation/dependencies.rs new file mode 100644 index 00000000000..84236b47587 --- /dev/null +++ b/crates/rspack_binding_values/src/compilation/dependencies.rs @@ -0,0 +1,128 @@ +use napi_derive::napi; +use rspack_core::Compilation; + +#[napi] +pub struct DependenciesDTO { + pub(crate) compilation: &'static Compilation, +} + +impl DependenciesDTO { + pub(crate) fn new(compilation: &'static Compilation) -> Self { + Self { compilation } + } +} + +#[napi] +impl DependenciesDTO { + #[napi(getter)] + pub fn file_dependencies(&self) -> Vec { + self + .compilation + .file_dependencies() + .0 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn added_file_dependencies(&self) -> Vec { + self + .compilation + .file_dependencies() + .1 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn removed_file_dependencies(&self) -> Vec { + self + .compilation + .file_dependencies() + .2 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + + #[napi(getter)] + pub fn context_dependencies(&self) -> Vec { + self + .compilation + .context_dependencies() + .0 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn added_context_dependencies(&self) -> Vec { + self + .compilation + .context_dependencies() + .1 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn removed_context_dependencies(&self) -> Vec { + self + .compilation + .context_dependencies() + .2 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + + #[napi(getter)] + pub fn missing_dependencies(&self) -> Vec { + self + .compilation + .missing_dependencies() + .0 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn added_missing_dependencies(&self) -> Vec { + self + .compilation + .missing_dependencies() + .1 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn removed_missing_dependencies(&self) -> Vec { + self + .compilation + .missing_dependencies() + .2 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + + #[napi(getter)] + pub fn build_dependencies(&self) -> Vec { + self + .compilation + .build_dependencies() + .0 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn added_build_dependencies(&self) -> Vec { + self + .compilation + .build_dependencies() + .1 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } + #[napi(getter)] + pub fn removed_build_dependencies(&self) -> Vec { + self + .compilation + .build_dependencies() + .2 + .map(|i| i.to_string_lossy().to_string()) + .collect() + } +} diff --git a/crates/rspack_binding_values/src/compilation/entries.rs b/crates/rspack_binding_values/src/compilation/entries.rs index 2bbbf24a443..3ae57f39501 100644 --- a/crates/rspack_binding_values/src/compilation/entries.rs +++ b/crates/rspack_binding_values/src/compilation/entries.rs @@ -2,8 +2,7 @@ use napi_derive::napi; use rspack_core::{ChunkLoading, Compilation, EntryData, EntryOptions, EntryRuntime}; use rspack_napi::napi::bindgen_prelude::*; -use super::dependency::DependencyDTO; -use crate::{entry::JsEntryOptions, library::JsLibraryOptions}; +use crate::{dependency::DependencyDTO, entry::JsEntryOptions, library::JsLibraryOptions}; #[napi] pub struct EntryOptionsDTO(EntryOptions); diff --git a/crates/rspack_binding_values/src/compilation/mod.rs b/crates/rspack_binding_values/src/compilation/mod.rs index 6628d79d83c..231eb9e3153 100644 --- a/crates/rspack_binding_values/src/compilation/mod.rs +++ b/crates/rspack_binding_values/src/compilation/mod.rs @@ -1,4 +1,4 @@ -mod dependency; +mod dependencies; mod entries; use std::cell::RefCell; @@ -7,7 +7,7 @@ use std::ops::Deref; use std::ops::DerefMut; use std::path::PathBuf; -pub use dependency::*; +use dependencies::DependenciesDTO; use entries::JsEntries; use napi_derive::napi; use rspack_collections::IdentifierSet; @@ -305,39 +305,8 @@ impl JsCompilation { } #[napi] - pub fn get_file_dependencies(&self) -> Vec { - self - .0 - .file_dependencies() - .map(|i| i.to_string_lossy().to_string()) - .collect() - } - - #[napi] - pub fn get_context_dependencies(&self) -> Vec { - self - .0 - .context_dependencies() - .map(|i| i.to_string_lossy().to_string()) - .collect() - } - - #[napi] - pub fn get_missing_dependencies(&self) -> Vec { - self - .0 - .missing_dependencies() - .map(|i| i.to_string_lossy().to_string()) - .collect() - } - - #[napi] - pub fn get_build_dependencies(&self) -> Vec { - self - .0 - .build_dependencies() - .map(|i| i.to_string_lossy().to_string()) - .collect() + pub fn dependencies(&'static self) -> DependenciesDTO { + DependenciesDTO::new(self.0) } #[napi] diff --git a/crates/rspack_binding_values/src/compilation/dependency.rs b/crates/rspack_binding_values/src/dependency.rs similarity index 100% rename from crates/rspack_binding_values/src/compilation/dependency.rs rename to crates/rspack_binding_values/src/dependency.rs diff --git a/crates/rspack_binding_values/src/lib.rs b/crates/rspack_binding_values/src/lib.rs index 8802c2dc2db..a260013bc9a 100644 --- a/crates/rspack_binding_values/src/lib.rs +++ b/crates/rspack_binding_values/src/lib.rs @@ -7,6 +7,7 @@ mod chunk_group; mod codegen_result; mod compilation; mod context_module_factory; +mod dependency; mod filename; mod identifier; mod module; @@ -30,6 +31,7 @@ pub use chunk_group::*; pub use codegen_result::*; pub use compilation::*; pub use context_module_factory::*; +pub use dependency::DependencyDTO; pub use filename::*; pub use module::*; pub use normal_module_factory::*; diff --git a/crates/rspack_core/src/compiler/compilation.rs b/crates/rspack_core/src/compiler/compilation.rs index 835d30750e5..fa1a397b555 100644 --- a/crates/rspack_core/src/compiler/compilation.rs +++ b/crates/rspack_core/src/compiler/compilation.rs @@ -329,72 +329,100 @@ impl Compilation { } } - pub fn file_dependencies(&self) -> impl Iterator { - self + pub fn file_dependencies( + &self, + ) -> ( + impl Iterator, + impl Iterator, + impl Iterator, + ) { + let all_files = self .make_artifact .file_dependencies .files() - .chain( - self - .module_executor - .as_ref() - .expect("should have module_executor") - .make_artifact - .file_dependencies - .files(), - ) - .chain(&self.file_dependencies) + .chain(&self.file_dependencies); + let added_files = self + .make_artifact + .file_dependencies + .added_files() + .iter() + .chain(&self.file_dependencies); + let removed_files = self.make_artifact.file_dependencies.removed_files().iter(); + (all_files, added_files, removed_files) } - pub fn context_dependencies(&self) -> impl Iterator { - self + pub fn context_dependencies( + &self, + ) -> ( + impl Iterator, + impl Iterator, + impl Iterator, + ) { + let all_files = self .make_artifact .context_dependencies .files() - .chain( - self - .module_executor - .as_ref() - .expect("should have module_executor") - .make_artifact - .context_dependencies - .files(), - ) - .chain(&self.context_dependencies) + .chain(&self.context_dependencies); + let added_files = self + .make_artifact + .context_dependencies + .added_files() + .iter() + .chain(&self.file_dependencies); + let removed_files = self + .make_artifact + .context_dependencies + .removed_files() + .iter(); + (all_files, added_files, removed_files) } - pub fn missing_dependencies(&self) -> impl Iterator { - self + pub fn missing_dependencies( + &self, + ) -> ( + impl Iterator, + impl Iterator, + impl Iterator, + ) { + let all_files = self .make_artifact .missing_dependencies .files() - .chain( - self - .module_executor - .as_ref() - .expect("should have module_executor") - .make_artifact - .missing_dependencies - .files(), - ) - .chain(&self.missing_dependencies) + .chain(&self.missing_dependencies); + let added_files = self + .make_artifact + .missing_dependencies + .added_files() + .iter() + .chain(&self.file_dependencies); + let removed_files = self + .make_artifact + .missing_dependencies + .removed_files() + .iter(); + (all_files, added_files, removed_files) } - pub fn build_dependencies(&self) -> impl Iterator { - self + pub fn build_dependencies( + &self, + ) -> ( + impl Iterator, + impl Iterator, + impl Iterator, + ) { + let all_files = self .make_artifact .build_dependencies .files() - .chain( - self - .module_executor - .as_ref() - .expect("should have module_executor") - .make_artifact - .build_dependencies - .files(), - ) - .chain(&self.build_dependencies) + .chain(&self.build_dependencies); + let added_files = self + .make_artifact + .build_dependencies + .added_files() + .iter() + .chain(&self.file_dependencies); + let removed_files = self.make_artifact.build_dependencies.removed_files().iter(); + (all_files, added_files, removed_files) } // TODO move out from compilation @@ -683,6 +711,8 @@ impl Compilation { #[instrument(name = "compilation:make", skip_all)] pub async fn make(&mut self) -> Result<()> { + self.make_artifact.reset_dependencies_incremental_info(); + // self.module_executor. // run module_executor if let Some(module_executor) = &mut self.module_executor { let mut module_executor = std::mem::take(module_executor); diff --git a/crates/rspack_core/src/compiler/make/mod.rs b/crates/rspack_core/src/compiler/make/mod.rs index 1e2bdbbf0a6..156dcd7c81f 100644 --- a/crates/rspack_core/src/compiler/make/mod.rs +++ b/crates/rspack_core/src/compiler/make/mod.rs @@ -1,5 +1,4 @@ mod cutout; -mod file_counter; pub mod repair; use std::path::PathBuf; @@ -8,8 +7,10 @@ use rspack_collections::IdentifierSet; use rspack_error::{Diagnostic, Result}; use rustc_hash::FxHashSet as HashSet; -use self::{cutout::Cutout, file_counter::FileCounter, repair::repair}; -use crate::{BuildDependency, Compilation, DependencyId, ModuleGraph, ModuleGraphPartial}; +use self::{cutout::Cutout, repair::repair}; +use crate::{ + utils::FileCounter, BuildDependency, Compilation, DependencyId, ModuleGraph, ModuleGraphPartial, +}; #[derive(Debug, Default)] pub struct MakeArtifact { @@ -79,6 +80,13 @@ impl MakeArtifact { } res } + + pub fn reset_dependencies_incremental_info(&mut self) { + self.file_dependencies.reset_incremental_info(); + self.context_dependencies.reset_incremental_info(); + self.missing_dependencies.reset_incremental_info(); + self.build_dependencies.reset_incremental_info(); + } } #[derive(Debug, Clone)] diff --git a/crates/rspack_core/src/compiler/module_executor/mod.rs b/crates/rspack_core/src/compiler/module_executor/mod.rs index 012bea692b4..34292fb6b21 100644 --- a/crates/rspack_core/src/compiler/module_executor/mod.rs +++ b/crates/rspack_core/src/compiler/module_executor/mod.rs @@ -143,6 +143,24 @@ impl ModuleExecutor { for id in code_generated_modules { compilation.code_generated_modules.insert(id); } + + // remove useless *_dependencies incremental info + self + .make_artifact + .file_dependencies + .reset_incremental_info(); + self + .make_artifact + .context_dependencies + .reset_incremental_info(); + self + .make_artifact + .missing_dependencies + .reset_incremental_info(); + self + .make_artifact + .build_dependencies + .reset_incremental_info(); } #[allow(clippy::too_many_arguments)] diff --git a/crates/rspack_core/src/utils/file_counter/incremental_info.rs b/crates/rspack_core/src/utils/file_counter/incremental_info.rs new file mode 100644 index 00000000000..c84e9b9a1d4 --- /dev/null +++ b/crates/rspack_core/src/utils/file_counter/incremental_info.rs @@ -0,0 +1,69 @@ +use std::path::PathBuf; + +use rustc_hash::FxHashSet as HashSet; + +/// Used to collect file add or remove. +#[derive(Debug, Default)] +pub struct IncrementalInfo { + added_files: HashSet, + removed_files: HashSet, +} + +impl IncrementalInfo { + /// Get added files + pub fn added_files(&self) -> &HashSet { + &self.added_files + } + + /// Get removed files + pub fn removed_files(&self) -> &HashSet { + &self.removed_files + } + + /// Add a file + pub fn add(&mut self, path: &PathBuf) { + if !self.removed_files.remove(path) { + self.added_files.insert(path.clone()); + } + } + + /// Remove a file + pub fn remove(&mut self, path: &PathBuf) { + if !self.added_files.remove(path) { + self.removed_files.insert(path.clone()); + } + } + + /// Reset added and removed files + pub fn reset(&mut self) { + self.added_files.clear(); + self.removed_files.clear(); + } +} + +#[cfg(test)] +mod test { + use super::IncrementalInfo; + #[test] + fn incremental_info_is_available() { + let mut info = IncrementalInfo::default(); + let file_a = std::path::PathBuf::from("/a"); + + info.add(&file_a); + info.add(&file_a); + assert_eq!(info.added_files().len(), 1); + assert_eq!(info.removed_files().len(), 0); + + info.remove(&file_a); + assert_eq!(info.added_files().len(), 0); + assert_eq!(info.removed_files().len(), 0); + + info.remove(&file_a); + assert_eq!(info.added_files().len(), 0); + assert_eq!(info.removed_files().len(), 1); + + info.remove(&file_a); + assert_eq!(info.added_files().len(), 0); + assert_eq!(info.removed_files().len(), 1); + } +} diff --git a/crates/rspack_core/src/compiler/make/file_counter.rs b/crates/rspack_core/src/utils/file_counter/mod.rs similarity index 61% rename from crates/rspack_core/src/compiler/make/file_counter.rs rename to crates/rspack_core/src/utils/file_counter/mod.rs index b8d5c821121..c5e62500f73 100644 --- a/crates/rspack_core/src/compiler/make/file_counter.rs +++ b/crates/rspack_core/src/utils/file_counter/mod.rs @@ -1,11 +1,15 @@ +mod incremental_info; + use std::path::PathBuf; +use incremental_info::IncrementalInfo; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; /// Used to count file usage #[derive(Debug, Default)] pub struct FileCounter { inner: HashMap, + incremental_info: IncrementalInfo, } impl FileCounter { @@ -16,6 +20,7 @@ impl FileCounter { if let Some(value) = self.inner.get_mut(path) { *value += 1; } else { + self.incremental_info.add(path); self.inner.insert(path.clone(), 1); } } @@ -30,6 +35,7 @@ impl FileCounter { if let Some(value) = self.inner.get_mut(path) { *value -= 1; if value == &0 { + self.incremental_info.remove(path); self.inner.remove(path); } } else { @@ -55,6 +61,21 @@ impl FileCounter { pub fn files(&self) -> impl Iterator { self.inner.keys() } + + /// reset incremental info + pub fn reset_incremental_info(&mut self) { + self.incremental_info.reset() + } + + /// Added files compared to the `files()` when call reset_incremental_info + pub fn added_files(&self) -> &HashSet { + self.incremental_info.added_files() + } + + /// Removed files compared to the `files()` when call reset_incremental_info + pub fn removed_files(&self) -> &HashSet { + self.incremental_info.removed_files() + } } #[cfg(test)] @@ -70,15 +91,23 @@ mod test { counter.add_file(&file_a); counter.add_file(&file_b); assert_eq!(counter.files().collect::>().len(), 2); + assert_eq!(counter.added_files().len(), 2); + assert_eq!(counter.removed_files().len(), 0); counter.remove_file(&file_a); assert_eq!(counter.files().collect::>().len(), 2); + assert_eq!(counter.added_files().len(), 2); + assert_eq!(counter.removed_files().len(), 0); counter.remove_file(&file_b); assert_eq!(counter.files().collect::>().len(), 1); + assert_eq!(counter.added_files().len(), 1); + assert_eq!(counter.removed_files().len(), 0); counter.remove_file(&file_a); assert_eq!(counter.files().collect::>().len(), 0); + assert_eq!(counter.added_files().len(), 0); + assert_eq!(counter.removed_files().len(), 0); } #[test] @@ -112,4 +141,30 @@ mod test { let file_a = std::path::PathBuf::from("/a"); counter.remove_file(&file_a); } + + #[test] + fn file_counter_reset_incremental_info() { + let mut counter = FileCounter::default(); + let file_a = std::path::PathBuf::from("/a"); + + counter.add_file(&file_a); + assert_eq!(counter.added_files().len(), 1); + assert_eq!(counter.removed_files().len(), 0); + + counter.reset_incremental_info(); + assert_eq!(counter.added_files().len(), 0); + assert_eq!(counter.removed_files().len(), 0); + + counter.add_file(&file_a); + assert_eq!(counter.added_files().len(), 0); + assert_eq!(counter.removed_files().len(), 0); + + counter.remove_file(&file_a); + assert_eq!(counter.added_files().len(), 0); + assert_eq!(counter.removed_files().len(), 0); + + counter.remove_file(&file_a); + assert_eq!(counter.added_files().len(), 0); + assert_eq!(counter.removed_files().len(), 1); + } } diff --git a/crates/rspack_core/src/utils/mod.rs b/crates/rspack_core/src/utils/mod.rs index 78e206fedd7..e153f3b2f5b 100644 --- a/crates/rspack_core/src/utils/mod.rs +++ b/crates/rspack_core/src/utils/mod.rs @@ -16,6 +16,7 @@ mod concatenated_module_visitor; mod concatenation_scope; mod extract_url_and_global; mod fast_actions; +mod file_counter; mod find_graph_roots; mod hash; mod identifier; @@ -35,6 +36,7 @@ pub use concatenation_scope::*; pub use self::comment::*; pub use self::extract_url_and_global::*; pub use self::fast_actions::*; +pub use self::file_counter::FileCounter; pub use self::find_graph_roots::*; pub use self::hash::*; pub use self::identifier::*; diff --git a/packages/rspack/src/Compilation.ts b/packages/rspack/src/Compilation.ts index c7aea54b0d3..b593ca522eb 100644 --- a/packages/rspack/src/Compilation.ts +++ b/packages/rspack/src/Compilation.ts @@ -991,22 +991,22 @@ BREAKING CHANGE: Asset processing hooks in Compilation has been merged into a si } fileDependencies = createFakeCompilationDependencies( - () => this.#inner.getFileDependencies(), + () => this.#inner.dependencies().fileDependencies, d => this.#inner.addFileDependencies(d) ); contextDependencies = createFakeCompilationDependencies( - () => this.#inner.getContextDependencies(), + () => this.#inner.dependencies().contextDependencies, d => this.#inner.addContextDependencies(d) ); missingDependencies = createFakeCompilationDependencies( - () => this.#inner.getMissingDependencies(), + () => this.#inner.dependencies().missingDependencies, d => this.#inner.addMissingDependencies(d) ); buildDependencies = createFakeCompilationDependencies( - () => this.#inner.getBuildDependencies(), + () => this.#inner.dependencies().buildDependencies, d => this.#inner.addBuildDependencies(d) );