perf: optimize execute module (#6415)

* feat: task_loop support listen task finish event

* perf: optimize execute module

* fix: clippy

* fix: clean event before run finish module task

* fix: self module FinishDeps bug

* fix: hmr panic

crates/rspack_binding_values/src/compilation.rs

@@ -463,13 +463,6 @@ impl JsCompilation {
     original_module_context: Option<String>,
     callback: JsFunction,
   ) -> Result<()> {
-    let options = self.0.options.clone();
-    let plugin_driver = self.0.plugin_driver.clone();
-    let resolver_factory = self.0.resolver_factory.clone();
-    let loader_resolver_factory = self.0.loader_resolver_factory.clone();
-    let cache = self.0.cache.clone();
-    let dependency_factories = self.0.dependency_factories.clone();
-
     callbackify(env, callback, async {
       let module_executor = self
         .0
@@ -478,12 +471,6 @@ impl JsCompilation {
         .expect("should have module executor");
       let result = module_executor
         .import_module(
-          options,
-          plugin_driver,
-          resolver_factory,
-          loader_resolver_factory,
-          cache,
-          dependency_factories,
           request,
           public_path,
           base_uri,

crates/rspack_core/src/compiler/compilation.rs

@@ -609,6 +609,13 @@ impl Compilation {
 
   #[instrument(name = "compilation:make", skip_all)]
   pub async fn make(&mut self, mut params: Vec<MakeParam>) -> Result<()> {
+    // run module_executor
+    if let Some(module_executor) = &mut self.module_executor {
+      let mut module_executor = std::mem::take(module_executor);
+      module_executor.hook_before_make(self, &params).await;
+      self.module_executor = Some(module_executor);
+    }
+
     let make_failed_module =
       MakeParam::ForceBuildModules(std::mem::take(&mut self.make_failed_module));
     let make_failed_dependencies =
@@ -1042,13 +1049,10 @@ impl Compilation {
     logger.time_end(start);
 
     // sync assets to compilation from module_executor
-    let assets = self
-      .module_executor
-      .as_mut()
-      .map(|module_executor| std::mem::take(&mut module_executor.assets))
-      .unwrap_or_default();
-    for (filename, asset) in assets {
-      self.emit_asset(filename, asset)
+    if let Some(module_executor) = &mut self.module_executor {
+      let mut module_executor = std::mem::take(module_executor);
+      module_executor.hook_before_process_assets(self).await;
+      self.module_executor = Some(module_executor);
     }
 
     let start = logger.time("process assets");

crates/rspack_core/src/compiler/hmr.rs

@@ -9,7 +9,9 @@ use rspack_sources::Source;
 use rustc_hash::FxHashSet as HashSet;
 
 use super::MakeParam;
-use crate::{fast_set, get_chunk_from_ukey, ChunkKind, Compilation, Compiler, RuntimeSpec};
+use crate::{
+  fast_set, get_chunk_from_ukey, ChunkKind, Compilation, Compiler, ModuleExecutor, RuntimeSpec,
+};
 
 impl<T> Compiler<T>
 where
@@ -73,8 +75,7 @@ where
         self.loader_resolver_factory.clone(),
         Some(records),
         self.cache.clone(),
-        // reuse module executor
-        std::mem::take(&mut self.compilation.module_executor),
+        Some(ModuleExecutor::default()),
       );
 
       if let Some(state) = self.options.get_incremental_rebuild_make_state() {
@@ -118,6 +119,9 @@ where
         new_compilation.code_splitting_cache =
           std::mem::take(&mut self.compilation.code_splitting_cache);
 
+        // reuse module executor
+        new_compilation.module_executor = std::mem::take(&mut self.compilation.module_executor);
+
         new_compilation.has_module_import_export_change = false;
       }
 

crates/rspack_core/src/compiler/make/mod.rs

@@ -1,5 +1,5 @@
 mod cutout;
-mod repair;
+pub mod repair;
 
 use std::{hash::BuildHasherDefault, path::PathBuf};
 
@@ -19,16 +19,16 @@ use crate::{
 #[derive(Debug, Default)]
 pub struct MakeArtifact {
   module_graph_partial: ModuleGraphPartial,
-  make_failed_dependencies: HashSet<BuildDependency>,
-  make_failed_module: HashSet<ModuleIdentifier>,
-  diagnostics: Vec<Diagnostic>,
+  pub make_failed_dependencies: HashSet<BuildDependency>,
+  pub make_failed_module: HashSet<ModuleIdentifier>,
+  pub diagnostics: Vec<Diagnostic>,
 
   entry_module_identifiers: IdentifierSet,
   optimize_analyze_result_map: IdentifierMap<OptimizeAnalyzeResult>,
-  file_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
-  context_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
-  missing_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
-  build_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
+  pub file_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
+  pub context_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
+  pub missing_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
+  pub build_dependencies: IndexSet<PathBuf, BuildHasherDefault<FxHasher>>,
 
   has_module_graph_change: bool,
 }
@@ -76,7 +76,7 @@ impl MakeArtifact {
   }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub enum MakeParam {
   ModifiedFiles(HashSet<PathBuf>),
   DeletedFiles(HashSet<PathBuf>),
@@ -99,10 +99,8 @@ pub async fn update_module_graph(
   let mut artifact = MakeArtifact::default();
   compilation.swap_make_artifact(&mut artifact);
   artifact.move_data_from_compilation(compilation);
-  let mut cutout = Cutout::default();
-  let build_dependencies = cutout.cutout_artifact(&mut artifact, params);
-  artifact = repair(compilation, artifact, build_dependencies)?;
-  cutout.fix_artifact(&mut artifact);
+
+  artifact = update_module_graph_with_artifact(compilation, artifact, params).await?;
 
   // Avoid to introduce too much overhead,
   // until we find a better way to align with webpack hmr behavior
@@ -149,3 +147,16 @@ pub async fn update_module_graph(
   compilation.swap_make_artifact(&mut artifact);
   Ok(())
 }
+
+pub async fn update_module_graph_with_artifact(
+  compilation: &Compilation,
+  mut artifact: MakeArtifact,
+  params: Vec<MakeParam>,
+) -> Result<MakeArtifact> {
+  let mut cutout = Cutout::default();
+  let build_dependencies = cutout.cutout_artifact(&mut artifact, params);
+  artifact = repair(compilation, artifact, build_dependencies)?;
+  cutout.fix_artifact(&mut artifact);
+
+  Ok(artifact)
+}

crates/rspack_core/src/compiler/make/repair/factorize.rs

@@ -147,7 +147,7 @@ pub struct ExportsInfoRelated {
 }
 
 #[derive(Debug)]
-struct FactorizeResultTask {
+pub struct FactorizeResultTask {
   //  pub dependency: DependencyId,
   pub original_module_identifier: Option<ModuleIdentifier>,
   /// Result will be available if [crate::ModuleFactory::create] returns `Ok`.

crates/rspack_core/src/compiler/make/repair/mod.rs

@@ -1,7 +1,7 @@
-mod add;
-mod build;
-mod factorize;
-mod process_dependencies;
+pub mod add;
+pub mod build;
+pub mod factorize;
+pub mod process_dependencies;
 
 use std::{hash::BuildHasherDefault, path::PathBuf, sync::Arc};
 
@@ -21,14 +21,14 @@ use crate::{
   NormalModuleSource, ResolverFactory, SharedPluginDriver,
 };
 
-struct MakeTaskContext {
+pub struct MakeTaskContext {
   // compilation info
-  plugin_driver: SharedPluginDriver,
-  compiler_options: Arc<CompilerOptions>,
-  resolver_factory: Arc<ResolverFactory>,
-  loader_resolver_factory: Arc<ResolverFactory>,
-  cache: Arc<Cache>,
-  dependency_factories: HashMap<DependencyType, Arc<dyn ModuleFactory>>,
+  pub plugin_driver: SharedPluginDriver,
+  pub compiler_options: Arc<CompilerOptions>,
+  pub resolver_factory: Arc<ResolverFactory>,
+  pub loader_resolver_factory: Arc<ResolverFactory>,
+  pub cache: Arc<Cache>,
+  pub dependency_factories: HashMap<DependencyType, Arc<dyn ModuleFactory>>,
 
   // TODO move outof context
   logger: CompilationLogger,
@@ -41,7 +41,7 @@ struct MakeTaskContext {
   /// Collecting all module that need to skip in tree-shaking ast modification phase
   //  bailout_module_identifiers: IdentifierMap<BailoutFlag>,
   // TODO change to artifact
-  module_graph_partial: ModuleGraphPartial,
+  pub module_graph_partial: ModuleGraphPartial,
   make_failed_dependencies: HashSet<BuildDependency>,
   make_failed_module: HashSet<ModuleIdentifier>,
 
@@ -56,7 +56,7 @@ struct MakeTaskContext {
 }
 
 impl MakeTaskContext {
-  fn new(compilation: &Compilation, artifact: MakeArtifact) -> Self {
+  pub fn new(compilation: &Compilation, artifact: MakeArtifact) -> Self {
     let logger = compilation.get_logger("rspack.Compilation");
     let mut build_cache_counter = None;
     let mut factorize_cache_counter = None;
@@ -97,7 +97,7 @@ impl MakeTaskContext {
     }
   }
 
-  fn transform_to_make_artifact(self) -> MakeArtifact {
+  pub fn transform_to_make_artifact(self) -> MakeArtifact {
     let Self {
       module_graph_partial,
       make_failed_dependencies,
@@ -137,9 +137,43 @@ impl MakeTaskContext {
   }
 
   // TODO use module graph with make artifact
-  fn get_module_graph_mut(partial: &mut ModuleGraphPartial) -> ModuleGraph {
+  pub fn get_module_graph_mut(partial: &mut ModuleGraphPartial) -> ModuleGraph {
     ModuleGraph::new(vec![], Some(partial))
   }
+
+  // TODO remove it after incremental rebuild cover all stage
+  pub fn transform_to_temp_compilation(&mut self) -> Compilation {
+    let mut compilation = Compilation::new(
+      self.compiler_options.clone(),
+      self.plugin_driver.clone(),
+      self.resolver_factory.clone(),
+      self.loader_resolver_factory.clone(),
+      None,
+      self.cache.clone(),
+      None,
+    );
+    compilation.dependency_factories = self.dependency_factories.clone();
+    let mut make_artifact = MakeArtifact {
+      module_graph_partial: std::mem::take(&mut self.module_graph_partial),
+      file_dependencies: std::mem::take(&mut self.file_dependencies),
+      context_dependencies: std::mem::take(&mut self.context_dependencies),
+      missing_dependencies: std::mem::take(&mut self.missing_dependencies),
+      build_dependencies: std::mem::take(&mut self.build_dependencies),
+      ..Default::default()
+    };
+    compilation.swap_make_artifact(&mut make_artifact);
+    compilation
+  }
+
+  pub fn recovery_from_temp_compilation(&mut self, mut compilation: Compilation) {
+    let mut make_artifact = Default::default();
+    compilation.swap_make_artifact(&mut make_artifact);
+    self.module_graph_partial = make_artifact.module_graph_partial;
+    self.file_dependencies = make_artifact.file_dependencies;
+    self.context_dependencies = make_artifact.context_dependencies;
+    self.missing_dependencies = make_artifact.missing_dependencies;
+    self.build_dependencies = make_artifact.build_dependencies;
+  }
 }
 
 pub fn repair(