Skip to content

Commit

Permalink
feat(incremental): named chunk ids (#8652)
Browse files Browse the repository at this point in the history
  • Loading branch information
ahabhgk authored Dec 11, 2024
1 parent f8c0056 commit 9b0a248
Show file tree
Hide file tree
Showing 65 changed files with 775 additions and 448 deletions.
15 changes: 10 additions & 5 deletions crates/rspack_binding_values/src/chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,17 +40,22 @@ impl JsChunk {

#[napi(getter)]
pub fn id(&self) -> napi::Result<Either<&str, ()>> {
let (_, chunk) = self.as_ref()?;
Ok(match chunk.id() {
Some(id) => Either::A(id),
let (compilation, chunk) = self.as_ref()?;
Ok(match chunk.id(&compilation.chunk_ids) {
Some(id) => Either::A(id.as_str()),
None => Either::B(()),
})
}

#[napi(getter)]
pub fn ids(&self) -> napi::Result<Vec<&str>> {
let (_, chunk) = self.as_ref()?;
Ok(chunk.id().map(|id| vec![id]).unwrap_or_default())
let (compilation, chunk) = self.as_ref()?;
Ok(
chunk
.id(&compilation.chunk_ids)
.map(|id| vec![id.as_str()])
.unwrap_or_default(),
)
}

#[napi(getter)]
Expand Down
57 changes: 34 additions & 23 deletions crates/rspack_core/src/chunk.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use rspack_error::Diagnostic;
use rspack_hash::{RspackHash, RspackHashDigest};
use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet, FxHasher};

use crate::chunk_graph_chunk::ChunkId;
use crate::{
compare_chunk_group, merge_runtime, sort_group_by_index, ChunkGraph, ChunkGroupOrderKey,
RenderManifestEntry,
Expand Down Expand Up @@ -58,7 +59,6 @@ pub struct Chunk {
// - The name of chunks create by dynamic import is `None` unless users use
// magic comment like `import(/* webpackChunkName: "someChunk" * / './someModule.js')` to specify it.
name: Option<String>,
id: Option<String>,
id_name_hints: HashSet<String>,
filename_template: Option<Filename>,
css_filename_template: Option<Filename>,
Expand Down Expand Up @@ -108,12 +108,23 @@ impl Chunk {
self.css_filename_template = filename_template;
}

pub fn id(&self) -> Option<&str> {
self.id.as_deref()
pub fn id<'a>(&self, chunk_ids: &'a UkeyMap<ChunkUkey, ChunkId>) -> Option<&'a ChunkId> {
ChunkGraph::get_chunk_id(chunk_ids, &self.ukey)
}

pub fn set_id(&mut self, id: Option<String>) {
self.id = id;
pub fn expect_id<'a>(&self, chunk_ids: &'a UkeyMap<ChunkUkey, ChunkId>) -> &'a ChunkId {
self
.id(chunk_ids)
.expect("Should set id before calling expect_id")
}

pub fn set_id(
&self,
chunk_ids: &mut UkeyMap<ChunkUkey, ChunkId>,
id: impl Into<ChunkId>,
) -> bool {
let id = id.into();
ChunkGraph::set_chunk_id(chunk_ids, self.ukey, id)
}

pub fn prevent_integration(&self) -> bool {
Expand Down Expand Up @@ -263,7 +274,6 @@ impl Chunk {
filename_template: None,
css_filename_template: None,
ukey: ChunkUkey::new(),
id: None,
id_name_hints: Default::default(),
prevent_integration: false,
files: Default::default(),
Expand Down Expand Up @@ -578,18 +588,14 @@ impl Chunk {
chunks
}

pub fn expect_id(&self) -> &str {
self
.id
.as_ref()
.expect("Should set id before calling expect_id")
}

pub fn name_for_filename_template(&self) -> Option<&str> {
pub fn name_for_filename_template<'a>(
&'a self,
chunk_ids: &'a UkeyMap<ChunkUkey, ChunkId>,
) -> Option<&'a str> {
if self.name.is_some() {
self.name.as_deref()
} else {
self.id.as_deref()
self.id(chunk_ids).map(|id| id.as_str())
}
}

Expand All @@ -602,7 +608,7 @@ impl Chunk {
}

pub fn update_hash(&self, hasher: &mut RspackHash, compilation: &Compilation) {
self.id.hash(hasher);
self.id(&compilation.chunk_ids).hash(hasher);
for module in compilation
.chunk_graph
.get_ordered_chunk_modules(&self.ukey, &compilation.get_module_graph())
Expand Down Expand Up @@ -716,16 +722,20 @@ impl Chunk {
&self,
order: &ChunkGroupOrderKey,
compilation: &Compilation,
) -> Option<Vec<String>> {
) -> Option<Vec<ChunkId>> {
self
.get_children_of_type_in_order(order, compilation, true)
.map(|order_children| {
order_children
.iter()
.flat_map(|(_, child_chunks)| {
child_chunks
.iter()
.filter_map(|chunk_ukey| compilation.chunk_by_ukey.expect_get(chunk_ukey).id.clone())
child_chunks.iter().filter_map(|chunk_ukey| {
compilation
.chunk_by_ukey
.expect_get(chunk_ukey)
.id(&compilation.chunk_ids)
.cloned()
})
})
.collect_vec()
})
Expand All @@ -735,21 +745,22 @@ impl Chunk {
&self,
include_direct_children: bool,
compilation: &Compilation,
) -> HashMap<ChunkGroupOrderKey, IndexMap<String, Vec<String>, BuildHasherDefault<FxHasher>>> {
) -> HashMap<ChunkGroupOrderKey, IndexMap<ChunkId, Vec<ChunkId>, BuildHasherDefault<FxHasher>>>
{
let mut result = HashMap::default();

fn add_child_ids_by_orders_to_map(
chunk_ukey: &ChunkUkey,
order: &ChunkGroupOrderKey,
result: &mut HashMap<
ChunkGroupOrderKey,
IndexMap<String, Vec<String>, BuildHasherDefault<FxHasher>>,
IndexMap<ChunkId, Vec<ChunkId>, BuildHasherDefault<FxHasher>>,
>,
compilation: &Compilation,
) {
let chunk = compilation.chunk_by_ukey.expect_get(chunk_ukey);
if let (Some(chunk_id), Some(child_chunk_ids)) = (
chunk.id.clone(),
chunk.id(&compilation.chunk_ids).cloned(),
chunk.get_child_ids_by_order(order, compilation),
) {
result
Expand Down
79 changes: 77 additions & 2 deletions crates/rspack_core/src/chunk_graph/chunk_graph_chunk.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
//! There are methods whose verb is `ChunkGraphChunk`
use std::borrow::Borrow;
use std::fmt;
use std::sync::Arc;

use hashlink::LinkedHashMap;
use indexmap::IndexSet;
use itertools::Itertools;
use rspack_collections::{DatabaseItem, IdentifierLinkedMap, IdentifierMap, IdentifierSet};
use rspack_cacheable::cacheable;
use rspack_collections::{
DatabaseItem, IdentifierLinkedMap, IdentifierMap, IdentifierSet, UkeyMap,
};
use rustc_hash::{FxHashMap as HashMap, FxHashSet};
use serde::{Serialize, Serializer};

use crate::{
find_graph_roots, merge_runtime, BoxModule, Chunk, ChunkByUkey, ChunkGraphModule,
Expand All @@ -21,6 +29,51 @@ pub struct ChunkSizeOptions {
pub entry_chunk_multiplicator: Option<f64>,
}

#[cacheable]
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ChunkId {
inner: Arc<str>,
}

impl From<String> for ChunkId {
fn from(s: String) -> Self {
Self { inner: s.into() }
}
}

impl From<&str> for ChunkId {
fn from(s: &str) -> Self {
Self { inner: s.into() }
}
}

impl fmt::Display for ChunkId {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.as_str())
}
}

impl Serialize for ChunkId {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
serializer.serialize_str(self.as_str())
}
}

impl Borrow<str> for ChunkId {
fn borrow(&self) -> &str {
self.as_str()
}
}

impl ChunkId {
pub fn as_str(&self) -> &str {
&self.inner
}
}

#[derive(Debug, Clone, Default)]
pub struct ChunkGraphChunk {
/// URI of modules => ChunkGroupUkey
Expand Down Expand Up @@ -550,7 +603,10 @@ impl ChunkGraph {
.iter()
{
let chunk = compilation.chunk_by_ukey.expect_get(c);
map.insert(chunk.expect_id().to_string(), filter(c, compilation));
map.insert(
chunk.expect_id(&compilation.chunk_ids).to_string(),
filter(c, compilation),
);
}

map
Expand Down Expand Up @@ -912,4 +968,23 @@ impl ChunkGraph {
})
.unwrap_or(module.source_types().iter().copied().collect())
}

pub fn get_chunk_id<'a>(
chunk_ids: &'a UkeyMap<ChunkUkey, ChunkId>,
chunk_ukey: &ChunkUkey,
) -> Option<&'a ChunkId> {
chunk_ids.get(chunk_ukey)
}

pub fn set_chunk_id(
chunk_ids: &mut UkeyMap<ChunkUkey, ChunkId>,
chunk_ukey: ChunkUkey,
id: ChunkId,
) -> bool {
if let Some(old_id) = chunk_ids.insert(chunk_ukey, id.clone()) {
old_id != id
} else {
true
}
}
}
2 changes: 1 addition & 1 deletion crates/rspack_core/src/chunk_group.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ impl ChunkGroup {
compilation
.chunk_by_ukey
.get(chunk)
.and_then(|item| item.id())
.and_then(|item| item.id(&compilation.chunk_ids))
})
.join("+")
}
Expand Down
21 changes: 16 additions & 5 deletions crates/rspack_core/src/compiler/compilation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ use crate::{
cache::Cache,
cgm_hash_results::CgmHashResults,
cgm_runtime_requirement_results::CgmRuntimeRequirementsResults,
chunk_graph_chunk::ChunkId,
chunk_graph_module::ModuleId,
get_runtime_key,
incremental::{Incremental, IncrementalPasses, Mutation},
Expand Down Expand Up @@ -181,6 +182,8 @@ pub struct Compilation {
pub dependencies_diagnostics: IdentifierMap<Vec<Diagnostic>>,
// artifact for module_ids
pub module_ids: IdentifierMap<ModuleId>,
// artifact for chunk_ids
pub chunk_ids: UkeyMap<ChunkUkey, ChunkId>,
// artifact for code_generation
pub code_generation_results: CodeGenerationResults,
// artifact for create_module_hashes
Expand All @@ -201,7 +204,6 @@ pub struct Compilation {
pub incremental: Incremental,

pub hash: Option<RspackHashDigest>,
pub used_chunk_ids: HashSet<String>,

pub file_dependencies: IndexSet<ArcPath, BuildHasherDefault<FxHasher>>,
pub context_dependencies: IndexSet<ArcPath, BuildHasherDefault<FxHasher>>,
Expand Down Expand Up @@ -289,6 +291,7 @@ impl Compilation {
async_modules: Default::default(),
dependencies_diagnostics: Default::default(),
module_ids: Default::default(),
chunk_ids: Default::default(),
code_generation_results: Default::default(),
cgm_hash_results: Default::default(),
cgm_runtime_requirements_results: Default::default(),
Expand All @@ -301,8 +304,8 @@ impl Compilation {
old_cache,
incremental,
code_splitting_cache: Default::default(),

hash: None,
used_chunk_ids: Default::default(),

file_dependencies: Default::default(),
context_dependencies: Default::default(),
Expand Down Expand Up @@ -1502,6 +1505,7 @@ impl Compilation {
entrypoint_ukey: &ChunkGroupUkey,
chunk_group_by_ukey: &ChunkGroupByUkey,
chunk_by_ukey: &ChunkByUkey,
chunk_ids: &UkeyMap<ChunkUkey, ChunkId>,
chunk_graph: &mut ChunkGraph,
) {
let entrypoint = chunk_group_by_ukey.expect_get(entrypoint_ukey);
Expand All @@ -1517,14 +1521,15 @@ impl Compilation {
runtime,
chunk_by_ukey.get(&entrypoint.get_runtime_chunk(chunk_group_by_ukey)),
) {
chunk_graph.set_runtime_id(runtime, chunk.id().map(ToOwned::to_owned));
chunk_graph.set_runtime_id(runtime, chunk.id(chunk_ids).map(|id| id.to_string()));
}
}
for i in self.entrypoints.iter() {
process_entrypoint(
i.1,
&self.chunk_group_by_ukey,
&self.chunk_by_ukey,
&self.chunk_ids,
&mut self.chunk_graph,
)
}
Expand All @@ -1533,6 +1538,7 @@ impl Compilation {
i,
&self.chunk_group_by_ukey,
&self.chunk_by_ukey,
&self.chunk_ids,
&mut self.chunk_graph,
)
}
Expand Down Expand Up @@ -1862,11 +1868,16 @@ impl Compilation {
.filter(|(_, (_, remaining))| *remaining != 0)
.map(|(chunk_ukey, _)| self.chunk_by_ukey.expect_get(chunk_ukey))
.collect();
circular.sort_unstable_by(|a, b| a.id().cmp(&b.id()));
circular.sort_unstable_by(|a, b| a.id(&self.chunk_ids).cmp(&b.id(&self.chunk_ids)));
runtime_chunks.extend(circular.iter().map(|chunk| chunk.ukey()));
let circular_names = circular
.iter()
.map(|chunk| chunk.name().or(chunk.id()).unwrap_or("no id chunk"))
.map(|chunk| {
chunk
.name()
.or(chunk.id(&self.chunk_ids).map(|id| id.as_str()))
.unwrap_or("no id chunk")
})
.join(", ");
self.push_diagnostic(diagnostic!(severity = Severity::Warn, "Circular dependency between chunks with runtime ({})\nThis prevents using hashes of each other and should be avoided.", circular_names).boxed().into());
}
Expand Down
Loading

2 comments on commit 9b0a248

@rspack-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Benchmark detail: Open

Name Base (2024-12-11 6a5b766) Current Change
10000_big_production-mode_disable-minimize + exec 37.6 s ± 447 ms 37.9 s ± 557 ms +0.85 %
10000_development-mode + exec 1.79 s ± 22 ms 1.77 s ± 23 ms -1.07 %
10000_development-mode_hmr + exec 659 ms ± 21 ms 641 ms ± 23 ms -2.67 %
10000_production-mode + exec 2.33 s ± 29 ms 2.34 s ± 42 ms +0.38 %
arco-pro_development-mode + exec 1.76 s ± 82 ms 1.74 s ± 72 ms -1.00 %
arco-pro_development-mode_hmr + exec 424 ms ± 1.1 ms 424 ms ± 1 ms +0.11 %
arco-pro_production-mode + exec 3.13 s ± 79 ms 3.12 s ± 69 ms -0.31 %
arco-pro_production-mode_generate-package-json-webpack-plugin + exec 3.15 s ± 56 ms 3.18 s ± 103 ms +0.95 %
arco-pro_production-mode_traverse-chunk-modules + exec 3.11 s ± 78 ms 3.15 s ± 82 ms +1.40 %
threejs_development-mode_10x + exec 1.61 s ± 18 ms 1.62 s ± 25 ms +0.61 %
threejs_development-mode_10x_hmr + exec 785 ms ± 21 ms 782 ms ± 7.8 ms -0.34 %
threejs_production-mode_10x + exec 4.87 s ± 23 ms 4.88 s ± 18 ms +0.07 %
10000_big_production-mode_disable-minimize + rss memory 9699 MiB ± 36.6 MiB 9791 MiB ± 189 MiB +0.95 %
10000_development-mode + rss memory 836 MiB ± 66.1 MiB 811 MiB ± 37.6 MiB -2.98 %
10000_development-mode_hmr + rss memory 1925 MiB ± 184 MiB 1895 MiB ± 409 MiB -1.57 %
10000_production-mode + rss memory 712 MiB ± 54.4 MiB 689 MiB ± 41.4 MiB -3.18 %
arco-pro_development-mode + rss memory 721 MiB ± 27.3 MiB 702 MiB ± 20 MiB -2.64 %
arco-pro_development-mode_hmr + rss memory 927 MiB ± 73 MiB 935 MiB ± 175 MiB +0.84 %
arco-pro_production-mode + rss memory 829 MiB ± 65.1 MiB 810 MiB ± 35.3 MiB -2.28 %
arco-pro_production-mode_generate-package-json-webpack-plugin + rss memory 801 MiB ± 26.3 MiB 807 MiB ± 52.8 MiB +0.70 %
arco-pro_production-mode_traverse-chunk-modules + rss memory 832 MiB ± 51.2 MiB 807 MiB ± 40.5 MiB -2.99 %
threejs_development-mode_10x + rss memory 768 MiB ± 34.8 MiB 769 MiB ± 39.3 MiB +0.16 %
threejs_development-mode_10x_hmr + rss memory 1585 MiB ± 286 MiB 1758 MiB ± 162 MiB +10.87 %
threejs_production-mode_10x + rss memory 1104 MiB ± 60.4 MiB 1082 MiB ± 76.3 MiB -2.03 %

@rspack-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Ran ecosystem CI: Open

suite result
modernjs ✅ success
_selftest ✅ success
rsdoctor ✅ success
rspress ✅ success
rslib ✅ success
rsbuild ✅ success
examples ✅ success
devserver ✅ success
nuxt ✅ success

Please sign in to comment.