diff --git a/crates/atlaspack/src/atlaspack.rs b/crates/atlaspack/src/atlaspack.rs index 50d142136..3501b8de5 100644 --- a/crates/atlaspack/src/atlaspack.rs +++ b/crates/atlaspack/src/atlaspack.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use std::sync::Arc; use atlaspack_config::atlaspack_rc_config_loader::{AtlaspackRcConfigLoader, LoadConfigOptions}; -use atlaspack_core::asset_graph::{AssetGraph, AssetNode}; +use atlaspack_core::asset_graph::{AssetGraph, AssetGraphNode, AssetNode}; use atlaspack_core::config_loader::ConfigLoader; use atlaspack_core::plugin::{PluginContext, PluginLogger, PluginOptions}; use atlaspack_core::types::AtlaspackOptions; @@ -128,7 +128,7 @@ impl Atlaspack { let asset_graph = match request_result { RequestResult::AssetGraph(result) => { - self.commit_assets(result.graph.assets.as_slice())?; + self.commit_assets(result.graph.nodes().collect())?; result.graph } @@ -139,10 +139,14 @@ impl Atlaspack { }) } - fn commit_assets(&self, assets: &[AssetNode]) -> anyhow::Result<()> { + fn commit_assets(&self, assets: Vec<&AssetGraphNode>) -> anyhow::Result<()> { let mut txn = self.db.environment().write_txn()?; - for AssetNode { asset, .. } in assets.iter() { + for asset_node in assets { + let AssetGraphNode::Asset(AssetNode { asset, .. }) = asset_node else { + continue; + }; + self.db.put(&mut txn, &asset.id, asset.code.bytes())?; if let Some(map) = &asset.map { // TODO: For some reason to_buffer strips data when rkyv was upgraded, so now we use json @@ -185,13 +189,12 @@ mod tests { rpc(), )?; - let assets = ["foo", "bar", "baz"]; - - atlaspack.commit_assets( - &assets - .iter() - .enumerate() - .map(|(idx, asset)| AssetNode { + let assets_names = ["foo", "bar", "baz"]; + let assets = assets_names + .iter() + .enumerate() + .map(|(idx, asset)| { + AssetGraphNode::Asset(AssetNode { asset: Asset { id: idx.to_string(), code: Code::from(asset.to_string()), @@ -199,11 +202,13 @@ mod tests { }, requested_symbols: HashSet::new(), }) - .collect::>(), - )?; + }) + .collect::>(); + + atlaspack.commit_assets(assets.iter().collect())?; let txn = db.environment().read_txn()?; - for (idx, asset) in assets.iter().enumerate() { + for (idx, asset) in assets_names.iter().enumerate() { let entry = db.get(&txn, &idx.to_string())?; assert_eq!(entry, Some(asset.to_string().into())); } diff --git a/crates/atlaspack/src/requests/asset_graph_request.rs b/crates/atlaspack/src/requests/asset_graph_request.rs index 93cb87b07..cf0e1dfe3 100644 --- a/crates/atlaspack/src/requests/asset_graph_request.rs +++ b/crates/atlaspack/src/requests/asset_graph_request.rs @@ -8,10 +8,11 @@ use indexmap::IndexMap; use pathdiff::diff_paths; use petgraph::graph::NodeIndex; -use atlaspack_core::asset_graph::{AssetGraph, DependencyNode, DependencyState}; -use atlaspack_core::types::{Asset, AssetWithDependencies, Dependency}; - use crate::request_tracker::{Request, ResultAndInvalidations, RunRequestContext, RunRequestError}; +use atlaspack_core::asset_graph::{ + propagate_requested_symbols, AssetGraph, DependencyNode, DependencyState, +}; +use atlaspack_core::types::{Asset, AssetWithDependencies, Dependency}; use super::asset_request::{AssetRequest, AssetRequestOutput}; use super::entry_request::{EntryRequest, EntryRequestOutput}; @@ -45,14 +46,14 @@ impl Request for AssetGraphRequest { } struct AssetGraphBuilder { - request_id_to_dep_node_index: HashMap, + request_id_to_dependency_idx: HashMap, graph: AssetGraph, visited: HashSet, work_count: u32, request_context: RunRequestContext, sender: ResultSender, receiver: ResultReceiver, - asset_request_to_asset: HashMap, + asset_request_to_asset_idx: HashMap, waiting_asset_requests: HashMap>, } @@ -61,14 +62,14 @@ impl AssetGraphBuilder { let (sender, receiver) = channel(); AssetGraphBuilder { - request_id_to_dep_node_index: HashMap::new(), + request_id_to_dependency_idx: HashMap::new(), graph: AssetGraph::new(), visited: HashSet::new(), work_count: 0, request_context, sender, receiver, - asset_request_to_asset: HashMap::new(), + asset_request_to_asset_idx: HashMap::new(), waiting_asset_requests: HashMap::new(), } } @@ -137,16 +138,16 @@ impl AssetGraphBuilder { } fn handle_path_result(&mut self, result: PathRequestOutput, request_id: u64) { - let node = *self - .request_id_to_dep_node_index + let dependency_idx = *self + .request_id_to_dependency_idx .get(&request_id) .expect("Missing node index for request id {request_id}"); - let dep_index = self.graph.dependency_index(node).unwrap(); + let DependencyNode { dependency, requested_symbols, state, - } = &mut self.graph.dependencies[dep_index]; + } = self.graph.get_dependency_node_mut(&dependency_idx).unwrap(); let asset_request = match result { PathRequestOutput::Resolved { @@ -185,16 +186,16 @@ impl AssetGraphBuilder { let id = asset_request.id(); if self.visited.insert(id) { - self.request_id_to_dep_node_index.insert(id, node); + self.request_id_to_dependency_idx.insert(id, dependency_idx); self.work_count += 1; let _ = self .request_context .queue_request(asset_request, self.sender.clone()); - } else if let Some(asset_node_index) = self.asset_request_to_asset.get(&id) { + } else if let Some(asset_node_index) = self.asset_request_to_asset_idx.get(&id) { // We have already completed this AssetRequest so we can connect the // Dependency to the Asset immediately - self.graph.add_edge(&node, asset_node_index); - self.propagate_requested_symbols(*asset_node_index, node); + self.graph.add_edge(&dependency_idx, asset_node_index); + self.propagate_requested_symbols(*asset_node_index, dependency_idx); } else { // The AssetRequest has already been kicked off but is yet to // complete. Register this Dependency to be connected once it @@ -203,9 +204,9 @@ impl AssetGraphBuilder { .waiting_asset_requests .entry(id) .and_modify(|nodes| { - nodes.insert(node); + nodes.insert(dependency_idx); }) - .or_insert_with(|| HashSet::from([node])); + .or_insert_with(|| HashSet::from([dependency_idx])); } } @@ -232,54 +233,57 @@ impl AssetGraphBuilder { discovered_assets, dependencies, } = result; - let incoming_dep_node_index = *self - .request_id_to_dep_node_index + + let incoming_dependency_idx = *self + .request_id_to_dependency_idx .get(&request_id) .expect("Missing node index for request id {request_id}"); // Connect the incoming DependencyNode to the new AssetNode - let asset_node_index = self.graph.add_asset(incoming_dep_node_index, asset.clone()); + let asset_idx = self.graph.add_asset(asset.clone()); + + self.graph.add_edge(&incoming_dependency_idx, &asset_idx); self - .asset_request_to_asset - .insert(request_id, asset_node_index); + .asset_request_to_asset_idx + .insert(request_id, asset_idx); - let root_asset = (&asset, asset_node_index); + let root_asset = (&asset, asset_idx); let mut added_discovered_assets: HashMap = HashMap::new(); // Attach the "direct" discovered assets to the graph let direct_discovered_assets = get_direct_discovered_assets(&discovered_assets, &dependencies); for discovered_asset in direct_discovered_assets { - let asset_node_index = self - .graph - .add_asset(incoming_dep_node_index, discovered_asset.asset.clone()); + let asset_idx = self.graph.add_asset(discovered_asset.asset.clone()); + + self.graph.add_edge(&incoming_dependency_idx, &asset_idx); self.add_asset_dependencies( &discovered_asset.dependencies, &discovered_assets, - asset_node_index, + asset_idx, &mut added_discovered_assets, root_asset, ); - self.propagate_requested_symbols(asset_node_index, incoming_dep_node_index); + self.propagate_requested_symbols(asset_idx, incoming_dependency_idx); } self.add_asset_dependencies( &dependencies, &discovered_assets, - asset_node_index, + asset_idx, &mut added_discovered_assets, root_asset, ); - self.propagate_requested_symbols(asset_node_index, incoming_dep_node_index); + self.propagate_requested_symbols(asset_idx, incoming_dependency_idx); // Connect any previously discovered Dependencies that were waiting // for this AssetNode to be created if let Some(waiting) = self.waiting_asset_requests.remove(&request_id) { for dep in waiting { - self.graph.add_edge(&dep, &asset_node_index); - self.propagate_requested_symbols(asset_node_index, dep); + self.graph.add_edge(&dep, &asset_idx); + self.propagate_requested_symbols(asset_idx, dep); } } } @@ -288,7 +292,7 @@ impl AssetGraphBuilder { &mut self, dependencies: &Vec, discovered_assets: &Vec, - asset_node_index: NodeIndex, + asset_idx: NodeIndex, added_discovered_assets: &mut HashMap, root_asset: (&Asset, NodeIndex), ) { @@ -333,10 +337,11 @@ impl AssetGraphBuilder { .as_ref() .is_some_and(|key| key == &dependency.specifier); - let dep_node = self.graph.add_dependency(asset_node_index, dependency); + let dependency_idx = self.graph.add_dependency(dependency); + self.graph.add_edge(&asset_idx, &dependency_idx); if dep_to_root_asset { - self.graph.add_edge(&dep_node, &root_asset.1); + self.graph.add_edge(&dependency_idx, &root_asset.1); } // If the dependency points to a dicovered asset then add the asset using the new @@ -351,22 +356,23 @@ impl AssetGraphBuilder { if let Some(asset_node_index) = existing_discovered_asset { // This discovered_asset has already been added to the graph so we // just need to connect the dependency node to the asset node - self.graph.add_edge(&dep_node, asset_node_index); + self.graph.add_edge(&dependency_idx, asset_node_index); } else { // This discovered_asset isn't yet in the graph so we'll need to add // it and assign it's dependencies by calling added_discovered_assets // recursively. - let asset_node_index = self.graph.add_asset(dep_node, asset.clone()); - added_discovered_assets.insert(asset.id.clone(), asset_node_index); + let asset_idx = self.graph.add_asset(asset.clone()); + self.graph.add_edge(&dependency_idx, &asset_idx); + added_discovered_assets.insert(asset.id.clone(), asset_idx); self.add_asset_dependencies( dependencies, discovered_assets, - asset_node_index, + asset_idx, added_discovered_assets, root_asset, ); - self.propagate_requested_symbols(asset_node_index, dep_node); + self.propagate_requested_symbols(asset_idx, dependency_idx); } } } @@ -374,19 +380,20 @@ impl AssetGraphBuilder { fn propagate_requested_symbols( &mut self, - asset_node_index: NodeIndex, - incoming_dep_node_index: NodeIndex, + asset_idx: NodeIndex, + incoming_dependency_idx: NodeIndex, ) { - self.graph.propagate_requested_symbols( - asset_node_index, - incoming_dep_node_index, - &mut |dependency_node_index: NodeIndex, dependency: Arc| { + propagate_requested_symbols( + &mut self.graph, + asset_idx, + incoming_dependency_idx, + &mut |dependency_idx: NodeIndex, dependency: Arc| { Self::on_undeferred( - &mut self.request_id_to_dep_node_index, + &mut self.request_id_to_dependency_idx, &mut self.work_count, &mut self.request_context, &self.sender, - dependency_node_index, + dependency_idx, dependency, ); }, @@ -407,7 +414,7 @@ impl AssetGraphBuilder { dependency: Arc::new(dependency), }; self - .request_id_to_dep_node_index + .request_id_to_dependency_idx .insert(request.id(), dep_node); self.work_count += 1; let _ = self @@ -491,9 +498,11 @@ mod tests { use std::path::{Path, PathBuf}; use std::sync::Arc; + use atlaspack_core::asset_graph::{AssetGraph, AssetGraphNode, AssetNode}; use atlaspack_core::types::{AtlaspackOptions, Code}; use atlaspack_filesystem::in_memory_file_system::InMemoryFileSystem; use atlaspack_filesystem::FileSystem; + use petgraph::visit::Bfs; use crate::requests::{AssetGraphRequest, RequestResult}; use crate::test_utils::{request_tracker, RequestTrackerTestOptions}; @@ -513,8 +522,14 @@ mod tests { return; }; - assert_eq!(asset_graph_request_result.graph.assets.len(), 0); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 0); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 0); + assert_eq!( + asset_graph_request_result + .graph + .get_dependency_nodes() + .len(), + 0 + ); } #[tokio::test(flavor = "multi_thread")] @@ -562,26 +577,22 @@ mod tests { return; }; - assert_eq!(asset_graph_request_result.graph.assets.len(), 1); - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 1); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 1); assert_eq!( asset_graph_request_result .graph - .assets - .first() - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") + .get_dependency_nodes() + .len(), + 1 ); + + let AssetNode { + asset: first_asset, .. + } = get_first_asset(&asset_graph_request_result.graph).expect("No assets in graph"); + + assert_eq!(first_asset.file_path, temporary_dir.join("entry.js")); assert_eq!( - asset_graph_request_result - .graph - .assets - .first() - .unwrap() - .asset - .code, + first_asset.code, (Code::from( String::from( r#" @@ -664,20 +675,21 @@ mod tests { }; // Entry, 2 assets + helpers file - assert_eq!(asset_graph_request_result.graph.assets.len(), 4); + assert_eq!(asset_graph_request_result.graph.get_asset_nodes().len(), 4); // Entry, entry to assets (2), assets to helpers (2) - assert_eq!(asset_graph_request_result.graph.dependencies.len(), 5); - assert_eq!( asset_graph_request_result .graph - .assets - .first() - .unwrap() - .asset - .file_path, - temporary_dir.join("entry.js") + .get_dependency_nodes() + .len(), + 5 ); + + let AssetNode { + asset: first_asset, .. + } = get_first_asset(&asset_graph_request_result.graph).expect("No assets in graph"); + + assert_eq!(first_asset.file_path, temporary_dir.join("entry.js")); } fn setup_core_modules(fs: &InMemoryFileSystem, core_path: &Path) { @@ -691,4 +703,24 @@ mod tests { String::from("/* helpers */"), ); } + + /// Do a BFS traversal of the the graph until the first AssetNode + /// is discovered. This should be the entry Asset. + fn get_first_asset(asset_graph: &AssetGraph) -> Option<&AssetNode> { + let mut first_asset = None::<&AssetNode>; + + let mut bfs = Bfs::new(&asset_graph.graph, asset_graph.root_node()); + + while let Some(idx) = bfs.next(&asset_graph.graph) { + match asset_graph.get_node(&idx) { + Some(AssetGraphNode::Asset(asset_node)) => { + first_asset.replace(asset_node); + break; + } + _ => continue, + } + } + + first_asset + } } diff --git a/crates/atlaspack_core/src/asset_graph.rs b/crates/atlaspack_core/src/asset_graph.rs deleted file mode 100644 index c59d16f55..000000000 --- a/crates/atlaspack_core/src/asset_graph.rs +++ /dev/null @@ -1,730 +0,0 @@ -use std::{collections::HashSet, sync::Arc}; - -use petgraph::{ - graph::{DiGraph, NodeIndex}, - visit::EdgeRef, - Direction, -}; -use serde::Serialize; - -use crate::types::{Asset, Dependency}; - -#[derive(Clone, Debug)] -pub struct AssetGraph { - graph: DiGraph, - pub assets: Vec, - pub dependencies: Vec, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetNode { - pub asset: Asset, - pub requested_symbols: HashSet, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct DependencyNode { - pub dependency: Arc, - pub requested_symbols: HashSet, - pub state: DependencyState, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum AssetGraphNode { - Root, - Entry, - Asset(usize), - Dependency(usize), -} - -#[derive(Clone, Debug, PartialEq)] -pub struct AssetGraphEdge {} - -#[derive(Clone, Debug, PartialEq)] -pub enum DependencyState { - New, - Deferred, - Excluded, - Resolved, -} - -impl PartialEq for AssetGraph { - fn eq(&self, other: &Self) -> bool { - let nodes = self.graph.raw_nodes().iter().map(|n| &n.weight); - let other_nodes = other.graph.raw_nodes().iter().map(|n| &n.weight); - - let edges = self - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - let other_edges = other - .graph - .raw_edges() - .iter() - .map(|e| (e.source(), e.target(), &e.weight)); - - nodes.eq(other_nodes) - && edges.eq(other_edges) - && self.assets == other.assets - && self.dependencies == other.dependencies - } -} - -impl Default for AssetGraph { - fn default() -> Self { - Self::new() - } -} - -impl AssetGraph { - pub fn new() -> Self { - let mut graph = DiGraph::new(); - - graph.add_node(AssetGraphNode::Root); - - AssetGraph { - graph, - assets: Vec::new(), - dependencies: Vec::new(), - } - } - - pub fn edges(&self) -> Vec { - let raw_edges = self.graph.raw_edges(); - let mut edges = Vec::with_capacity(raw_edges.len() * 2); - - for edge in raw_edges { - edges.push(edge.source().index() as u32); - edges.push(edge.target().index() as u32); - } - - edges - } - - pub fn nodes(&self) -> impl Iterator { - let nodes = self.graph.node_weights(); - - nodes - } - - pub fn add_asset(&mut self, parent_idx: NodeIndex, asset: Asset) -> NodeIndex { - let idx = self.assets.len(); - - self.assets.push(AssetNode { - asset, - requested_symbols: HashSet::default(), - }); - - let asset_idx = self.graph.add_node(AssetGraphNode::Asset(idx)); - - self - .graph - .add_edge(parent_idx, asset_idx, AssetGraphEdge {}); - - asset_idx - } - - pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { - // The root node index will always be 0 - let root_node_index = NodeIndex::new(0); - - let is_library = dependency.env.is_library; - let node_index = self.add_dependency(root_node_index, dependency); - - if is_library { - if let Some(dependency_index) = &self.dependency_index(node_index) { - self.dependencies[*dependency_index] - .requested_symbols - .insert("*".into()); - } - } - - node_index - } - - pub fn add_dependency(&mut self, parent_idx: NodeIndex, dependency: Dependency) -> NodeIndex { - let idx = self.dependencies.len(); - - self.dependencies.push(DependencyNode { - dependency: Arc::new(dependency), - requested_symbols: HashSet::default(), - state: DependencyState::New, - }); - - let dependency_idx = self.graph.add_node(AssetGraphNode::Dependency(idx)); - - self - .graph - .add_edge(parent_idx, dependency_idx, AssetGraphEdge {}); - - dependency_idx - } - - pub fn add_edge(&mut self, parent_idx: &NodeIndex, child_idx: &NodeIndex) { - self - .graph - .add_edge(*parent_idx, *child_idx, AssetGraphEdge {}); - } - - pub fn dependency_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Dependency(idx) => Some(*idx), - _ => None, - } - } - - pub fn asset_index(&self, node_index: NodeIndex) -> Option { - match self.graph.node_weight(node_index).unwrap() { - AssetGraphNode::Asset(idx) => Some(*idx), - _ => None, - } - } - - /// Propagates the requested symbols from an incoming dependency to an asset, - /// and forwards those symbols to re-exported dependencies if needed. - /// This may result in assets becoming un-deferred and transformed if they - /// now have requested symbols. - pub fn propagate_requested_symbols)>( - &mut self, - asset_node: NodeIndex, - incoming_dep_node: NodeIndex, - on_undeferred: &mut F, - ) { - let DependencyNode { - requested_symbols, .. - } = &self.dependencies[self.dependency_index(incoming_dep_node).unwrap()]; - - let asset_index = self.asset_index(asset_node).unwrap(); - let AssetNode { - asset, - requested_symbols: asset_requested_symbols, - } = &mut self.assets[asset_index]; - - let mut re_exports = HashSet::::default(); - let mut wildcards = HashSet::::default(); - let star = String::from("*"); - - if requested_symbols.contains(&star) { - // If the requested symbols includes the "*" namespace, we need to include all of the asset's - // exported symbols. - if let Some(symbols) = &asset.symbols { - for sym in symbols { - if asset_requested_symbols.insert(sym.exported.clone()) && sym.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(sym.local.clone()); - } - } - } - - // Propagate to all export * wildcard dependencies. - wildcards.insert(star); - } else { - // Otherwise, add each of the requested symbols to the asset. - for sym in requested_symbols.iter() { - if asset_requested_symbols.insert(sym.clone()) { - if let Some(asset_symbol) = asset - .symbols - .as_ref() - .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) - { - if asset_symbol.is_weak { - // Propagate re-exported symbol to dependency. - re_exports.insert(asset_symbol.local.clone()); - } - } else { - // If symbol wasn't found in the asset or a named re-export. - // This means the symbol is in one of the export * wildcards, but we don't know - // which one yet, so we propagate it to _all_ wildcard dependencies. - wildcards.insert(sym.clone()); - } - } - } - } - - let deps: Vec<_> = self - .graph - .neighbors_directed(asset_node, Direction::Outgoing) - .collect(); - for dep_node in deps { - let dep_index = self.dependency_index(dep_node).unwrap(); - let DependencyNode { - dependency, - requested_symbols, - state, - } = &mut self.dependencies[dep_index]; - - let mut updated = false; - if let Some(symbols) = &dependency.symbols { - for sym in symbols { - if sym.is_weak { - // This is a re-export. If it is a wildcard, add all unmatched symbols - // to this dependency, otherwise attempt to match a named re-export. - if sym.local == "*" { - for wildcard in &wildcards { - if requested_symbols.insert(wildcard.clone()) { - updated = true; - } - } - } else if re_exports.contains(&sym.local) - && requested_symbols.insert(sym.exported.clone()) - { - updated = true; - } - } else if requested_symbols.insert(sym.exported.clone()) { - // This is a normal import. Add the requested symbol. - updated = true; - } - } - } - - // If the dependency was updated, propagate to the target asset if there is one, - // or un-defer this dependency so we transform the requested asset. - // We must always resolve new dependencies to determine whether they have side effects. - if updated || *state == DependencyState::New { - if let Some(resolved) = self - .graph - .edges_directed(dep_node, Direction::Outgoing) - .next() - { - // Avoid infintite loops for self references - if resolved.target() != asset_node { - self.propagate_requested_symbols(resolved.target(), dep_node, on_undeferred); - } - } else { - on_undeferred(dep_node, Arc::clone(dependency)); - } - } - } - } - - pub fn serialize_nodes(&self, max_str_len: usize) -> serde_json::Result> { - let mut nodes: Vec = Vec::new(); - let mut curr_node = String::default(); - - for node in self.nodes() { - let serialized_node = match node { - AssetGraphNode::Root => SerializedAssetGraphNode::Root, - AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, - AssetGraphNode::Asset(idx) => { - let asset = self.assets[*idx].asset.clone(); - - SerializedAssetGraphNode::Asset { value: asset } - } - AssetGraphNode::Dependency(idx) => { - let dependency = self.dependencies[*idx].dependency.clone(); - SerializedAssetGraphNode::Dependency { - value: SerializedDependency { - id: dependency.id(), - dependency: dependency.as_ref().clone(), - }, - has_deferred: self.dependencies[*idx].state == DependencyState::Deferred, - } - } - }; - - let str = serde_json::to_string(&serialized_node)?; - if curr_node.len() + str.len() < (max_str_len - 3) { - if !curr_node.is_empty() { - curr_node.push(','); - } - curr_node.push_str(&str); - } else { - // Add the existing node now as it has reached the max JavaScript string size - nodes.push(format!("[{curr_node}]")); - curr_node = str; - } - } - - // Add the current node if it did not overflow in size - if curr_node.len() < (max_str_len - 3) { - nodes.push(format!("[{curr_node}]")); - } - - Ok(nodes) - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SerializedDependency { - id: String, - dependency: Dependency, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { - Root, - Entry, - Asset { - value: Asset, - }, - Dependency { - value: SerializedDependency, - has_deferred: bool, - }, -} - -impl std::hash::Hash for AssetGraph { - fn hash(&self, state: &mut H) { - for node in self.graph.node_weights() { - std::mem::discriminant(node).hash(state); - match node { - AssetGraphNode::Asset(idx) => self.assets[*idx].asset.id.hash(state), - AssetGraphNode::Dependency(idx) => self.dependencies[*idx].dependency.id().hash(state), - _ => {} - } - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use serde_json::{json, Value}; - - use crate::types::{Symbol, Target}; - - use super::*; - - type TestSymbol<'a> = (&'a str, &'a str, bool); - fn symbol(test_symbol: &TestSymbol) -> Symbol { - let (local, exported, is_weak) = test_symbol; - Symbol { - local: String::from(*local), - exported: String::from(*exported), - is_weak: is_weak.to_owned(), - ..Symbol::default() - } - } - - fn assert_requested_symbols(graph: &AssetGraph, node_index: NodeIndex, expected: Vec<&str>) { - assert_eq!( - graph.dependencies[graph.dependency_index(node_index).unwrap()].requested_symbols, - expected - .into_iter() - .map(|s| s.into()) - .collect::>() - ); - } - - fn add_asset( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - file_path: &str, - ) -> NodeIndex { - let index_asset = Asset { - file_path: PathBuf::from(file_path), - symbols: Some(symbols.iter().map(symbol).collect()), - ..Asset::default() - }; - graph.add_asset(parent_node, index_asset) - } - - fn add_dependency( - graph: &mut AssetGraph, - parent_node: NodeIndex, - symbols: Vec, - ) -> NodeIndex { - let dep = Dependency { - symbols: Some(symbols.iter().map(symbol).collect()), - ..Dependency::default() - }; - graph.add_dependency(parent_node, dep) - } - - #[test] - fn should_request_entry_asset() { - let mut requested = HashSet::new(); - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); - let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols( - index_asset_node, - entry_dep_node, - &mut |dependency_node_index, _dependency| { - requested.insert(dependency_node_index); - }, - ); - - assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); - assert_requested_symbols(&graph, dep_a_node, vec!["a"]); - } - - #[test] - fn should_propagate_named_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "a" from a.js and "b" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true), ("b", "b", true)], - "library.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be the only requested symbol - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec![]); - } - - #[test] - fn should_propagate_wildcard_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library.js - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from a.js and "*" from b.js - // only "a" is used in entry.js - let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); - - let mut requested_deps = Vec::new(); - graph.propagate_requested_symbols( - library_asset_node, - library_dep_node, - &mut |dependency_node_index, _dependency| { - requested_deps.push(dependency_node_index); - }, - ); - assert_eq!( - requested_deps, - vec![b_dep, a_dep], - "Should request both new deps" - ); - - // "a" should be marked as requested on all deps as wildcards make it - // unclear who the owning dep is - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - assert_requested_symbols(&graph, b_dep, vec!["a"]); - } - - #[test] - fn should_propagate_nested_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from library/index.js - let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); - let library_reexport_dep_node = - add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // library/index.js re-exports "a" from a.js - let library_asset_node = add_asset( - &mut graph, - library_reexport_dep_node, - vec![("a", "a", true)], - "library/index.js", - ); - let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); - graph.propagate_requested_symbols(library_entry_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on all deps until the a dep is reached - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); - assert_requested_symbols(&graph, a_dep, vec!["a"]); - } - - #[test] - fn should_propagate_renamed_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "b" from b.js renamed as "a" - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("b", "a", true)], - "library.js", - ); - let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "b" should be marked as requested on the b dep - assert_requested_symbols(&graph, b_dep, vec!["b"]); - } - - #[test] - fn should_propagate_namespace_reexports() { - let mut graph = AssetGraph::new(); - let target = Target::default(); - let dep = Dependency::entry(String::from("index.js"), target); - let entry_dep_node = graph.add_entry_dependency(dep); - - // entry.js imports "a" from library - let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); - let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); - graph.propagate_requested_symbols(entry_asset_node, entry_dep_node, &mut |_, _| {}); - - // library.js re-exports "*" from stuff.js renamed as "a"" - // export * as a from './stuff.js' - let library_asset_node = add_asset( - &mut graph, - library_dep_node, - vec![("a", "a", true)], - "library.js", - ); - let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); - graph.propagate_requested_symbols(library_asset_node, library_dep_node, &mut |_, _| {}); - - // "a" should be marked as requested on the library dep - assert_requested_symbols(&graph, library_dep_node, vec!["a"]); - // "*" should be marked as requested on the stuff dep - assert_requested_symbols(&graph, stuff_dep, vec!["*"]); - } - - #[test] - fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { - let mut graph = AssetGraph::new(); - - let entry = graph.add_entry_dependency(Dependency { - specifier: String::from("entry"), - ..Dependency::default() - }); - - let entry_asset = graph.add_asset( - entry, - Asset { - file_path: PathBuf::from("entry"), - ..Asset::default() - }, - ); - - for i in 1..100 { - graph.add_dependency( - entry_asset, - Dependency { - specifier: format!("dependency-{}", i), - ..Dependency::default() - }, - ); - } - - let max_str_len = 10000; - let nodes = graph.serialize_nodes(max_str_len)?; - - assert_eq!(nodes.len(), 7); - - // Assert each string is less than the max size - for node in nodes.iter() { - assert!(node.len() < max_str_len); - } - - // Assert all the nodes are included and in the correct order - let first_entry = serde_json::from_str::(&nodes[0])?; - let first_entry = first_entry.as_array().unwrap(); - - assert_eq!(get_type(&first_entry[0]), json!("root")); - assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); - assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); - - for i in 1..first_entry.len() - 2 { - assert_eq!( - get_dependency(&first_entry[i + 2]), - Some(json!(format!("dependency-{}", i))) - ); - } - - let mut specifier = first_entry.len() - 2; - for node in nodes[1..].iter() { - let entry = serde_json::from_str::(node)?; - let entry = entry.as_array().unwrap(); - - for value in entry { - assert_eq!( - get_dependency(value), - Some(json!(format!("dependency-{}", specifier))) - ); - - specifier += 1; - } - } - - Ok(()) - } - - fn get_type(node: &Value) -> Value { - node.get("type").unwrap().to_owned() - } - - fn get_dependency(value: &Value) -> Option { - assert_eq!(get_type(value), json!("dependency")); - - value - .get("value") - .unwrap() - .get("dependency") - .unwrap() - .get("specifier") - .map(|s| s.to_owned()) - } - - fn get_asset(value: &Value) -> Option { - assert_eq!(get_type(value), json!("asset")); - - value - .get("value") - .unwrap() - .get("filePath") - .map(|s| s.to_owned()) - } -} diff --git a/crates/atlaspack_core/src/asset_graph/asset_graph.rs b/crates/atlaspack_core/src/asset_graph/asset_graph.rs new file mode 100644 index 000000000..a7179e681 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/asset_graph.rs @@ -0,0 +1,500 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use petgraph::graph::NodeIndex; +use petgraph::stable_graph::StableDiGraph; +use petgraph::visit::EdgeRef; +use petgraph::visit::IntoEdgeReferences; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; + +#[derive(Clone, Debug, PartialEq)] +pub struct AssetNode { + pub asset: Asset, + pub requested_symbols: HashSet, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DependencyNode { + pub dependency: Arc, + pub requested_symbols: HashSet, + pub state: DependencyState, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum DependencyState { + New, + Deferred, + Excluded, + Resolved, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum AssetGraphNode { + Root, + Entry, + Asset(AssetNode), + Dependency(DependencyNode), +} + +#[derive(Clone, Debug)] +pub struct AssetGraph { + pub graph: StableDiGraph, + root_node_index: NodeIndex, +} + +impl Default for AssetGraph { + fn default() -> Self { + Self::new() + } +} + +impl AssetGraph { + pub fn new() -> Self { + let mut graph = StableDiGraph::new(); + let root_node_index = graph.add_node(AssetGraphNode::Root); + AssetGraph { + graph, + root_node_index, + } + } + + pub fn edges(&self) -> Vec { + let raw_edges = self.graph.edge_references(); + let mut edges = Vec::new(); + + for edge in raw_edges { + edges.push(edge.source().index() as u32); + edges.push(edge.target().index() as u32); + } + + edges + } + + pub fn nodes(&self) -> impl Iterator { + self.graph.node_weights() + } + + pub fn nodes_from(&self, node_index: &NodeIndex) -> Vec<(NodeIndex, &AssetGraphNode)> { + let mut result = vec![]; + + for edge in self.graph.edges_directed(*node_index, Direction::Outgoing) { + let target_idx = edge.target(); + let target = self.graph.node_weight(target_idx).unwrap(); + result.push((target_idx, target)); + } + + result + } + + pub fn root_node(&self) -> NodeIndex { + self.root_node_index + } + + pub fn get_node(&self, idx: &NodeIndex) -> Option<&AssetGraphNode> { + self.graph.node_weight(*idx) + } + + pub fn get_node_mut(&mut self, idx: &NodeIndex) -> Option<&mut AssetGraphNode> { + self.graph.node_weight_mut(*idx) + } + + pub fn add_asset(&mut self, asset: Asset) -> NodeIndex { + self.graph.add_node(AssetGraphNode::Asset(AssetNode { + asset, + requested_symbols: HashSet::default(), + })) + } + + pub fn get_asset_node(&self, idx: &NodeIndex) -> Option<&AssetNode> { + let value = self.graph.node_weight(*idx)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_node_mut(&mut self, idx: &NodeIndex) -> Option<&mut AssetNode> { + let value = self.graph.node_weight_mut(*idx)?; + let AssetGraphNode::Asset(asset_node) = value else { + return None; + }; + Some(asset_node) + } + + pub fn get_asset_nodes(&self) -> Vec<&AssetNode> { + let mut results = vec![]; + for n in self.nodes() { + let AssetGraphNode::Asset(asset) = n else { + continue; + }; + results.push(asset); + } + results + } + + pub fn add_dependency(&mut self, dependency: Dependency) -> NodeIndex { + self + .graph + .add_node(AssetGraphNode::Dependency(DependencyNode { + dependency: Arc::new(dependency), + requested_symbols: HashSet::default(), + state: DependencyState::New, + })) + } + + pub fn get_dependency_node(&self, idx: &NodeIndex) -> Option<&DependencyNode> { + let value = self.graph.node_weight(*idx)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn get_dependency_nodes(&self) -> Vec<&DependencyNode> { + let mut results = vec![]; + for n in self.nodes() { + let AssetGraphNode::Dependency(dependency) = n else { + continue; + }; + results.push(dependency); + } + results + } + + pub fn get_dependency_node_mut(&mut self, idx: &NodeIndex) -> Option<&mut DependencyNode> { + let value = self.graph.node_weight_mut(*idx)?; + let AssetGraphNode::Dependency(node) = value else { + return None; + }; + Some(node) + } + + pub fn add_entry_dependency(&mut self, dependency: Dependency) -> NodeIndex { + let is_library = dependency.env.is_library; + let dependency_idx = self.add_dependency(dependency); + self.add_edge(&self.root_node_index.clone(), &dependency_idx); + + if is_library { + if let Some(dependency_node) = self.get_dependency_node_mut(&dependency_idx) { + dependency_node.requested_symbols.insert("*".into()); + } + } + + dependency_idx + } + + pub fn add_edge(&mut self, from_idx: &NodeIndex, to_idx: &NodeIndex) { + self.graph.add_edge(*from_idx, *to_idx, ()); + } +} + +impl PartialEq for AssetGraph { + fn eq(&self, other: &Self) -> bool { + let nodes = self.nodes(); + let other_nodes = other.nodes(); + + let edges = self.edges(); + let other_edges = other.edges(); + + nodes.eq(other_nodes) && edges.eq(&other_edges) + } +} + +impl std::hash::Hash for AssetGraph { + fn hash(&self, state: &mut H) { + for node in self.graph.node_weights() { + std::mem::discriminant(node).hash(state); + match node { + AssetGraphNode::Asset(asset_node) => asset_node.asset.id.hash(state), + AssetGraphNode::Dependency(dependency_node) => dependency_node.dependency.id().hash(state), + _ => {} + } + } + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::types::Symbol; + use crate::types::Target; + + use super::super::propagate_requested_symbols::propagate_requested_symbols; + use super::*; + + type TestSymbol<'a> = (&'a str, &'a str, bool); + fn symbol(test_symbol: &TestSymbol) -> Symbol { + let (local, exported, is_weak) = test_symbol; + Symbol { + local: String::from(*local), + exported: String::from(*exported), + is_weak: is_weak.to_owned(), + ..Symbol::default() + } + } + + fn assert_requested_symbols(graph: &AssetGraph, idx: NodeIndex, expected: Vec<&str>) { + assert_eq!( + graph.get_dependency_node(&idx).unwrap().requested_symbols, + expected + .into_iter() + .map(|s| s.into()) + .collect::>() + ); + } + + fn add_asset( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + file_path: &str, + ) -> NodeIndex { + let index_asset = Asset { + file_path: PathBuf::from(file_path), + symbols: Some(symbols.iter().map(symbol).collect()), + ..Asset::default() + }; + let asset_nid = graph.add_asset(index_asset); + graph.add_edge(&parent_node, &asset_nid); + asset_nid + } + + fn add_dependency( + graph: &mut AssetGraph, + parent_node: NodeIndex, + symbols: Vec, + ) -> NodeIndex { + let dep = Dependency { + symbols: Some(symbols.iter().map(symbol).collect()), + ..Dependency::default() + }; + let node_index = graph.add_dependency(dep); + graph.add_edge(&parent_node, &node_index); + node_index + } + + #[test] + fn should_request_entry_asset() { + let mut requested = HashSet::new(); + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + let index_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "index.js"); + let dep_a_node = add_dependency(&mut graph, index_asset_node, vec![("a", "a", false)]); + + propagate_requested_symbols( + &mut graph, + index_asset_node, + entry_dep_node, + &mut |dependency_node_index, _| { + requested.insert(dependency_node_index); + }, + ); + + assert_eq!(requested, HashSet::from_iter(vec![dep_a_node])); + assert_requested_symbols(&graph, dep_a_node, vec!["a"]); + } + + #[test] + fn should_propagate_named_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node, &mut |_, _| {}); + + // library.js re-exports "a" from a.js and "b" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true), ("b", "b", true)], + "library.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + + let mut requested_deps = Vec::new(); + + propagate_requested_symbols( + &mut graph, + library_asset_node, + library_dep_node, + &mut |dependency_node_index, _| { + requested_deps.push(dependency_node_index); + }, + ); + + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be the only requested symbol + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec![]); + } + + #[test] + fn should_propagate_wildcard_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library.js + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node, &mut |_, _| {}); + + // library.js re-exports "*" from a.js and "*" from b.js + // only "a" is used in entry.js + let library_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("*", "*", true)]); + + let mut requested_deps = Vec::new(); + propagate_requested_symbols( + &mut graph, + library_asset_node, + library_dep_node, + &mut |dependency_node_index, _| { + requested_deps.push(dependency_node_index); + }, + ); + assert_eq!( + requested_deps, + vec![b_dep, a_dep], + "Should request both new deps" + ); + + // "a" should be marked as requested on all deps as wildcards make it + // unclear who the owning dep is + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + assert_requested_symbols(&graph, b_dep, vec!["a"]); + } + + #[test] + fn should_propagate_nested_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node, &mut |_, _| {}); + + // library.js re-exports "*" from library/index.js + let library_entry_asset_node = add_asset(&mut graph, library_dep_node, vec![], "library.js"); + let library_reexport_dep_node = + add_dependency(&mut graph, library_entry_asset_node, vec![("*", "*", true)]); + propagate_requested_symbols( + &mut graph, + library_entry_asset_node, + library_dep_node, + &mut |_, _| {}, + ); + + // library/index.js re-exports "a" from a.js + let library_asset_node = add_asset( + &mut graph, + library_reexport_dep_node, + vec![("a", "a", true)], + "library/index.js", + ); + let a_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "a", true)]); + propagate_requested_symbols( + &mut graph, + library_entry_asset_node, + library_dep_node, + &mut |_, _| {}, + ); + + // "a" should be marked as requested on all deps until the a dep is reached + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + assert_requested_symbols(&graph, library_reexport_dep_node, vec!["a"]); + assert_requested_symbols(&graph, a_dep, vec!["a"]); + } + + #[test] + fn should_propagate_renamed_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node, &mut |_, _| {}); + + // library.js re-exports "b" from b.js renamed as "a" + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("b", "a", true)], + "library.js", + ); + let b_dep = add_dependency(&mut graph, library_asset_node, vec![("b", "b", true)]); + propagate_requested_symbols( + &mut graph, + library_asset_node, + library_dep_node, + &mut |_, _| {}, + ); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "b" should be marked as requested on the b dep + assert_requested_symbols(&graph, b_dep, vec!["b"]); + } + + #[test] + fn should_propagate_namespace_reexports() { + let mut graph = AssetGraph::new(); + let target = Target::default(); + let dep = Dependency::entry(String::from("index.js"), target); + let entry_dep_node = graph.add_entry_dependency(dep); + + // entry.js imports "a" from library + let entry_asset_node = add_asset(&mut graph, entry_dep_node, vec![], "entry.js"); + let library_dep_node = add_dependency(&mut graph, entry_asset_node, vec![("a", "a", false)]); + propagate_requested_symbols(&mut graph, entry_asset_node, entry_dep_node, &mut |_, _| {}); + + // library.js re-exports "*" from stuff.js renamed as "a"" + // export * as a from './stuff.js' + let library_asset_node = add_asset( + &mut graph, + library_dep_node, + vec![("a", "a", true)], + "library.js", + ); + let stuff_dep = add_dependency(&mut graph, library_asset_node, vec![("a", "*", true)]); + propagate_requested_symbols( + &mut graph, + library_asset_node, + library_dep_node, + &mut |_, _| {}, + ); + + // "a" should be marked as requested on the library dep + assert_requested_symbols(&graph, library_dep_node, vec!["a"]); + // "*" should be marked as requested on the stuff dep + assert_requested_symbols(&graph, stuff_dep, vec!["*"]); + } +} diff --git a/crates/atlaspack_core/src/asset_graph/mod.rs b/crates/atlaspack_core/src/asset_graph/mod.rs new file mode 100644 index 000000000..222511364 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/mod.rs @@ -0,0 +1,8 @@ +#[allow(clippy::module_inception)] +mod asset_graph; +mod propagate_requested_symbols; +mod serialize_asset_graph; + +pub use self::asset_graph::*; +pub use self::propagate_requested_symbols::*; +pub use self::serialize_asset_graph::*; diff --git a/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs new file mode 100644 index 000000000..c3f7d2444 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/propagate_requested_symbols.rs @@ -0,0 +1,166 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use petgraph::graph::NodeIndex; +use petgraph::visit::EdgeRef; +use petgraph::Direction; + +use crate::types::Asset; +use crate::types::Dependency; +use crate::types::Symbol; + +use super::asset_graph::DependencyState; +use super::asset_graph::{AssetGraph, DependencyNode}; + +const CHAR_STAR: &str = "*"; + +/// Propagates the requested symbols from an incoming dependency to an asset, +/// and forwards those symbols to re-exported dependencies if needed. +/// This may result in assets becoming un-deferred and transformed if they +/// now have requested symbols. +pub fn propagate_requested_symbols( + asset_graph: &mut AssetGraph, + initial_asset_idx: NodeIndex, + initial_dependency_idx: NodeIndex, + on_undeferred: &mut F, +) where + F: FnMut(NodeIndex, Arc), +{ + let mut next = vec![(initial_asset_idx, initial_dependency_idx)]; + + while let Some((asset_idx, dependency_idx)) = next.pop() { + let mut dependency_re_exports = HashSet::::default(); + let mut dependency_wildcards = HashSet::::default(); + let mut asset_requested_symbols_buf = HashSet::::default(); + + let dependency_node = asset_graph.get_dependency_node(&dependency_idx).unwrap(); + let asset_node = asset_graph.get_asset_node(&asset_idx).unwrap(); + + if dependency_node.requested_symbols.contains(CHAR_STAR) { + // If the requested symbols includes the "*" namespace, we + // need to include all of the asset's exported symbols. + if let Some(symbols) = &asset_node.asset.symbols { + for sym in symbols { + if !asset_node.requested_symbols.contains(&sym.exported) { + continue; + } + asset_requested_symbols_buf.insert(sym.exported.clone()); + if !sym.is_weak { + continue; + } + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(sym.local.clone()); + } + } + + // Propagate to all export * wildcard dependencies. + dependency_wildcards.insert(CHAR_STAR.to_string()); + } else { + // Otherwise, add each of the requested symbols to the asset. + for sym in dependency_node.requested_symbols.iter() { + if asset_node.requested_symbols.contains(sym) { + continue; + } + asset_requested_symbols_buf.insert(sym.clone()); + + let Some(asset_symbol) = get_symbol_by_name(&asset_node.asset, sym) else { + // If symbol wasn't found in the asset or a named re-export. + // This means the symbol is in one of the export * wildcards, but we don't know + // which one yet, so we propagate it to _all_ wildcard dependencies. + dependency_wildcards.insert(sym.clone()); + continue; + }; + + if !asset_symbol.is_weak { + continue; + } + + // If the asset exports this symbol + // Propagate re-exported symbol to dependency. + dependency_re_exports.insert(asset_symbol.local.clone()); + } + } + + // Add dependencies to asset + asset_graph + .get_asset_node_mut(&asset_idx) + .unwrap() + .requested_symbols + .extend(asset_requested_symbols_buf); + + let deps: Vec<_> = asset_graph + .graph + .neighbors_directed(asset_idx, Direction::Outgoing) + .collect(); + + for nested_dependency_idx in deps { + let mut updated = false; + + { + let DependencyNode { + dependency, + requested_symbols, + state: _, + } = asset_graph + .get_dependency_node_mut(&nested_dependency_idx) + .unwrap(); + + if let Some(symbols) = &dependency.symbols { + for sym in symbols { + if sym.is_weak { + // This is a re-export. If it is a wildcard, add all unmatched symbols + // to this dependency, otherwise attempt to match a named re-export. + if sym.local == "*" { + for wildcard in &dependency_wildcards { + if requested_symbols.insert(wildcard.clone()) { + updated = true; + } + } + } else if dependency_re_exports.contains(&sym.local) + && requested_symbols.insert(sym.exported.clone()) + { + updated = true; + } + } else if requested_symbols.insert(sym.exported.clone()) { + // This is a normal import. Add the requested symbol. + updated = true; + } + } + } + } + + let DependencyNode { + dependency, + requested_symbols: _, + state, + } = asset_graph + .get_dependency_node(&nested_dependency_idx) + .unwrap(); + + // If the dependency was updated, propagate to the target asset if there is one, + // or un-defer this dependency so we transform the requested asset. + // We must always resolve new dependencies to determine whether they have side effects. + if updated || *state == DependencyState::New { + let Some(resolved) = asset_graph + .graph + .edges_directed(nested_dependency_idx, Direction::Outgoing) + .next() + else { + on_undeferred(nested_dependency_idx, Arc::clone(dependency)); + continue; + }; + if resolved.target() == asset_idx { + continue; + } + next.push((resolved.target(), nested_dependency_idx)) + } + } + } +} + +fn get_symbol_by_name<'a>(asset: &'a Asset, sym: &str) -> Option<&'a Symbol> { + asset + .symbols + .as_ref() + .and_then(|symbols| symbols.iter().find(|s| s.exported == *sym)) +} diff --git a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs new file mode 100644 index 000000000..2c4e13f85 --- /dev/null +++ b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs @@ -0,0 +1,172 @@ +use serde::Serialize; + +use crate::types::{Asset, Dependency}; + +use super::{AssetGraph, AssetGraphNode, DependencyState}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SerializedDependency { + id: String, + dependency: Dependency, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum SerializedAssetGraphNode { + Root, + Entry, + Asset { + value: Asset, + }, + Dependency { + value: SerializedDependency, + has_deferred: bool, + }, +} + +pub fn serialize_asset_graph( + asset_graph: &AssetGraph, + max_str_len: usize, +) -> serde_json::Result> { + let mut nodes: Vec = Vec::new(); + let mut curr_node = String::default(); + + for node in asset_graph.nodes() { + let serialized_node = match node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }; + + let str = serde_json::to_string(&serialized_node)?; + if curr_node.len() + str.len() < (max_str_len - 3) { + if !curr_node.is_empty() { + curr_node.push(','); + } + curr_node.push_str(&str); + } else { + // Add the existing node now as it has reached the max JavaScript string size + nodes.push(format!("[{curr_node}]")); + curr_node = str; + } + } + + // Add the current node if it did not overflow in size + if curr_node.len() < (max_str_len - 3) { + nodes.push(format!("[{curr_node}]")); + } + + Ok(nodes) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use serde_json::{json, Value}; + + use super::*; + + #[test] + fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { + let mut graph = AssetGraph::new(); + + let entry = graph.add_entry_dependency(Dependency { + specifier: String::from("entry"), + ..Dependency::default() + }); + + let entry_asset = graph.add_asset(Asset { + file_path: PathBuf::from("entry"), + ..Asset::default() + }); + + graph.add_edge(&entry, &entry_asset); + + for i in 1..100 { + let node_index = graph.add_dependency(Dependency { + specifier: format!("dependency-{}", i), + ..Dependency::default() + }); + graph.add_edge(&entry_asset, &node_index); + } + + let max_str_len = 10000; + let nodes = serialize_asset_graph(&graph, max_str_len)?; + + assert_eq!(nodes.len(), 7); + + // Assert each string is less than the max size + for node in nodes.iter() { + assert!(node.len() < max_str_len); + } + + // Assert all the nodes are included and in the correct order + let first_entry = serde_json::from_str::(&nodes[0])?; + let first_entry = first_entry.as_array().unwrap(); + + assert_eq!(get_type(&first_entry[0]), json!("root")); + assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); + assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); + + for i in 1..first_entry.len() - 2 { + assert_eq!( + get_dependency(&first_entry[i + 2]), + Some(json!(format!("dependency-{}", i))) + ); + } + + let mut specifier = first_entry.len() - 2; + for node in nodes[1..].iter() { + let entry = serde_json::from_str::(node)?; + let entry = entry.as_array().unwrap(); + + for value in entry { + assert_eq!( + get_dependency(value), + Some(json!(format!("dependency-{}", specifier))) + ); + + specifier += 1; + } + } + + Ok(()) + } + + fn get_type(node: &Value) -> Value { + node.get("type").unwrap().to_owned() + } + + fn get_dependency(value: &Value) -> Option { + assert_eq!(get_type(value), json!("dependency")); + + value + .get("value") + .unwrap() + .get("dependency") + .unwrap() + .get("specifier") + .map(|s| s.to_owned()) + } + + fn get_asset(value: &Value) -> Option { + assert_eq!(get_type(value), json!("asset")); + + value + .get("value") + .unwrap() + .get("filePath") + .map(|s| s.to_owned()) + } +} diff --git a/crates/node-bindings/src/atlaspack/atlaspack.rs b/crates/node-bindings/src/atlaspack/atlaspack.rs index 76ada51ab..6ed7f9d29 100644 --- a/crates/node-bindings/src/atlaspack/atlaspack.rs +++ b/crates/node-bindings/src/atlaspack/atlaspack.rs @@ -19,6 +19,7 @@ use atlaspack::rpc::nodejs::NodejsRpcFactory; use atlaspack::rpc::nodejs::NodejsWorker; use atlaspack::rpc::RpcFactoryRef; use atlaspack::Atlaspack; +use atlaspack_core::asset_graph::serialize_asset_graph; use atlaspack_core::types::AtlaspackOptions; use atlaspack_napi_helpers::JsTransferable; use atlaspack_package_manager::PackageManagerRef; @@ -150,8 +151,10 @@ impl AtlaspackNapi { let mut js_object = env.create_object()?; js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?; - js_object - .set_named_property("nodes", asset_graph.serialize_nodes(MAX_STRING_LENGTH)?)?; + js_object.set_named_property( + "nodes", + serialize_asset_graph(&asset_graph, MAX_STRING_LENGTH)?, + )?; NapiAtlaspackResult::ok(&env, js_object) }