From 45a39973c647cae5fa3bc97c891446ef0aca04ba Mon Sep 17 00:00:00 2001 From: hardfist Date: Sun, 22 Sep 2024 13:28:49 +0800 Subject: [PATCH] feat(core): integrate rspack_resolver with input_filesystem (#7858) --- Cargo.lock | 11 +- Cargo.toml | 1 + crates/node_binding/Cargo.toml | 1 + crates/node_binding/src/lib.rs | 5 +- crates/node_binding/src/resolver_factory.rs | 15 ++- crates/rspack_core/Cargo.toml | 4 +- .../rspack_core/src/compiler/compilation.rs | 10 +- crates/rspack_core/src/compiler/hmr.rs | 1 + .../src/compiler/make/repair/add.rs | 1 + .../src/compiler/make/repair/build.rs | 10 +- .../src/compiler/make/repair/mod.rs | 4 + crates/rspack_core/src/compiler/mod.rs | 33 ++++-- crates/rspack_core/src/external_module.rs | 3 +- crates/rspack_core/src/module.rs | 3 + crates/rspack_core/src/normal_module.rs | 2 +- crates/rspack_core/src/resolver/boxfs.rs | 32 ++++++ crates/rspack_core/src/resolver/factory.rs | 11 +- crates/rspack_core/src/resolver/mod.rs | 1 + .../rspack_core/src/resolver/resolver_impl.rs | 14 +-- crates/rspack_fs/Cargo.toml | 16 +-- crates/rspack_fs/src/error.rs | 3 - crates/rspack_fs/src/lib.rs | 19 +--- crates/rspack_fs/src/native.rs | 101 ++++++++++-------- crates/rspack_fs/src/sync.rs | 23 +++- crates/rspack_fs_node/Cargo.toml | 5 +- crates/rspack_fs_node/src/lib.rs | 13 +-- crates/rspack_fs_node/src/node.rs | 31 +++--- crates/rspack_loader_runner/Cargo.toml | 1 + crates/rspack_loader_runner/src/runner.rs | 18 +++- .../src/plugin/module_concatenation_plugin.rs | 1 + 30 files changed, 243 insertions(+), 150 deletions(-) create mode 100644 crates/rspack_core/src/resolver/boxfs.rs diff --git a/Cargo.lock b/Cargo.lock index a52c6036c71..cd8228ef932 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -950,9 +950,9 @@ dependencies = [ [[package]] name = "dunce" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "dyn-clone" @@ -3135,6 +3135,7 @@ dependencies = [ name = "rspack_fs" version = "0.1.0" dependencies = [ + "dunce", "futures", "rspack_error", "rspack_paths", @@ -3242,6 +3243,7 @@ dependencies = [ "regex", "rspack_collections", "rspack_error", + "rspack_fs", "rspack_paths", "rspack_sources", "rspack_util", @@ -3349,6 +3351,7 @@ dependencies = [ "rspack_collections", "rspack_core", "rspack_error", + "rspack_fs", "rspack_fs_node", "rspack_hash", "rspack_hook", @@ -3986,9 +3989,9 @@ dependencies = [ [[package]] name = "rspack_resolver" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e53e65a19bc1b2f784acaa4c481968778cf2f26db564beb70956822abd3293" +checksum = "51106b568dee8123f549ed5163818d4efa74fbca567a4bb88222b4f45e8b074d" dependencies = [ "cfg-if", "dashmap 6.0.1", diff --git a/Cargo.toml b/Cargo.toml index 76bfdce0442..31b0f6d3f24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,7 @@ proc-macro2 = { version = "1.0.79" } quote = { version = "1.0.35" } rayon = { version = "1.10.0" } regex = { version = "1.10.4" } +rspack_resolver = { version = "0.3.3", features = ["package_json_raw_json_api"] } ropey = "1.6.1" rspack_sources = { version = "=0.3.2" } rustc-hash = { version = "1.1.0" } diff --git a/crates/node_binding/Cargo.toml b/crates/node_binding/Cargo.toml index d3371e975e3..df785a65fb8 100644 --- a/crates/node_binding/Cargo.toml +++ b/crates/node_binding/Cargo.toml @@ -21,6 +21,7 @@ rspack_binding_values = { version = "0.1.0", path = "../rspack_binding_values rspack_collections = { version = "0.1.0", path = "../rspack_collections" } rspack_core = { version = "0.1.0", path = "../rspack_core" } rspack_error = { version = "0.1.0", path = "../rspack_error" } +rspack_fs = { version = "0.1.0", path = "../rspack_fs" } rspack_fs_node = { version = "0.1.0", path = "../rspack_fs_node" } rspack_hash = { version = "0.1.0", path = "../rspack_hash" } rspack_hook = { version = "0.1.0", path = "../rspack_hook" } diff --git a/crates/node_binding/src/lib.rs b/crates/node_binding/src/lib.rs index 63bd061b3ce..6feb78d6e12 100644 --- a/crates/node_binding/src/lib.rs +++ b/crates/node_binding/src/lib.rs @@ -69,10 +69,11 @@ impl Rspack { let rspack = rspack_core::Compiler::new( compiler_options, plugins, - Box::new( + Some(Box::new( AsyncNodeWritableFileSystem::new(output_filesystem) .map_err(|e| Error::from_reason(format!("Failed to create writable filesystem: {e}",)))?, - ), + )), + None, Some(resolver_factory), Some(loader_resolver_factory), ); diff --git a/crates/node_binding/src/resolver_factory.rs b/crates/node_binding/src/resolver_factory.rs index c367e7b6cf1..4d3517262c6 100644 --- a/crates/node_binding/src/resolver_factory.rs +++ b/crates/node_binding/src/resolver_factory.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use napi_derive::napi; use rspack_core::{Resolve, ResolverFactory}; +use rspack_fs::{NativeFileSystem, ReadableFileSystem}; use crate::{ raw_resolve::{ @@ -14,23 +15,30 @@ use crate::{ pub struct JsResolverFactory { pub(crate) resolver_factory: Option>, pub(crate) loader_resolver_factory: Option>, + pub(crate) input_filesystem: Arc, } #[napi] impl JsResolverFactory { #[napi(constructor)] pub fn new() -> napi::Result { + let input_filesystem = Arc::new(NativeFileSystem {}); Ok(Self { resolver_factory: None, loader_resolver_factory: None, + input_filesystem, }) } pub fn get_resolver_factory(&mut self, resolve_options: Resolve) -> Arc { match &self.resolver_factory { Some(resolver_factory) => resolver_factory.clone(), + None => { - let resolver_factory = Arc::new(ResolverFactory::new(resolve_options)); + let resolver_factory = Arc::new(ResolverFactory::new( + resolve_options, + self.input_filesystem.clone(), + )); self.resolver_factory = Some(resolver_factory.clone()); resolver_factory } @@ -41,7 +49,10 @@ impl JsResolverFactory { match &self.loader_resolver_factory { Some(resolver_factory) => resolver_factory.clone(), None => { - let resolver_factory = Arc::new(ResolverFactory::new(resolve_options)); + let resolver_factory = Arc::new(ResolverFactory::new( + resolve_options, + self.input_filesystem.clone(), + )); self.loader_resolver_factory = Some(resolver_factory.clone()); resolver_factory } diff --git a/crates/rspack_core/Cargo.toml b/crates/rspack_core/Cargo.toml index 5cd76ab5d8d..ffd45eb7683 100644 --- a/crates/rspack_core/Cargo.toml +++ b/crates/rspack_core/Cargo.toml @@ -31,7 +31,7 @@ regex = { workspace = true } rspack_ast = { version = "0.1.0", path = "../rspack_ast" } rspack_collections = { version = "0.1.0", path = "../rspack_collections" } rspack_error = { version = "0.1.0", path = "../rspack_error" } -rspack_fs = { version = "0.1.0", path = "../rspack_fs", features = ["async", "rspack-error"] } +rspack_fs = { version = "0.1.0", path = "../rspack_fs" } rspack_futures = { version = "0.1.0", path = "../rspack_futures" } rspack_hash = { version = "0.1.0", path = "../rspack_hash" } rspack_hook = { version = "0.1.0", path = "../rspack_hook" } @@ -39,7 +39,7 @@ rspack_loader_runner = { version = "0.1.0", path = "../rspack_loader_runner" } rspack_macros = { version = "0.1.0", path = "../rspack_macros" } rspack_paths = { version = "0.1.0", path = "../rspack_paths" } rspack_regex = { version = "0.1.0", path = "../rspack_regex" } -rspack_resolver = { version = "0.3.1", features = ["package_json_raw_json_api"] } +rspack_resolver = { workspace = true } rspack_sources = { workspace = true } rspack_util = { version = "0.1.0", path = "../rspack_util" } rustc-hash = { workspace = true } diff --git a/crates/rspack_core/src/compiler/compilation.rs b/crates/rspack_core/src/compiler/compilation.rs index bbbe6e7ce82..ef4d4d34475 100644 --- a/crates/rspack_core/src/compiler/compilation.rs +++ b/crates/rspack_core/src/compiler/compilation.rs @@ -7,6 +7,7 @@ use std::{ }; use dashmap::DashSet; +use derivative::Derivative; use indexmap::{IndexMap, IndexSet}; use itertools::Itertools; use rayon::prelude::*; @@ -14,6 +15,7 @@ use rspack_collections::{ Identifiable, Identifier, IdentifierDashMap, IdentifierMap, IdentifierSet, UkeySet, }; use rspack_error::{error, Diagnostic, Result, Severity}; +use rspack_fs::ReadableFileSystem; use rspack_futures::FuturesResults; use rspack_hash::{RspackHash, RspackHashDigest}; use rspack_hook::define_hook; @@ -129,8 +131,8 @@ impl Default for CompilationId { type ValueCacheVersions = HashMap; static COMPILATION_ID: AtomicU32 = AtomicU32::new(0); - -#[derive(Debug)] +#[derive(Derivative)] +#[derivative(Debug)] pub struct Compilation { /// get_compilation_hooks(compilation.id) id: CompilationId, @@ -193,6 +195,8 @@ pub struct Compilation { pub modified_files: HashSet, pub removed_files: HashSet, make_artifact: MakeArtifact, + #[derivative(Debug = "ignore")] + pub input_filesystem: Arc, } impl Compilation { @@ -228,6 +232,7 @@ impl Compilation { module_executor: Option, modified_files: HashSet, removed_files: HashSet, + input_filesystem: Arc, ) -> Self { let mutations = options.new_incremental_enabled().then(Mutations::default); Self { @@ -285,6 +290,7 @@ impl Compilation { make_artifact: Default::default(), modified_files, removed_files, + input_filesystem, } } diff --git a/crates/rspack_core/src/compiler/hmr.rs b/crates/rspack_core/src/compiler/hmr.rs index 8c318d6d003..864cd3fecb5 100644 --- a/crates/rspack_core/src/compiler/hmr.rs +++ b/crates/rspack_core/src/compiler/hmr.rs @@ -73,6 +73,7 @@ impl Compiler { Some(ModuleExecutor::default()), modified_files, removed_files, + self.input_filesystem.clone(), ); if let Some(state) = self.options.get_incremental_rebuild_make_state() { diff --git a/crates/rspack_core/src/compiler/make/repair/add.rs b/crates/rspack_core/src/compiler/make/repair/add.rs index db2d2d90a8c..8d98c3af939 100644 --- a/crates/rspack_core/src/compiler/make/repair/add.rs +++ b/crates/rspack_core/src/compiler/make/repair/add.rs @@ -77,6 +77,7 @@ impl Task for AddTask { resolver_factory: context.resolver_factory.clone(), compiler_options: context.compiler_options.clone(), plugin_driver: context.plugin_driver.clone(), + fs: context.fs.clone(), })]) } } diff --git a/crates/rspack_core/src/compiler/make/repair/build.rs b/crates/rspack_core/src/compiler/make/repair/build.rs index dc012b8e9c4..edbc12c3550 100644 --- a/crates/rspack_core/src/compiler/make/repair/build.rs +++ b/crates/rspack_core/src/compiler/make/repair/build.rs @@ -1,6 +1,8 @@ use std::{collections::VecDeque, sync::Arc}; +use derivative::Derivative; use rspack_error::{Diagnostic, IntoTWithDiagnosticArray}; +use rspack_fs::ReadableFileSystem; use super::{process_dependencies::ProcessDependenciesTask, MakeTaskContext}; use crate::{ @@ -10,13 +12,17 @@ use crate::{ SharedPluginDriver, }; -#[derive(Debug)] +#[derive(Derivative)] +#[derivative(Debug)] + pub struct BuildTask { pub module: Box, pub current_profile: Option>, pub resolver_factory: Arc, pub compiler_options: Arc, pub plugin_driver: SharedPluginDriver, + #[derivative(Debug = "ignore")] + pub fs: Arc, } #[async_trait::async_trait] @@ -31,6 +37,7 @@ impl Task for BuildTask { plugin_driver, current_profile, mut module, + fs, } = *self; if let Some(current_profile) = ¤t_profile { current_profile.mark_building_start(); @@ -53,6 +60,7 @@ impl Task for BuildTask { }, plugin_driver: plugin_driver.clone(), compiler_options: &compiler_options, + fs: fs.clone(), }, None, ) diff --git a/crates/rspack_core/src/compiler/make/repair/mod.rs b/crates/rspack_core/src/compiler/make/repair/mod.rs index 75f250fedee..d2a590adc60 100644 --- a/crates/rspack_core/src/compiler/make/repair/mod.rs +++ b/crates/rspack_core/src/compiler/make/repair/mod.rs @@ -6,6 +6,7 @@ pub mod process_dependencies; use std::sync::Arc; use rspack_error::Result; +use rspack_fs::ReadableFileSystem; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use super::MakeArtifact; @@ -21,6 +22,7 @@ use crate::{ pub struct MakeTaskContext { // compilation info pub plugin_driver: SharedPluginDriver, + pub fs: Arc, pub compiler_options: Arc, pub resolver_factory: Arc, pub loader_resolver_factory: Arc, @@ -41,6 +43,7 @@ impl MakeTaskContext { old_cache: compilation.old_cache.clone(), unaffected_modules_cache: compilation.unaffected_modules_cache.clone(), dependency_factories: compilation.dependency_factories.clone(), + fs: compilation.input_filesystem.clone(), artifact, } } @@ -68,6 +71,7 @@ impl MakeTaskContext { None, Default::default(), Default::default(), + self.fs.clone(), ); compilation.dependency_factories = self.dependency_factories.clone(); compilation.swap_make_artifact(&mut self.artifact); diff --git a/crates/rspack_core/src/compiler/mod.rs b/crates/rspack_core/src/compiler/mod.rs index 6f38dc889b5..e59a6580fd2 100644 --- a/crates/rspack_core/src/compiler/mod.rs +++ b/crates/rspack_core/src/compiler/mod.rs @@ -6,7 +6,9 @@ use std::sync::Arc; use derivative::Derivative; use rspack_error::Result; -use rspack_fs::AsyncWritableFileSystem; +use rspack_fs::{ + AsyncNativeFileSystem, AsyncWritableFileSystem, NativeFileSystem, ReadableFileSystem, +}; use rspack_futures::FuturesResults; use rspack_hook::define_hook; use rspack_paths::{Utf8Path, Utf8PathBuf}; @@ -55,6 +57,8 @@ pub struct Compiler { pub options: Arc, #[derivative(Debug = "ignore")] pub output_filesystem: Box, + #[derivative(Debug = "ignore")] + pub input_filesystem: Arc, pub compilation: Compilation, pub plugin_driver: SharedPluginDriver, pub resolver_factory: Arc, @@ -71,7 +75,9 @@ impl Compiler { pub fn new( options: CompilerOptions, plugins: Vec, - output_filesystem: Box, + output_filesystem: Option>, + // only supports passing input_filesystem in rust api, no support for js api + input_filesystem: Option>, // no need to pass resolve_factory in rust api resolver_factory: Option>, loader_resolver_factory: Option>, @@ -82,14 +88,26 @@ impl Compiler { debug_info.with_context(options.context.to_string()); } } - let resolver_factory = - resolver_factory.unwrap_or_else(|| Arc::new(ResolverFactory::new(options.resolve.clone()))); - let loader_resolver_factory = loader_resolver_factory - .unwrap_or_else(|| Arc::new(ResolverFactory::new(options.resolve_loader.clone()))); + let input_filesystem = input_filesystem.unwrap_or_else(|| Arc::new(NativeFileSystem {})); + + let resolver_factory = resolver_factory.unwrap_or_else(|| { + Arc::new(ResolverFactory::new( + options.resolve.clone(), + input_filesystem.clone(), + )) + }); + let loader_resolver_factory = loader_resolver_factory.unwrap_or_else(|| { + Arc::new(ResolverFactory::new( + options.resolve_loader.clone(), + input_filesystem.clone(), + )) + }); let (plugin_driver, options) = PluginDriver::new(options, plugins, resolver_factory.clone()); let old_cache = Arc::new(OldCache::new(options.clone())); let unaffected_modules_cache = Arc::new(UnaffectedModulesCache::default()); let module_executor = ModuleExecutor::default(); + let output_filesystem = output_filesystem.unwrap_or_else(|| Box::new(AsyncNativeFileSystem {})); + Self { options: options.clone(), compilation: Compilation::new( @@ -103,6 +121,7 @@ impl Compiler { Some(module_executor), Default::default(), Default::default(), + input_filesystem.clone(), ), output_filesystem, plugin_driver, @@ -111,6 +130,7 @@ impl Compiler { old_cache, emitted_asset_versions: Default::default(), unaffected_modules_cache, + input_filesystem, } } @@ -140,6 +160,7 @@ impl Compiler { Some(module_executor), Default::default(), Default::default(), + self.input_filesystem.clone(), ), ); diff --git a/crates/rspack_core/src/external_module.rs b/crates/rspack_core/src/external_module.rs index a7185adf593..d98b87350ed 100644 --- a/crates/rspack_core/src/external_module.rs +++ b/crates/rspack_core/src/external_module.rs @@ -1,5 +1,4 @@ -use std::borrow::Cow; -use std::iter; +use std::{borrow::Cow, iter}; use rspack_collections::{Identifiable, Identifier}; use rspack_error::{error, impl_empty_diagnosable_trait, Diagnostic, Result}; diff --git a/crates/rspack_core/src/module.rs b/crates/rspack_core/src/module.rs index 68733cc6a87..690df8bc669 100644 --- a/crates/rspack_core/src/module.rs +++ b/crates/rspack_core/src/module.rs @@ -1,12 +1,14 @@ use std::fmt::Display; use std::hash::Hash; use std::path::PathBuf; +use std::sync::Arc; use std::{any::Any, borrow::Cow, fmt::Debug}; use async_trait::async_trait; use json::JsonValue; use rspack_collections::{Identifiable, Identifier, IdentifierSet}; use rspack_error::{Diagnosable, Diagnostic, Result}; +use rspack_fs::ReadableFileSystem; use rspack_hash::RspackHashDigest; use rspack_sources::Source; use rspack_util::atom::Atom; @@ -28,6 +30,7 @@ pub struct BuildContext<'a> { pub runner_context: RunnerContext, pub plugin_driver: SharedPluginDriver, pub compiler_options: &'a CompilerOptions, + pub fs: Arc, } #[derive(Debug, Clone, Hash, PartialEq, Eq)] diff --git a/crates/rspack_core/src/normal_module.rs b/crates/rspack_core/src/normal_module.rs index 74b20f3d02d..b7510c1030c 100644 --- a/crates/rspack_core/src/normal_module.rs +++ b/crates/rspack_core/src/normal_module.rs @@ -228,7 +228,6 @@ impl NormalModule { build_info: None, build_meta: None, parsed: false, - source_map_kind: SourceMapKind::empty(), last_successful_build_meta: BuildMeta::default(), } @@ -415,6 +414,7 @@ impl Module for NormalModule { self.resource_data.clone(), Some(plugin.clone()), build_context.runner_context, + build_context.fs.clone(), ) .await; let (mut loader_result, ds) = match loader_result { diff --git a/crates/rspack_core/src/resolver/boxfs.rs b/crates/rspack_core/src/resolver/boxfs.rs new file mode 100644 index 00000000000..89bbd26f173 --- /dev/null +++ b/crates/rspack_core/src/resolver/boxfs.rs @@ -0,0 +1,32 @@ +use std::{io, sync::Arc}; + +use rspack_fs::ReadableFileSystem; +use rspack_resolver::{FileMetadata, FileSystem as ResolverFileSystem}; + +#[derive(Clone)] +pub struct BoxFS(Arc); + +impl BoxFS { + pub fn new(fs: Arc) -> Self { + Self(fs) + } +} +impl ResolverFileSystem for BoxFS { + fn read_to_string(&self, path: &std::path::Path) -> std::io::Result { + self.0.read(path).and_then(|x| { + String::from_utf8(x).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err)) + }) + } + + fn metadata(&self, path: &std::path::Path) -> std::io::Result { + self.0.metadata(path).map(FileMetadata::from) + } + + fn symlink_metadata(&self, path: &std::path::Path) -> std::io::Result { + self.0.symlink_metadata(path).map(FileMetadata::from) + } + + fn canonicalize(&self, path: &std::path::Path) -> std::io::Result { + self.0.canonicalize(path) + } +} diff --git a/crates/rspack_core/src/resolver/factory.rs b/crates/rspack_core/src/resolver/factory.rs index 0694ea1dd91..89ef2916bbb 100644 --- a/crates/rspack_core/src/resolver/factory.rs +++ b/crates/rspack_core/src/resolver/factory.rs @@ -1,6 +1,7 @@ use std::{hash::BuildHasherDefault, sync::Arc}; use dashmap::DashMap; +use rspack_fs::ReadableFileSystem; use rustc_hash::FxHasher; use super::resolver_impl::Resolver; @@ -23,21 +24,15 @@ pub struct ResolverFactory { resolvers: DashMap, BuildHasherDefault>, } -impl Default for ResolverFactory { - fn default() -> Self { - Self::new(Resolve::default()) - } -} - impl ResolverFactory { pub fn clear_cache(&self) { self.resolver.clear_cache(); } - pub fn new(options: Resolve) -> Self { + pub fn new(options: Resolve, fs: Arc) -> Self { Self { base_options: options.clone(), - resolver: Resolver::new(options), + resolver: Resolver::new(options, fs), resolvers: Default::default(), } } diff --git a/crates/rspack_core/src/resolver/mod.rs b/crates/rspack_core/src/resolver/mod.rs index 694c61f942b..0c0ac472732 100644 --- a/crates/rspack_core/src/resolver/mod.rs +++ b/crates/rspack_core/src/resolver/mod.rs @@ -1,3 +1,4 @@ +mod boxfs; mod factory; mod resolver_impl; use std::borrow::Borrow; diff --git a/crates/rspack_core/src/resolver/resolver_impl.rs b/crates/rspack_core/src/resolver/resolver_impl.rs index 4ac1ba18fd4..2a78b866ffa 100644 --- a/crates/rspack_core/src/resolver/resolver_impl.rs +++ b/crates/rspack_core/src/resolver/resolver_impl.rs @@ -8,11 +8,12 @@ use rspack_error::{ miette::{diagnostic, Diagnostic}, DiagnosticExt, Severity, TraceableError, }; +use rspack_fs::ReadableFileSystem; use rspack_loader_runner::DescriptionData; use rspack_paths::AssertUtf8; use rustc_hash::FxHashSet as HashSet; -use super::{ResolveResult, Resource}; +use super::{boxfs::BoxFS, ResolveResult, Resource}; use crate::{AliasMap, DependencyCategory, Resolve, ResolveArgs, ResolveOptionsWithDependencyType}; #[derive(Debug, Default, Clone)] @@ -78,17 +79,18 @@ impl<'a> ResolveInnerOptions<'a> { /// Internal caches are shared. #[derive(Debug)] pub struct Resolver { - resolver: rspack_resolver::Resolver, + resolver: rspack_resolver::ResolverGeneric, } impl Resolver { - pub fn new(options: Resolve) -> Self { - Self::new_rspack_resolver(options) + pub fn new(options: Resolve, fs: Arc) -> Self { + Self::new_rspack_resolver(options, fs) } - fn new_rspack_resolver(options: Resolve) -> Self { + fn new_rspack_resolver(options: Resolve, fs: Arc) -> Self { let options = to_rspack_resolver_options(options, false, DependencyCategory::Unknown); - let resolver = rspack_resolver::Resolver::new(options); + let boxfs = BoxFS::new(fs); + let resolver = rspack_resolver::ResolverGeneric::new_with_file_system(boxfs, options); Self { resolver } } diff --git a/crates/rspack_fs/Cargo.toml b/crates/rspack_fs/Cargo.toml index da0136ad6b9..669efe198e6 100644 --- a/crates/rspack_fs/Cargo.toml +++ b/crates/rspack_fs/Cargo.toml @@ -5,17 +5,11 @@ license = "MIT" name = "rspack_fs" repository = "https://github.com/web-infra-dev/rspack" version = "0.1.0" -[features] -async = ["dep:futures", "dep:tokio"] -default = ["native"] -native = [] -rspack-error = ["dep:rspack_error"] - - +[dependencies] +dunce = { version = "1.0.5" } [dependencies.rspack_error] -optional = true -path = "../rspack_error" -version = "0.1.0" +path = "../rspack_error" +version = "0.1.0" [dependencies.rspack_paths] @@ -23,10 +17,8 @@ path = "../rspack_paths" version = "0.1.0" [dependencies.futures] -optional = true workspace = true [dependencies.tokio] features = ["fs"] -optional = true workspace = true diff --git a/crates/rspack_fs/src/error.rs b/crates/rspack_fs/src/error.rs index 73c6d9d63b3..805a1846335 100644 --- a/crates/rspack_fs/src/error.rs +++ b/crates/rspack_fs/src/error.rs @@ -1,12 +1,10 @@ use std::fmt::Display; -#[cfg(feature = "rspack-error")] use rspack_error::{ miette::{self, Diagnostic}, thiserror::{self, Error}, }; -#[cfg(feature = "rspack-error")] #[derive(Debug, Error, Diagnostic)] #[error("Rspack FS Error: {0}")] struct FsError(#[source] std::io::Error); @@ -23,7 +21,6 @@ impl From for Error { } } -#[cfg(feature = "rspack-error")] impl From for rspack_error::Error { fn from(value: Error) -> Self { match value { diff --git a/crates/rspack_fs/src/lib.rs b/crates/rspack_fs/src/lib.rs index 4a413dca48d..36b3555138b 100644 --- a/crates/rspack_fs/src/lib.rs +++ b/crates/rspack_fs/src/lib.rs @@ -1,19 +1,10 @@ +pub mod r#async; mod macros; - -cfg_async! { - pub mod r#async; - pub use r#async::{AsyncFileSystem, AsyncReadableFileSystem, AsyncWritableFileSystem}; -} +mod native; +pub use r#async::{AsyncFileSystem, AsyncReadableFileSystem, AsyncWritableFileSystem}; pub mod sync; pub use sync::{FileSystem, ReadableFileSystem, WritableFileSystem}; - mod error; pub use error::{Error, Result}; - -cfg_native! { - mod native; - pub use native::{NativeFileSystem}; - - #[cfg(feature = "async")] - pub use native::AsyncNativeFileSystem; -} +pub use native::AsyncNativeFileSystem; +pub use native::NativeFileSystem; diff --git a/crates/rspack_fs/src/native.rs b/crates/rspack_fs/src/native.rs index ff30b559df9..ded705c0c9c 100644 --- a/crates/rspack_fs/src/native.rs +++ b/crates/rspack_fs/src/native.rs @@ -1,9 +1,12 @@ -use std::fs; +use std::{ + fs::{self, Metadata}, + io, + path::{Path, PathBuf}, +}; use rspack_paths::Utf8Path; use super::{ - cfg_async, sync::{ReadableFileSystem, WritableFileSystem}, Error, Result, }; @@ -25,54 +28,60 @@ impl WritableFileSystem for NativeFileSystem { } impl ReadableFileSystem for NativeFileSystem { - fn read(&self, file: &Utf8Path) -> Result> { - fs::read(file).map_err(Error::from) + fn read(&self, path: &Path) -> io::Result> { + fs::read(path) + } + + fn metadata(&self, path: &Path) -> io::Result { + fs::metadata(path) + } + + fn symlink_metadata(&self, path: &Path) -> io::Result { + fs::symlink_metadata(path) + } + + fn canonicalize(&self, path: &Path) -> io::Result { + dunce::canonicalize(path) } } -cfg_async! { - use futures::future::BoxFuture; - - use crate::{AsyncReadableFileSystem, AsyncWritableFileSystem}; - pub struct AsyncNativeFileSystem; - - impl AsyncWritableFileSystem for AsyncNativeFileSystem { - fn create_dir<'a>(&'a self, dir: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { - let dir = dir.to_path_buf(); - let fut = async move { tokio::fs::create_dir(dir).await.map_err(Error::from) }; - Box::pin(fut) - } - - fn create_dir_all<'a>(&'a self, dir: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { - let fut = async move { tokio::fs::create_dir_all(dir).await.map_err(Error::from) }; - Box::pin(fut) - } - - fn write<'a>( - &'a self, - file: &'a Utf8Path, - data: &'a [u8], - ) -> BoxFuture<'a, Result<()>> { - let fut = async move { tokio::fs::write(file, data).await.map_err(Error::from) }; - Box::pin(fut) - } - - fn remove_file<'a>(&'a self, file: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { - let fut = async move { tokio::fs::remove_file(file).await.map_err(Error::from) }; - Box::pin(fut) - } - - fn remove_dir_all<'a>(&'a self, dir: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { - let dir = dir.to_path_buf(); - let fut = async move { tokio::fs::remove_dir_all(dir).await.map_err(Error::from) }; - Box::pin(fut) - } +use futures::future::BoxFuture; + +use crate::{AsyncReadableFileSystem, AsyncWritableFileSystem}; +pub struct AsyncNativeFileSystem; + +impl AsyncWritableFileSystem for AsyncNativeFileSystem { + fn create_dir<'a>(&'a self, dir: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { + let dir = dir.to_path_buf(); + let fut = async move { tokio::fs::create_dir(dir).await.map_err(Error::from) }; + Box::pin(fut) } - impl AsyncReadableFileSystem for AsyncNativeFileSystem { - fn read<'a>(&'a self, file: &'a Utf8Path) -> BoxFuture<'a, Result>> { - let fut = async move { tokio::fs::read(file).await.map_err(Error::from) }; - Box::pin(fut) - } + fn create_dir_all<'a>(&'a self, dir: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { + let fut = async move { tokio::fs::create_dir_all(dir).await.map_err(Error::from) }; + Box::pin(fut) + } + + fn write<'a>(&'a self, file: &'a Utf8Path, data: &'a [u8]) -> BoxFuture<'a, Result<()>> { + let fut = async move { tokio::fs::write(file, data).await.map_err(Error::from) }; + Box::pin(fut) + } + + fn remove_file<'a>(&'a self, file: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { + let fut = async move { tokio::fs::remove_file(file).await.map_err(Error::from) }; + Box::pin(fut) + } + + fn remove_dir_all<'a>(&'a self, dir: &'a Utf8Path) -> BoxFuture<'a, Result<()>> { + let dir = dir.to_path_buf(); + let fut = async move { tokio::fs::remove_dir_all(dir).await.map_err(Error::from) }; + Box::pin(fut) + } +} + +impl AsyncReadableFileSystem for AsyncNativeFileSystem { + fn read<'a>(&'a self, file: &'a Utf8Path) -> BoxFuture<'a, Result>> { + let fut = async move { tokio::fs::read(file).await.map_err(Error::from) }; + Box::pin(fut) } } diff --git a/crates/rspack_fs/src/sync.rs b/crates/rspack_fs/src/sync.rs index 1b729aee4f4..c5b4d118848 100644 --- a/crates/rspack_fs/src/sync.rs +++ b/crates/rspack_fs/src/sync.rs @@ -1,5 +1,11 @@ +use std::fs::Metadata; +use std::io; +use std::path::Path; +use std::path::PathBuf; + use rspack_paths::Utf8Path; +// pubResolverFileSystem use super::Result; pub trait WritableFileSystem { @@ -23,11 +29,18 @@ pub trait WritableFileSystem { fn write(&self, file: &Utf8Path, data: &[u8]) -> Result<()>; } -pub trait ReadableFileSystem { - /// Read the entire contents of a file into a bytes vector. - /// - /// Error: This function will return an error if path does not already exist. - fn read(&self, file: &Utf8Path) -> Result>; +pub trait ReadableFileSystem: Send + Sync { + /// See [std::fs::read] + fn read(&self, path: &Path) -> io::Result>; + + /// See [std::fs::metadata] + fn metadata(&self, path: &Path) -> io::Result; + + /// See [std::fs::symlink_metadata] + fn symlink_metadata(&self, path: &Path) -> io::Result; + + /// See [std::fs::canonicalize] + fn canonicalize(&self, path: &Path) -> io::Result; } /// Readable and writable file system representation. diff --git a/crates/rspack_fs_node/Cargo.toml b/crates/rspack_fs_node/Cargo.toml index 3c4698665cd..cfbf620a23a 100644 --- a/crates/rspack_fs_node/Cargo.toml +++ b/crates/rspack_fs_node/Cargo.toml @@ -8,15 +8,12 @@ version = "0.1.0" [lib] crate-type = ["cdylib", "rlib"] -[features] -async = ["rspack_fs/async"] -default = ["async"] [dependencies] futures = { workspace = true } napi = { workspace = true, features = ["napi4", "tokio_rt"] } napi-derive = { workspace = true } -rspack_fs = { version = "0.1.0", path = "../rspack_fs", default-features = false, features = ["rspack-error"] } +rspack_fs = { version = "0.1.0", path = "../rspack_fs" } rspack_napi = { version = "0.1.0", path = "../rspack_napi" } rspack_paths = { version = "0.1.0", path = "../rspack_paths" } diff --git a/crates/rspack_fs_node/src/lib.rs b/crates/rspack_fs_node/src/lib.rs index e1bd40dfcf0..6ec5ce1977b 100644 --- a/crates/rspack_fs_node/src/lib.rs +++ b/crates/rspack_fs_node/src/lib.rs @@ -1,20 +1,13 @@ #![allow(clippy::unwrap_in_result)] -use rspack_fs::cfg_async; - -cfg_async! { - mod r#async; - pub use r#async::AsyncNodeWritableFileSystem; -} +mod r#async; +pub use r#async::AsyncNodeWritableFileSystem; mod sync; pub use sync::NodeWritableFileSystem; mod node; pub use node::NodeFS; - -cfg_async! { - pub use node::ThreadsafeNodeFS; -} +pub use node::ThreadsafeNodeFS; #[cfg(node)] mod node_test { diff --git a/crates/rspack_fs_node/src/node.rs b/crates/rspack_fs_node/src/node.rs index 823aa022488..7864331736d 100644 --- a/crates/rspack_fs_node/src/node.rs +++ b/crates/rspack_fs_node/src/node.rs @@ -1,6 +1,5 @@ use napi::{bindgen_prelude::Buffer, Env, JsFunction, Ref}; use napi_derive::napi; -use rspack_fs::cfg_async; pub(crate) struct JsFunctionRef { env: Env, @@ -55,21 +54,19 @@ pub(crate) struct NodeFSRef { pub(crate) mkdirp: JsFunctionRef, } -cfg_async! { - use napi::Either; - use rspack_napi::threadsafe_function::ThreadsafeFunction; +use napi::Either; +use rspack_napi::threadsafe_function::ThreadsafeFunction; - #[napi(object, object_to_js = false, js_name = "ThreadsafeNodeFS")] - pub struct ThreadsafeNodeFS { - #[napi(ts_type = "(name: string, content: Buffer) => Promise | void")] - pub write_file: ThreadsafeFunction<(String, Buffer), ()>, - #[napi(ts_type = "(name: string) => Promise | void")] - pub remove_file: ThreadsafeFunction, - #[napi(ts_type = "(name: string) => Promise | void")] - pub mkdir: ThreadsafeFunction, - #[napi(ts_type = "(name: string) => Promise | string | void")] - pub mkdirp: ThreadsafeFunction>, - #[napi(ts_type = "(name: string) => Promise | string | void")] - pub remove_dir_all: ThreadsafeFunction>, - } +#[napi(object, object_to_js = false, js_name = "ThreadsafeNodeFS")] +pub struct ThreadsafeNodeFS { + #[napi(ts_type = "(name: string, content: Buffer) => Promise | void")] + pub write_file: ThreadsafeFunction<(String, Buffer), ()>, + #[napi(ts_type = "(name: string) => Promise | void")] + pub remove_file: ThreadsafeFunction, + #[napi(ts_type = "(name: string) => Promise | void")] + pub mkdir: ThreadsafeFunction, + #[napi(ts_type = "(name: string) => Promise | string | void")] + pub mkdirp: ThreadsafeFunction>, + #[napi(ts_type = "(name: string) => Promise | string | void")] + pub remove_dir_all: ThreadsafeFunction>, } diff --git a/crates/rspack_loader_runner/Cargo.toml b/crates/rspack_loader_runner/Cargo.toml index e1808f0cf53..a9202e6f514 100644 --- a/crates/rspack_loader_runner/Cargo.toml +++ b/crates/rspack_loader_runner/Cargo.toml @@ -16,6 +16,7 @@ once_cell = { workspace = true } regex = { workspace = true } rspack_collections = { version = "0.1.0", path = "../rspack_collections" } rspack_error = { version = "0.1.0", path = "../rspack_error" } +rspack_fs = { version = "0.1.0", path = "../rspack_fs" } rspack_paths = { version = "0.1.0", path = "../rspack_paths" } rspack_sources = { workspace = true } rspack_util = { version = "0.1.0", path = "../rspack_util" } diff --git a/crates/rspack_loader_runner/src/runner.rs b/crates/rspack_loader_runner/src/runner.rs index 0cdfd65fc0f..ae0fda05703 100644 --- a/crates/rspack_loader_runner/src/runner.rs +++ b/crates/rspack_loader_runner/src/runner.rs @@ -1,8 +1,10 @@ use std::{fmt::Debug, path::PathBuf, sync::Arc}; use rspack_error::{error, IntoTWithDiagnosticArray, Result, TWithDiagnosticArray}; +use rspack_fs::ReadableFileSystem; use rspack_sources::SourceMap; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; +use tokio::task::spawn_blocking; use crate::{ content::{AdditionalData, Content, ResourceData}, @@ -25,6 +27,7 @@ impl LoaderContext { async fn process_resource( loader_context: &mut LoaderContext, + fs: Arc, ) -> Result<()> { if let Some(plugin) = &loader_context.plugin && let Some(processed_resource) = plugin @@ -39,9 +42,12 @@ async fn process_resource( if let Some(resource_path) = resource_data.resource_path.as_deref() && !resource_path.as_str().is_empty() { - let result = tokio::fs::read(resource_path) + let resource_path_owned = resource_path.to_owned(); + // use spawn_blocking to avoid block,see https://docs.rs/tokio/latest/src/tokio/fs/read.rs.html#48 + let result = spawn_blocking(move || fs.read(resource_path_owned.as_std_path())) .await - .map_err(|e| error!("{e}, failed to read {resource_path}"))?; + .map_err(|e| error!("{e}, spawn task failed"))?; + let result = result.map_err(|e| error!("{e}, failed to read {resource_path}"))?; loader_context.content = Some(Content::from(result)); } else if !resource_data.get_scheme().is_none() { let resource = &resource_data.resource; @@ -102,6 +108,7 @@ pub async fn run_loaders( resource_data: Arc, plugins: Option>>, context: Context, + fs: Arc, ) -> Result> { let loaders = loaders .into_iter() @@ -144,7 +151,7 @@ pub async fn run_loaders( } } State::ProcessResource => { - process_resource(&mut cx).await?; + process_resource(&mut cx, fs.clone()).await?; cx.loader_index = cx.loader_items.len() as i32 - 1; cx.state.transition(State::Normal); } @@ -234,6 +241,7 @@ mod test { use once_cell::sync::OnceCell; use rspack_collections::{Identifiable, Identifier}; use rspack_error::Result; + use rspack_fs::NativeFileSystem; use super::{run_loaders, Loader, LoaderContext, ResourceData}; use crate::{content::Content, plugin::LoaderRunnerPlugin, AdditionalData}; @@ -416,6 +424,7 @@ mod test { rs.clone(), Some(Arc::new(TestContentPlugin)), (), + Arc::new(NativeFileSystem {}) ) .await .err() @@ -433,6 +442,7 @@ mod test { rs.clone(), Some(Arc::new(TestContentPlugin)), (), + Arc::new(NativeFileSystem {}) ) .await .err() @@ -512,6 +522,7 @@ mod test { rs, Some(Arc::new(TestContentPlugin)), (), + Arc::new(NativeFileSystem {}), ) .await .unwrap(); @@ -574,6 +585,7 @@ mod test { rs, Some(Arc::new(TestContentPlugin)), (), + Arc::new(NativeFileSystem {}) ) .await .err() diff --git a/crates/rspack_plugin_javascript/src/plugin/module_concatenation_plugin.rs b/crates/rspack_plugin_javascript/src/plugin/module_concatenation_plugin.rs index 1db1876b6be..e998a178758 100644 --- a/crates/rspack_plugin_javascript/src/plugin/module_concatenation_plugin.rs +++ b/crates/rspack_plugin_javascript/src/plugin/module_concatenation_plugin.rs @@ -642,6 +642,7 @@ impl ModuleConcatenationPlugin { }, plugin_driver: compilation.plugin_driver.clone(), compiler_options: &compilation.options, + fs: compilation.input_filesystem.clone(), }, Some(compilation), )