diff --git a/crates/rspack_core/src/cache/persistent/storage/memory.rs b/crates/rspack_core/src/cache/persistent/storage/memory.rs index a83b10310a0..70c4bbd6fe6 100644 --- a/crates/rspack_core/src/cache/persistent/storage/memory.rs +++ b/crates/rspack_core/src/cache/persistent/storage/memory.rs @@ -1,7 +1,6 @@ use std::sync::{Arc, Mutex}; -use rspack_error::Result; -use rspack_storage::Storage; +use rspack_storage::{Result, Storage}; use rustc_hash::FxHashMap as HashMap; use tokio::sync::oneshot::{channel, Receiver}; diff --git a/crates/rspack_core/src/cache/persistent/storage/mod.rs b/crates/rspack_core/src/cache/persistent/storage/mod.rs index 28cb5df2f78..2ab30af3362 100644 --- a/crates/rspack_core/src/cache/persistent/storage/mod.rs +++ b/crates/rspack_core/src/cache/persistent/storage/mod.rs @@ -6,7 +6,7 @@ use std::{path::PathBuf, sync::Arc}; pub use memory::MemoryStorage; use rspack_fs::IntermediateFileSystem; pub use rspack_storage::Storage; -use rspack_storage::{PackBridgeFS, PackStorage, PackStorageOptions}; +use rspack_storage::{BridgeFileSystem, PackStorage, PackStorageOptions}; /// Storage Options /// @@ -31,7 +31,7 @@ pub fn create_storage( bucket_size: 20, pack_size: 500 * 1024, expire: 7 * 24 * 60 * 60 * 1000, - fs: Arc::new(PackBridgeFS(fs)), + fs: Arc::new(BridgeFileSystem(fs)), version, }; Arc::new(PackStorage::new(option)) diff --git a/crates/rspack_storage/src/error.rs b/crates/rspack_storage/src/error.rs new file mode 100644 index 00000000000..ea987744b78 --- /dev/null +++ b/crates/rspack_storage/src/error.rs @@ -0,0 +1,177 @@ +use rspack_error::miette; + +use crate::fs::{BatchFSError, FSError}; + +#[derive(Debug)] +pub struct InvalidDetail { + pub reason: String, + pub packs: Vec, +} + +#[derive(Debug)] +pub enum ValidateResult { + NotExists, + Valid, + Invalid(InvalidDetail), +} + +impl ValidateResult { + pub fn invalid(reason: &str) -> Self { + Self::Invalid(InvalidDetail { + reason: reason.to_string(), + packs: vec![], + }) + } + pub fn invalid_with_packs(reason: &str, packs: Vec) -> Self { + Self::Invalid(InvalidDetail { + reason: reason.to_string(), + packs, + }) + } + pub fn is_valid(&self) -> bool { + matches!(self, ValidateResult::Valid) + } +} + +#[derive(Debug)] +enum ErrorReason { + Reason(String), + Detail(InvalidDetail), + Error(Box), +} + +#[derive(Debug)] +pub enum ErrorType { + Validate, + Save, + Load, +} + +impl std::fmt::Display for ErrorType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ErrorType::Validate => write!(f, "validate"), + ErrorType::Save => write!(f, "save"), + ErrorType::Load => write!(f, "load"), + } + } +} + +#[derive(Debug)] +pub struct Error { + r#type: Option, + scope: Option<&'static str>, + inner: ErrorReason, +} + +impl From for Error { + fn from(e: FSError) -> Self { + Self { + r#type: None, + scope: None, + inner: ErrorReason::Error(Box::new(e)), + } + } +} + +impl From for Error { + fn from(e: BatchFSError) -> Self { + Self { + r#type: None, + scope: None, + inner: ErrorReason::Error(Box::new(e)), + } + } +} + +impl Error { + pub fn from_detail( + r#type: Option, + scope: Option<&'static str>, + detail: InvalidDetail, + ) -> Self { + Self { + r#type, + scope, + inner: ErrorReason::Detail(detail), + } + } + pub fn from_error( + r#type: Option, + scope: Option<&'static str>, + error: Box, + ) -> Self { + Self { + r#type, + scope, + inner: ErrorReason::Error(error), + } + } + pub fn from_reason( + r#type: Option, + scope: Option<&'static str>, + reason: String, + ) -> Self { + Self { + r#type, + scope, + inner: ErrorReason::Reason(reason), + } + } +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(t) = &self.r#type { + write!(f, "{} ", t)?; + } + if let Some(scope) = self.scope { + write!(f, "scope `{}` ", scope)?; + } + write!(f, "failed due to")?; + + match &self.inner { + ErrorReason::Detail(detail) => { + write!(f, " {}", detail.reason)?; + let mut pack_info_lines = detail + .packs + .iter() + .map(|p| format!("- {}", p)) + .collect::>(); + if pack_info_lines.len() > 5 { + pack_info_lines.truncate(5); + pack_info_lines.push("...".to_string()); + } + if !pack_info_lines.is_empty() { + write!(f, ":\n{}", pack_info_lines.join("\n"))?; + } + } + ErrorReason::Error(e) => { + write!(f, " {}", e)?; + } + ErrorReason::Reason(e) => { + write!(f, " {}", e)?; + } + } + Ok(()) + } +} + +impl std::error::Error for Error {} + +impl miette::Diagnostic for Error { + fn code<'a>(&'a self) -> Option> { + Some(Box::new(format!( + "Error::{}", + self + .r#type + .as_ref() + .map_or("".to_string(), |t| t.to_string()) + ))) + } + fn severity(&self) -> Option { + Some(miette::Severity::Warning) + } +} + +pub type Result = std::result::Result; diff --git a/crates/rspack_storage/src/fs/error.rs b/crates/rspack_storage/src/fs/error.rs new file mode 100644 index 00000000000..3acb07259f8 --- /dev/null +++ b/crates/rspack_storage/src/fs/error.rs @@ -0,0 +1,160 @@ +use std::io::ErrorKind; + +use cow_utils::CowUtils; +use rspack_paths::Utf8Path; +use tokio::task::JoinError; + +#[derive(Debug)] +pub enum FSOperation { + Read, + Write, + Dir, + Remove, + Stat, + Move, + Redirect, +} + +impl std::fmt::Display for FSOperation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Read => write!(f, "read"), + Self::Write => write!(f, "write"), + Self::Dir => write!(f, "create dir"), + Self::Remove => write!(f, "remove"), + Self::Stat => write!(f, "stat"), + Self::Move => write!(f, "move"), + Self::Redirect => write!(f, "redirect"), + } + } +} + +#[derive(Debug)] +pub struct FSError { + file: String, + inner: rspack_fs::Error, + opt: FSOperation, +} + +impl std::error::Error for FSError {} + +impl FSError { + pub fn from_fs_error(file: &Utf8Path, opt: FSOperation, error: rspack_fs::Error) -> Self { + Self { + file: file.to_string(), + inner: error, + opt, + } + } + pub fn from_message(file: &Utf8Path, opt: FSOperation, message: String) -> Self { + Self { + file: file.to_string(), + inner: rspack_fs::Error::Io(std::io::Error::new(std::io::ErrorKind::Other, message)), + opt, + } + } + pub fn is_not_found(&self) -> bool { + if matches!(self.kind(), ErrorKind::NotFound) { + return true; + } + let error_content = self.inner.to_string(); + let lower_case_error_content = error_content.cow_to_lowercase(); + lower_case_error_content.contains("no such file") + || lower_case_error_content.contains("file not exists") + } + pub fn kind(&self) -> ErrorKind { + match &self.inner { + rspack_fs::Error::Io(e) => e.kind(), + } + } +} + +impl std::fmt::Display for FSError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{} `{}` failed due to `{}`", + self.opt, + self.file, + match &self.inner { + rspack_fs::Error::Io(e) => e, + } + ) + } +} + +#[derive(Debug)] +pub struct BatchFSError { + message: String, + join_error: Option, + errors: Vec>, +} + +impl From for BatchFSError { + fn from(error: FSError) -> Self { + Self { + message: "".to_string(), + join_error: None, + errors: vec![Box::new(error)], + } + } +} + +impl std::fmt::Display for BatchFSError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message)?; + if let Some(join_error) = &self.join_error { + write!(f, " due to `{}`", join_error)?; + } + if self.errors.len() == 1 { + write!(f, "{}", self.errors[0])?; + } else { + for error in &self.errors { + write!(f, "\n- {}", error)?; + } + } + + Ok(()) + } +} + +impl BatchFSError { + pub fn try_from_joined_result( + message: &str, + res: Result>, JoinError>, + ) -> Result, Self> { + match res { + Ok(res) => Self::try_from_results(message, res), + Err(join_error) => Err(Self { + message: message.to_string(), + errors: vec![], + join_error: Some(join_error), + }), + } + } + + pub fn try_from_results( + message: &str, + results: Vec>, + ) -> Result, Self> { + let mut errors = vec![]; + let mut res = vec![]; + for r in results { + match r { + Ok(r) => res.push(r), + Err(e) => errors.push(Box::new(e).into()), + } + } + if errors.is_empty() { + Ok(res) + } else { + Err(Self { + message: message.to_string(), + errors, + join_error: None, + }) + } + } +} + +impl std::error::Error for BatchFSError {} diff --git a/crates/rspack_storage/src/fs/mod.rs b/crates/rspack_storage/src/fs/mod.rs new file mode 100644 index 00000000000..907c53b356c --- /dev/null +++ b/crates/rspack_storage/src/fs/mod.rs @@ -0,0 +1,346 @@ +use std::sync::Arc; + +mod error; +pub use error::{BatchFSError, FSError, FSOperation}; +use rspack_fs::{FileMetadata, IntermediateFileSystem, ReadStream, WriteStream}; +use rspack_paths::{Utf8Path, Utf8PathBuf}; +use rustc_hash::FxHashSet as HashSet; + +pub type FSResult = Result; + +#[async_trait::async_trait] +pub trait FileSystem: std::fmt::Debug + Sync + Send { + async fn exists(&self, path: &Utf8Path) -> FSResult; + async fn remove_dir(&self, path: &Utf8Path) -> FSResult<()>; + async fn ensure_dir(&self, path: &Utf8Path) -> FSResult<()>; + async fn write_file(&self, path: &Utf8Path) -> FSResult; + async fn read_file(&self, path: &Utf8Path) -> FSResult; + async fn read_dir(&self, path: &Utf8Path) -> FSResult>; + async fn metadata(&self, path: &Utf8Path) -> FSResult; + async fn remove_file(&self, path: &Utf8Path) -> FSResult<()>; + async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> FSResult<()>; +} + +#[derive(Debug)] +pub struct Writer { + path: Utf8PathBuf, + stream: Box, +} + +impl Writer { + pub async fn write_line(&mut self, line: &str) -> FSResult<()> { + self + .stream + .write_line(line) + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Write, e)) + } + pub async fn write(&mut self, buf: &[u8]) -> FSResult { + self + .stream + .write(buf) + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Write, e)) + } + pub async fn write_all(&mut self, buf: &[u8]) -> FSResult<()> { + self + .stream + .write_all(buf) + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Write, e)) + } + pub async fn flush(&mut self) -> FSResult<()> { + self + .stream + .flush() + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Write, e)) + } + pub async fn close(&mut self) -> FSResult<()> { + self + .stream + .close() + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Write, e)) + } +} + +#[derive(Debug)] +pub struct Reader { + path: Utf8PathBuf, + stream: Box, +} + +impl Reader { + pub async fn read_line(&mut self) -> FSResult { + self + .stream + .read_line() + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Read, e)) + } + pub async fn read(&mut self, length: usize) -> FSResult> { + self + .stream + .read(length) + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Read, e)) + } + pub async fn read_until(&mut self, byte: u8) -> FSResult> { + self + .stream + .read_until(byte) + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Read, e)) + } + pub async fn read_to_end(&mut self) -> FSResult> { + self + .stream + .read_to_end() + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Read, e)) + } + pub async fn skip(&mut self, offset: usize) -> FSResult<()> { + self + .stream + .skip(offset) + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Read, e)) + } + pub async fn close(&mut self) -> FSResult<()> { + self + .stream + .close() + .await + .map_err(|e| FSError::from_fs_error(&self.path, FSOperation::Read, e)) + } +} + +#[derive(Debug)] +pub struct BridgeFileSystem(pub Arc); + +#[async_trait::async_trait] +impl FileSystem for BridgeFileSystem { + async fn exists(&self, path: &Utf8Path) -> FSResult { + match self.metadata(path).await { + Ok(_) => Ok(true), + Err(e) => { + if e.is_not_found() { + Ok(false) + } else { + Err(e) + } + } + } + } + + async fn remove_dir(&self, path: &Utf8Path) -> FSResult<()> { + if self.exists(path).await? { + self + .0 + .remove_dir_all(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Remove, e))?; + } + Ok(()) + } + + async fn ensure_dir(&self, path: &Utf8Path) -> FSResult<()> { + self + .0 + .create_dir_all(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Dir, e))?; + Ok(()) + } + + async fn write_file(&self, path: &Utf8Path) -> FSResult { + if self.exists(path).await? { + self.remove_file(path).await?; + } + self + .ensure_dir(path.parent().expect("should have parent")) + .await?; + + let stream = self + .0 + .create_write_stream(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Write, e))?; + + Ok(Writer { + path: path.to_path_buf(), + stream, + }) + } + + async fn read_file(&self, path: &Utf8Path) -> FSResult { + let stream = self + .0 + .create_read_stream(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Read, e))?; + Ok(Reader { + path: path.to_path_buf(), + stream, + }) + } + + async fn read_dir(&self, path: &Utf8Path) -> FSResult> { + let files = self + .0 + .read_dir(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Read, e))?; + Ok(files.into_iter().collect::>()) + } + + async fn metadata(&self, path: &Utf8Path) -> FSResult { + let res = self + .0 + .stat(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Stat, e))?; + Ok(res) + } + + async fn remove_file(&self, path: &Utf8Path) -> FSResult<()> { + if self.exists(path).await? { + self + .0 + .remove_file(path) + .await + .map_err(|e| FSError::from_fs_error(path, FSOperation::Remove, e))?; + } + Ok(()) + } + + async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> FSResult<()> { + if self.exists(from).await? { + self + .ensure_dir(to.parent().expect("should have parent")) + .await?; + self + .0 + .rename(from, to) + .await + .map_err(|e| FSError::from_fs_error(from, FSOperation::Move, e))?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use rspack_fs::MemoryFileSystem; + use rspack_paths::Utf8PathBuf; + + use super::{BridgeFileSystem, FSResult}; + use crate::FileSystem; + + fn get_path(p: &str) -> Utf8PathBuf { + Utf8PathBuf::from(p) + } + + async fn test_create_dir(fs: &BridgeFileSystem) -> FSResult<()> { + fs.ensure_dir(&get_path("/parent/from")).await?; + fs.ensure_dir(&get_path("/parent/to")).await?; + + assert!(fs.exists(&get_path("/parent/from")).await?); + assert!(fs.exists(&get_path("/parent/to")).await?); + + assert!(fs.metadata(&get_path("/parent/from")).await?.is_directory); + assert!(fs.metadata(&get_path("/parent/to")).await?.is_directory); + + Ok(()) + } + + async fn test_write_file(fs: &BridgeFileSystem) -> FSResult<()> { + let mut writer = fs.write_file(&get_path("/parent/from/file.txt")).await?; + + writer.write_line("hello").await?; + writer.write(b" world").await?; + writer.flush().await?; + + assert!(fs.exists(&get_path("/parent/from/file.txt")).await?); + assert!( + fs.metadata(&get_path("/parent/from/file.txt")) + .await? + .is_file + ); + + Ok(()) + } + + async fn test_read_file(fs: &BridgeFileSystem) -> FSResult<()> { + let mut reader = fs.read_file(&get_path("/parent/from/file.txt")).await?; + + assert_eq!(reader.read_line().await?, "hello"); + assert_eq!(reader.read(b" world".len()).await?, b" world"); + + Ok(()) + } + + async fn test_move_file(fs: &BridgeFileSystem) -> FSResult<()> { + fs.move_file( + &get_path("/parent/from/file.txt"), + &get_path("/parent/to/file.txt"), + ) + .await?; + assert!(!fs.exists(&get_path("/parent/from/file.txt")).await?); + assert!(fs.exists(&get_path("/parent/to/file.txt")).await?); + assert!(fs.metadata(&get_path("/parent/to/file.txt")).await?.is_file); + + Ok(()) + } + + async fn test_remove_file(fs: &BridgeFileSystem) -> FSResult<()> { + fs.remove_file(&get_path("/parent/to/file.txt")).await?; + assert!(!fs.exists(&get_path("/parent/to/file.txt")).await?); + Ok(()) + } + + async fn test_remove_dir(fs: &BridgeFileSystem) -> FSResult<()> { + fs.remove_dir(&get_path("/parent/from")).await?; + fs.remove_dir(&get_path("/parent/to")).await?; + assert!(!fs.exists(&get_path("/parent/from")).await?); + assert!(!fs.exists(&get_path("/parent/to")).await?); + Ok(()) + } + + async fn test_error(fs: &BridgeFileSystem) -> FSResult<()> { + match fs.metadata(&get_path("/parent/from/not_exist.txt")).await { + Ok(_) => panic!("should error"), + Err(e) => assert_eq!( + e.to_string(), + r#"stat `/parent/from/not_exist.txt` failed due to `file not exist`"# + ), + }; + + Ok(()) + } + + async fn test_memory_fs(fs: &BridgeFileSystem) -> FSResult<()> { + test_create_dir(fs).await?; + test_write_file(fs).await?; + test_read_file(fs).await?; + test_move_file(fs).await?; + test_remove_file(fs).await?; + test_remove_dir(fs).await?; + test_error(fs).await?; + + Ok(()) + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn should_storage_bridge_fs_work() { + let fs = BridgeFileSystem(Arc::new(MemoryFileSystem::default())); + + let _ = test_memory_fs(&fs).await.map_err(|e| { + panic!("{}", e); + }); + } +} diff --git a/crates/rspack_storage/src/lib.rs b/crates/rspack_storage/src/lib.rs index 3cb0bf410d5..995494dc7cf 100644 --- a/crates/rspack_storage/src/lib.rs +++ b/crates/rspack_storage/src/lib.rs @@ -1,14 +1,17 @@ +mod error; +mod fs; mod pack; use std::sync::Arc; -pub use pack::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions}; -use rspack_error::Result; +pub use error::Result; +pub use fs::{BridgeFileSystem, FSError, FSOperation, FSResult, FileSystem, Reader, Writer}; +pub use pack::{PackStorage, PackStorageOptions}; use tokio::sync::oneshot::Receiver; -type StorageItemKey = Vec; -type StorageItemValue = Vec; -type StorageContent = Vec<(Arc, Arc)>; +type ItemKey = Vec; +type ItemValue = Vec; +type ItemPairs = Vec<(Arc, Arc)>; #[async_trait::async_trait] pub trait Storage: std::fmt::Debug + Sync + Send { diff --git a/crates/rspack_storage/src/pack/data/pack.rs b/crates/rspack_storage/src/pack/data/pack.rs index e7f12377453..ba62b0ac02d 100644 --- a/crates/rspack_storage/src/pack/data/pack.rs +++ b/crates/rspack_storage/src/pack/data/pack.rs @@ -2,10 +2,10 @@ use std::sync::Arc; use rspack_paths::Utf8PathBuf; -use crate::{StorageItemKey, StorageItemValue}; +use crate::{ItemKey, ItemValue}; -pub type PackKeys = Vec>; -pub type PackContents = Vec>; +pub type PackKeys = Vec>; +pub type PackContents = Vec>; #[derive(Debug, Default)] pub enum PackKeysState { diff --git a/crates/rspack_storage/src/pack/data/scope.rs b/crates/rspack_storage/src/pack/data/scope.rs index 4ca9181e7f8..a96e35e31e1 100644 --- a/crates/rspack_storage/src/pack/data/scope.rs +++ b/crates/rspack_storage/src/pack/data/scope.rs @@ -5,7 +5,7 @@ use rspack_paths::Utf8PathBuf; use rustc_hash::FxHashSet as HashSet; use super::{Pack, PackOptions, RootMeta, ScopeMeta}; -use crate::StorageContent; +use crate::ItemPairs; #[derive(Debug, Default)] pub enum RootMetaState { @@ -95,6 +95,7 @@ impl ScopePacksState { #[derive(Debug)] pub struct PackScope { + pub name: &'static str, pub path: Utf8PathBuf, pub options: Arc, pub meta: ScopeMetaState, @@ -103,8 +104,9 @@ pub struct PackScope { } impl PackScope { - pub fn new(path: Utf8PathBuf, options: Arc) -> Self { + pub fn new(name: &'static str, path: Utf8PathBuf, options: Arc) -> Self { Self { + name, path, options, meta: ScopeMetaState::Pending, @@ -113,8 +115,8 @@ impl PackScope { } } - pub fn empty(path: Utf8PathBuf, options: Arc) -> Self { - let mut scope = Self::new(path, options); + pub fn empty(name: &'static str, path: Utf8PathBuf, options: Arc) -> Self { + let mut scope = Self::new(name, path, options); scope.clear(); scope } @@ -130,7 +132,7 @@ impl PackScope { .all(|pack| pack.loaded()) } - pub fn get_contents(&self) -> StorageContent { + pub fn get_contents(&self) -> ItemPairs { self .packs .expect_value() diff --git a/crates/rspack_storage/src/pack/fs/error.rs b/crates/rspack_storage/src/pack/fs/error.rs deleted file mode 100644 index 4fc5fa52617..00000000000 --- a/crates/rspack_storage/src/pack/fs/error.rs +++ /dev/null @@ -1,79 +0,0 @@ -use std::io::ErrorKind; - -use cow_utils::CowUtils; -use rspack_error::{ - miette::{self}, - thiserror::{self, Error}, - Error, -}; -use rspack_paths::Utf8Path; - -#[derive(Debug)] -pub enum PackFsErrorOpt { - Read, - Write, - Dir, - Remove, - Stat, - Move, -} - -impl std::fmt::Display for PackFsErrorOpt { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Read => write!(f, "read"), - Self::Write => write!(f, "write"), - Self::Dir => write!(f, "create dir"), - Self::Remove => write!(f, "remove"), - Self::Stat => write!(f, "stat"), - Self::Move => write!(f, "move"), - } - } -} - -#[derive(Debug, Error)] -#[error(r#"Rspack Storage FS Error: {opt} `{file}` failed with `{inner}`"#)] -pub struct PackFsError { - file: String, - inner: Error, - opt: PackFsErrorOpt, - kind: Option, -} - -impl PackFsError { - pub fn from_fs_error(file: &Utf8Path, opt: PackFsErrorOpt, error: rspack_fs::Error) -> Self { - let kind = match &error { - rspack_fs::Error::Io(e) => Some(e.kind()), - }; - Self { - file: file.to_string(), - inner: error.into(), - opt, - kind, - } - } - pub fn is_not_found(&self) -> bool { - if self.kind.is_some_and(|k| matches!(k, ErrorKind::NotFound)) { - return true; - } - let error_content = self.inner.to_string(); - let lower_case_error_content = error_content.cow_to_lowercase(); - lower_case_error_content.contains("no such file") - || lower_case_error_content.contains("file not exists") - } -} - -impl miette::Diagnostic for PackFsError { - fn code<'a>(&'a self) -> Option> { - Some(Box::new("PackFsError")) - } - fn severity(&self) -> Option { - Some(miette::Severity::Warning) - } - fn url<'a>(&'a self) -> Option> { - Some(Box::new(self.file.clone())) - } - fn diagnostic_source(&self) -> Option<&dyn miette::Diagnostic> { - Some(self.inner.as_ref()) - } -} diff --git a/crates/rspack_storage/src/pack/fs/mod.rs b/crates/rspack_storage/src/pack/fs/mod.rs deleted file mode 100644 index d38eab3c821..00000000000 --- a/crates/rspack_storage/src/pack/fs/mod.rs +++ /dev/null @@ -1,249 +0,0 @@ -use std::sync::Arc; - -use rspack_error::Result; - -mod error; -pub use error::{PackFsError, PackFsErrorOpt}; -use rspack_fs::{FileMetadata, IntermediateFileSystem, ReadStream, WriteStream}; -use rspack_paths::Utf8Path; -use rustc_hash::FxHashSet as HashSet; - -#[async_trait::async_trait] -pub trait PackFS: std::fmt::Debug + Sync + Send { - async fn exists(&self, path: &Utf8Path) -> Result; - async fn remove_dir(&self, path: &Utf8Path) -> Result<()>; - async fn ensure_dir(&self, path: &Utf8Path) -> Result<()>; - async fn write_file(&self, path: &Utf8Path) -> Result>; - async fn read_file(&self, path: &Utf8Path) -> Result>; - async fn read_dir(&self, path: &Utf8Path) -> Result>; - async fn metadata(&self, path: &Utf8Path) -> Result; - async fn remove_file(&self, path: &Utf8Path) -> Result<()>; - async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()>; -} - -#[derive(Debug)] -pub struct PackBridgeFS(pub Arc); - -#[async_trait::async_trait] -impl PackFS for PackBridgeFS { - async fn exists(&self, path: &Utf8Path) -> Result { - match self.metadata(path).await { - Ok(_) => Ok(true), - Err(e) => match e.downcast::() { - Ok(e) => { - if e.is_not_found() { - Ok(false) - } else { - Err(e.into()) - } - } - Err(e) => Err(e), - }, - } - } - - async fn remove_dir(&self, path: &Utf8Path) -> Result<()> { - if self.exists(path).await? { - self - .0 - .remove_dir_all(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Remove, e))?; - } - Ok(()) - } - - async fn ensure_dir(&self, path: &Utf8Path) -> Result<()> { - self - .0 - .create_dir_all(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Dir, e))?; - Ok(()) - } - - async fn write_file(&self, path: &Utf8Path) -> Result> { - if self.exists(path).await? { - self.remove_file(path).await?; - } - self - .ensure_dir(path.parent().expect("should have parent")) - .await?; - - let res = self - .0 - .create_write_stream(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Write, e))?; - - Ok(res) - } - - async fn read_file(&self, path: &Utf8Path) -> Result> { - let res = self - .0 - .create_read_stream(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Read, e))?; - Ok(res) - } - - async fn read_dir(&self, path: &Utf8Path) -> Result> { - let files = self - .0 - .read_dir(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Read, e))?; - Ok(files.into_iter().collect::>()) - } - - async fn metadata(&self, path: &Utf8Path) -> Result { - let res = self - .0 - .stat(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Stat, e))?; - Ok(res) - } - - async fn remove_file(&self, path: &Utf8Path) -> Result<()> { - if self.exists(path).await? { - self - .0 - .remove_file(path) - .await - .map_err(|e| PackFsError::from_fs_error(path, PackFsErrorOpt::Remove, e))?; - } - Ok(()) - } - - async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()> { - if self.exists(from).await? { - self - .ensure_dir(to.parent().expect("should have parent")) - .await?; - self - .0 - .rename(from, to) - .await - .map_err(|e| PackFsError::from_fs_error(from, PackFsErrorOpt::Move, e))?; - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use rspack_error::Result; - use rspack_fs::MemoryFileSystem; - use rspack_paths::Utf8PathBuf; - - use super::PackBridgeFS; - use crate::PackFS; - - fn get_path(p: &str) -> Utf8PathBuf { - Utf8PathBuf::from(p) - } - - async fn test_create_dir(fs: &PackBridgeFS) -> Result<()> { - fs.ensure_dir(&get_path("/parent/from")).await?; - fs.ensure_dir(&get_path("/parent/to")).await?; - - assert!(fs.exists(&get_path("/parent/from")).await?); - assert!(fs.exists(&get_path("/parent/to")).await?); - - assert!(fs.metadata(&get_path("/parent/from")).await?.is_directory); - assert!(fs.metadata(&get_path("/parent/to")).await?.is_directory); - - Ok(()) - } - - async fn test_write_file(fs: &PackBridgeFS) -> Result<()> { - let mut writer = fs.write_file(&get_path("/parent/from/file.txt")).await?; - - writer.write_line("hello").await?; - writer.write(b" world").await?; - writer.flush().await?; - - assert!(fs.exists(&get_path("/parent/from/file.txt")).await?); - assert!( - fs.metadata(&get_path("/parent/from/file.txt")) - .await? - .is_file - ); - - Ok(()) - } - - async fn test_read_file(fs: &PackBridgeFS) -> Result<()> { - let mut reader = fs.read_file(&get_path("/parent/from/file.txt")).await?; - - assert_eq!(reader.read_line().await?, "hello"); - assert_eq!(reader.read(b" world".len()).await?, b" world"); - - Ok(()) - } - - async fn test_move_file(fs: &PackBridgeFS) -> Result<()> { - fs.move_file( - &get_path("/parent/from/file.txt"), - &get_path("/parent/to/file.txt"), - ) - .await?; - assert!(!fs.exists(&get_path("/parent/from/file.txt")).await?); - assert!(fs.exists(&get_path("/parent/to/file.txt")).await?); - assert!(fs.metadata(&get_path("/parent/to/file.txt")).await?.is_file); - - Ok(()) - } - - async fn test_remove_file(fs: &PackBridgeFS) -> Result<()> { - fs.remove_file(&get_path("/parent/to/file.txt")).await?; - assert!(!fs.exists(&get_path("/parent/to/file.txt")).await?); - Ok(()) - } - - async fn test_remove_dir(fs: &PackBridgeFS) -> Result<()> { - fs.remove_dir(&get_path("/parent/from")).await?; - fs.remove_dir(&get_path("/parent/to")).await?; - assert!(!fs.exists(&get_path("/parent/from")).await?); - assert!(!fs.exists(&get_path("/parent/to")).await?); - Ok(()) - } - - async fn test_error(fs: &PackBridgeFS) -> Result<()> { - match fs.metadata(&get_path("/parent/from/not_exist.txt")).await { - Ok(_) => panic!("should error"), - Err(e) => assert_eq!( - e.to_string(), - r#"Rspack Storage FS Error: stat `/parent/from/not_exist.txt` failed with `Rspack FS Error: file not exist`"# - ), - }; - - Ok(()) - } - - async fn test_memory_fs(fs: &PackBridgeFS) -> Result<()> { - test_create_dir(fs).await?; - test_write_file(fs).await?; - test_read_file(fs).await?; - test_move_file(fs).await?; - test_remove_file(fs).await?; - test_remove_dir(fs).await?; - test_error(fs).await?; - - Ok(()) - } - - #[tokio::test] - #[cfg_attr(miri, ignore)] - async fn should_pack_bridge_fs_work() { - let fs = PackBridgeFS(Arc::new(MemoryFileSystem::default())); - - let _ = test_memory_fs(&fs).await.map_err(|e| { - panic!("{}", e); - }); - } -} diff --git a/crates/rspack_storage/src/pack/manager/mod.rs b/crates/rspack_storage/src/pack/manager/mod.rs index b6dae30869f..c58256e85ef 100644 --- a/crates/rspack_storage/src/pack/manager/mod.rs +++ b/crates/rspack_storage/src/pack/manager/mod.rs @@ -7,15 +7,15 @@ use itertools::Itertools; use pollster::block_on; use queue::TaskQueue; use rayon::iter::{ParallelBridge, ParallelIterator}; -use rspack_error::{error, Error, Result}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use tokio::sync::oneshot::Receiver; use tokio::sync::{oneshot, Mutex}; use super::data::{PackOptions, PackScope, RootMeta, RootMetaState, RootOptions}; -use super::strategy::{ScopeStrategy, ValidateResult, WriteScopeResult}; +use super::strategy::{ScopeStrategy, WriteScopeResult}; use super::ScopeUpdates; -use crate::StorageContent; +use crate::error::{Error, ErrorType, ValidateResult}; +use crate::{ItemPairs, Result}; type ScopeMap = HashMap; @@ -69,7 +69,7 @@ impl ScopeManager { ))); Ok(()) } - Err(e) => Err(Error::from(e)), + Err(e) => Err(e), } })?; @@ -110,13 +110,19 @@ impl ScopeManager { .clear(); } - pub async fn load(&self, name: &'static str) -> Result { + pub async fn load(&self, name: &'static str) -> Result { self .scopes .lock() .await .entry(name.to_string()) - .or_insert_with(|| PackScope::new(self.strategy.get_path(name), self.pack_options.clone())); + .or_insert_with(|| { + PackScope::new( + name, + self.strategy.get_path(name), + self.pack_options.clone(), + ) + }); // only check lock file and root meta for the first time if matches!(*self.root_meta.lock().await, RootMetaState::Pending) { @@ -148,15 +154,23 @@ impl ScopeManager { Ok(vec![]) } // clear scope if invalid - ValidateResult::Invalid(_) => { + ValidateResult::Invalid(detail) => { self.clear_scope(name).await; - Err(error!(validated.to_string())) + Err(Error::from_detail( + Some(ErrorType::Validate), + Some(name), + detail, + )) } }, Err(e) => { // clear scope if error self.clear_scope(name).await; - Err(Error::from(e)) + Err(Error::from_error( + Some(ErrorType::Validate), + Some(name), + Box::new(e), + )) } } } @@ -197,9 +211,13 @@ fn update_scopes( strategy: &dyn ScopeStrategy, ) -> Result<()> { for (scope_name, _) in updates.iter() { - scopes - .entry(scope_name.to_string()) - .or_insert_with(|| PackScope::empty(strategy.get_path(scope_name), pack_options.clone())); + scopes.entry(scope_name.to_string()).or_insert_with(|| { + PackScope::empty( + scope_name, + strategy.get_path(scope_name), + pack_options.clone(), + ) + }); } scopes @@ -268,28 +286,27 @@ async fn save_scopes( mod tests { use std::sync::Arc; - use rspack_error::Result; use rspack_fs::MemoryFileSystem; use rspack_paths::{Utf8Path, Utf8PathBuf}; use rustc_hash::FxHashMap as HashMap; use crate::{ + error::Result, pack::{ data::{PackOptions, RootOptions}, - fs::{PackBridgeFS, PackFS}, manager::ScopeManager, strategy::SplitPackStrategy, }, - StorageItemKey, StorageItemValue, + BridgeFileSystem, FileSystem, ItemKey, ItemValue, }; - fn mock_key(id: usize) -> StorageItemKey { + fn mock_key(id: usize) -> ItemKey { format!("{:0>length$}_key", id, length = 46) .as_bytes() .to_vec() } - fn mock_insert_value(id: usize) -> Option { + fn mock_insert_value(id: usize) -> Option { Some( format!("{:0>length$}_val", id, length = 46) .as_bytes() @@ -297,7 +314,7 @@ mod tests { ) } - fn mock_update_value(id: usize) -> Option { + fn mock_update_value(id: usize) -> Option { Some( format!("{:0>length$}_new", id, length = 46) .as_bytes() @@ -305,7 +322,11 @@ mod tests { ) } - async fn test_cold_start(root: &Utf8Path, temp: &Utf8Path, fs: Arc) -> Result<()> { + async fn test_cold_start( + root: &Utf8Path, + temp: &Utf8Path, + fs: Arc, + ) -> Result<()> { let root_options = Arc::new(RootOptions { expire: 60000, root: root.parent().expect("should get parent").to_path_buf(), @@ -359,7 +380,7 @@ mod tests { Ok(()) } - async fn test_hot_start(root: &Utf8Path, temp: &Utf8Path, fs: Arc) -> Result<()> { + async fn test_hot_start(root: &Utf8Path, temp: &Utf8Path, fs: Arc) -> Result<()> { let root_options = Arc::new(RootOptions { expire: 60000, root: root.parent().expect("should get parent").to_path_buf(), @@ -437,7 +458,11 @@ mod tests { Ok(()) } - async fn test_invalid_start(root: &Utf8Path, temp: &Utf8Path, fs: Arc) -> Result<()> { + async fn test_invalid_start( + root: &Utf8Path, + temp: &Utf8Path, + fs: Arc, + ) -> Result<()> { let root_options = Arc::new(RootOptions { expire: 60000, root: root.parent().expect("should get parent").to_path_buf(), @@ -458,7 +483,7 @@ mod tests { // should report error when invalid failed assert_eq!( manager.load("scope1").await.unwrap_err().to_string(), - "validation failed due to `options.bucketSize` changed" + "validate scope `scope1` failed due to `options.bucketSize` changed" ); // clear after invalid, can be used as a empty scope @@ -484,7 +509,7 @@ mod tests { } async fn test_manager() -> Result<()> { - let fs = Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))); + let fs = Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))); let root = Utf8PathBuf::from("/cache/test_manager"); let temp = Utf8PathBuf::from("/temp/test_manager"); test_cold_start(&root, &temp, fs.clone()).await?; diff --git a/crates/rspack_storage/src/pack/mod.rs b/crates/rspack_storage/src/pack/mod.rs index 1fa676e1029..e47f262287f 100644 --- a/crates/rspack_storage/src/pack/mod.rs +++ b/crates/rspack_storage/src/pack/mod.rs @@ -1,5 +1,4 @@ mod data; -mod fs; mod manager; mod strategy; @@ -9,15 +8,13 @@ use std::{ }; use data::{PackOptions, RootOptions}; -pub use fs::{PackBridgeFS, PackFS}; use manager::ScopeManager; -use rspack_error::Result; use rspack_paths::AssertUtf8; use rustc_hash::FxHashMap as HashMap; use strategy::{ScopeUpdate, SplitPackStrategy}; use tokio::sync::oneshot::Receiver; -use crate::{Storage, StorageContent, StorageItemKey, StorageItemValue}; +use crate::{error::Result, FileSystem, ItemKey, ItemPairs, ItemValue, Storage}; pub type ScopeUpdates = HashMap<&'static str, ScopeUpdate>; #[derive(Debug)] @@ -29,7 +26,7 @@ pub struct PackStorage { pub struct PackStorageOptions { pub root: PathBuf, pub temp_root: PathBuf, - pub fs: Arc, + pub fs: Arc, pub bucket_size: usize, pub pack_size: usize, pub expire: u64, @@ -63,10 +60,10 @@ impl PackStorage { #[async_trait::async_trait] impl Storage for PackStorage { - async fn load(&self, name: &'static str) -> Result { + async fn load(&self, name: &'static str) -> Result { self.manager.load(name).await } - fn set(&self, scope: &'static str, key: StorageItemKey, value: StorageItemValue) { + fn set(&self, scope: &'static str, key: ItemKey, value: ItemValue) { let mut updates = self.updates.lock().expect("should get lock"); let scope_update = updates.entry(scope).or_default(); scope_update.insert(key, Some(value)); diff --git a/crates/rspack_storage/src/pack/strategy/mod.rs b/crates/rspack_storage/src/pack/strategy/mod.rs index 03de839a4bf..0952631e8e8 100644 --- a/crates/rspack_storage/src/pack/strategy/mod.rs +++ b/crates/rspack_storage/src/pack/strategy/mod.rs @@ -1,7 +1,6 @@ mod split; use async_trait::async_trait; -use rspack_error::Result; use rspack_paths::{Utf8Path, Utf8PathBuf}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; pub use split::SplitPackStrategy; @@ -9,7 +8,10 @@ pub use split::SplitPackStrategy; use super::data::{ Pack, PackContents, PackFileMeta, PackKeys, PackOptions, PackScope, RootMeta, RootOptions, }; -use crate::{StorageItemKey, StorageItemValue}; +use crate::{ + error::{Result, ValidateResult}, + ItemKey, ItemValue, +}; pub struct UpdatePacksResult { pub new_packs: Vec<(PackFileMeta, Pack)>, @@ -56,7 +58,7 @@ pub trait PackWriteStrategy { dir: Utf8PathBuf, options: &PackOptions, packs: HashMap, - updates: HashMap>, + updates: HashMap>, ) -> UpdatePacksResult; async fn write_pack(&self, pack: &Pack) -> Result<()>; } @@ -70,67 +72,6 @@ pub trait ScopeReadStrategy { async fn ensure_contents(&self, scope: &mut PackScope) -> Result<()>; } -#[derive(Debug)] -pub struct InvalidDetail { - pub reason: String, - pub packs: Vec, -} - -#[derive(Debug)] -pub enum ValidateResult { - NotExists, - Valid, - Invalid(InvalidDetail), -} - -impl ValidateResult { - pub fn invalid(reason: &str) -> Self { - Self::Invalid(InvalidDetail { - reason: reason.to_string(), - packs: vec![], - }) - } - pub fn invalid_with_packs(reason: &str, packs: Vec) -> Self { - Self::Invalid(InvalidDetail { - reason: reason.to_string(), - packs, - }) - } - pub fn is_valid(&self) -> bool { - matches!(self, ValidateResult::Valid) - } -} - -impl std::fmt::Display for ValidateResult { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ValidateResult::NotExists => write!(f, "validation failed due to not exists"), - ValidateResult::Valid => write!(f, "validation passed"), - ValidateResult::Invalid(e) => { - let mut pack_info_lines = e - .packs - .iter() - .map(|p| format!("- {}", p)) - .collect::>(); - if pack_info_lines.len() > 5 { - pack_info_lines.truncate(5); - pack_info_lines.push("...".to_string()); - } - write!( - f, - "validation failed due to {}{}", - e.reason, - if pack_info_lines.is_empty() { - "".to_string() - } else { - format!(":\n{}", pack_info_lines.join("\n")) - } - ) - } - } - } -} - #[async_trait] pub trait ScopeValidateStrategy { async fn validate_meta(&self, scope: &mut PackScope) -> Result; @@ -150,7 +91,7 @@ impl WriteScopeResult { } } -pub type ScopeUpdate = HashMap>; +pub type ScopeUpdate = HashMap>; #[async_trait] pub trait ScopeWriteStrategy { fn update_scope(&self, scope: &mut PackScope, updates: ScopeUpdate) -> Result<()>; diff --git a/crates/rspack_storage/src/pack/strategy/split/handle_file.rs b/crates/rspack_storage/src/pack/strategy/split/handle_file.rs index ea50952e8a4..1080b040c77 100644 --- a/crates/rspack_storage/src/pack/strategy/split/handle_file.rs +++ b/crates/rspack_storage/src/pack/strategy/split/handle_file.rs @@ -1,21 +1,25 @@ use std::sync::Arc; -use futures::{future::join_all, TryFutureExt}; -use rspack_error::{error, Result}; +use futures::future::join_all; use rspack_paths::{Utf8Path, Utf8PathBuf}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; +use tokio::task::JoinError; use crate::{ + fs::{BatchFSError, FSError, FSOperation}, pack::data::{current_time, PackScope, RootMeta, RootOptions, ScopeMeta}, - PackFS, + FileSystem, }; +type HandleFileResult = Result; +type BatchHandleFileResult = Result; + pub async fn prepare_scope( scope_path: &Utf8Path, root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result<()> { + fs: Arc, +) -> HandleFileResult<()> { let temp_path = redirect_to_path(scope_path, root, temp_root)?; fs.remove_dir(&temp_path).await?; fs.ensure_dir(&temp_path).await?; @@ -27,62 +31,43 @@ pub async fn prepare_scope_dirs( scopes: &HashMap, root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result<()> { + fs: Arc, +) -> BatchHandleFileResult<()> { let tasks = scopes.values().map(|scope| { let fs = fs.clone(); let scope_path = scope.path.clone(); let root_path = root.to_path_buf(); let temp_root_path = temp_root.to_path_buf(); tokio::spawn(async move { prepare_scope(&scope_path, &root_path, &temp_root_path, fs).await }) - .map_err(|e| error!("{e}")) }); - let res = join_all(tasks) - .await - .into_iter() - .collect::>>()?; - - let mut errors = vec![]; - for task_result in res { - if let Err(e) = task_result { - errors.push(format!("- {}", e)); - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(error!( - "prepare scopes directories failed:\n{}", - errors.join("\n") - )) - } + BatchFSError::try_from_joined_result( + "prepare scopes directories failed", + join_all(tasks) + .await + .into_iter() + .collect::, JoinError>>(), + ) + .map(|_| ()) } -pub async fn remove_files(files: HashSet, fs: Arc) -> Result<()> { +pub async fn remove_files( + files: HashSet, + fs: Arc, +) -> BatchHandleFileResult<()> { let tasks = files.into_iter().map(|path| { let fs = fs.clone(); - tokio::spawn(async move { fs.remove_file(&path).await }).map_err(|e| error!("{e}")) + tokio::spawn(async move { fs.remove_file(&path).await }) }); - let res = join_all(tasks) - .await - .into_iter() - .collect::>>()?; - - let mut errors = vec![]; - for task_result in res { - if let Err(e) = task_result { - errors.push(format!("- {}", e)); - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(error!("remove files failed:\n{}", errors.join("\n"))) - } + BatchFSError::try_from_joined_result( + "remove files failed", + join_all(tasks) + .await + .into_iter() + .collect::, JoinError>>(), + ) + .map(|_| ()) } pub async fn write_lock( @@ -90,11 +75,11 @@ pub async fn write_lock( files: &HashSet, root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result<()> { + fs: Arc, +) -> HandleFileResult<()> { let lock_file = root.join(lock_file); let mut lock_writer = fs.write_file(&lock_file).await?; - let mut contents = vec![temp_root.to_string()]; + let mut contents = vec![root.to_string(), temp_root.to_string()]; contents.extend(files.iter().map(|path| path.to_string())); lock_writer @@ -104,7 +89,11 @@ pub async fn write_lock( Ok(()) } -pub async fn remove_lock(lock_file: &str, root: &Utf8Path, fs: Arc) -> Result<()> { +pub async fn remove_lock( + lock_file: &str, + root: &Utf8Path, + fs: Arc, +) -> HandleFileResult<()> { let lock_file = root.join(lock_file); fs.remove_file(&lock_file).await?; Ok(()) @@ -114,18 +103,8 @@ pub async fn move_files( files: HashSet, root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result<()> { - let lock_file = root.join("move.lock"); - let mut lock_writer = fs.write_file(&lock_file).await?; - let mut contents = vec![temp_root.to_string()]; - contents.extend(files.iter().map(|path| path.to_string())); - - lock_writer - .write_all(contents.join("\n").as_bytes()) - .await?; - lock_writer.flush().await?; - + fs: Arc, +) -> BatchHandleFileResult<()> { let mut candidates = vec![]; for to in files { let from = redirect_to_path(&to, root, temp_root)?; @@ -134,63 +113,72 @@ pub async fn move_files( let tasks = candidates.into_iter().map(|(from, to)| { let fs = fs.clone(); - tokio::spawn(async move { fs.move_file(&from, &to).await }).map_err(|e| error!("{e}")) + tokio::spawn(async move { fs.move_file(&from, &to).await }) }); - let res = join_all(tasks) - .await - .into_iter() - .collect::>>>()?; - - let mut errors = vec![]; - for task_result in res { - if let Err(e) = task_result { - errors.push(format!("- {}", e)); - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(error!("move temp files failed:\n{}", errors.join("\n"))) - } + BatchFSError::try_from_joined_result( + "move temp files failed", + join_all(tasks) + .await + .into_iter() + .collect::, JoinError>>(), + ) + .map(|_| ()) } async fn recovery_lock( lock: &str, root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result> { + fs: Arc, +) -> HandleFileResult> { let lock_file = root.join(lock); if !fs.exists(&lock_file).await? { return Ok(vec![]); } let mut lock_reader = fs.read_file(&lock_file).await?; - let lock_file_content = String::from_utf8(lock_reader.read_to_end().await?) - .map_err(|e| error!("parse utf8 failed: {}", e))?; + let lock_file_content = String::from_utf8(lock_reader.read_to_end().await?).map_err(|e| { + FSError::from_message( + &lock_file, + FSOperation::Read, + format!("parse utf8 failed: {}", e), + ) + })?; let files = lock_file_content .split("\n") .map(|i| i.to_owned()) .collect::>(); fs.remove_file(&lock_file).await?; - if files.is_empty() { - return Err(error!("incomplete storage due to empty `move.lock`")); + if files.len() < 2 { + return Err(FSError::from_message( + &lock_file, + FSOperation::Read, + "incomplete storage due to illegal `move.lock`".to_string(), + )); + } + if files.first().is_some_and(|p: &String| !p.eq(root)) { + return Err(FSError::from_message( + &lock_file, + FSOperation::Read, + "incomplete storage due to `move.lock` to an unexpected directory".to_string(), + )); } - if files.first().is_some_and(|root| !root.eq(temp_root)) { - return Err(error!( - "incomplete storage due to `move.lock` from an unexpected directory" + if files.get(1).is_some_and(|p| !p.eq(temp_root)) { + return Err(FSError::from_message( + &lock_file, + FSOperation::Read, + "incomplete storage due to `move.lock` from an unexpected directory".to_string(), )); } - Ok(files[1..].to_vec()) + Ok(files[2..].to_vec()) } pub async fn recovery_move_lock( root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result<()> { + fs: Arc, +) -> BatchHandleFileResult<()> { let moving_files = recovery_lock("move.lock", root, temp_root, fs.clone()).await?; if moving_files.is_empty() { return Ok(()); @@ -211,8 +199,8 @@ pub async fn recovery_move_lock( pub async fn recovery_remove_lock( root: &Utf8Path, temp_root: &Utf8Path, - fs: Arc, -) -> Result<()> { + fs: Arc, +) -> BatchHandleFileResult<()> { let removing_files = recovery_lock("remove.lock", root, temp_root, fs.clone()).await?; if removing_files.is_empty() { return Ok(()); @@ -228,7 +216,10 @@ pub async fn recovery_remove_lock( Ok(()) } -pub async fn walk_dir(root: &Utf8Path, fs: Arc) -> Result> { +pub async fn walk_dir( + root: &Utf8Path, + fs: Arc, +) -> BatchHandleFileResult> { let mut files = HashSet::default(); let mut stack = vec![root.to_owned()]; while let Some(path) = stack.pop() { @@ -255,14 +246,21 @@ pub async fn walk_dir(root: &Utf8Path, fs: Arc) -> Result Result { +pub fn redirect_to_path( + path: &Utf8Path, + src: &Utf8Path, + dist: &Utf8Path, +) -> HandleFileResult { let relative_path = path .strip_prefix(src) - .map_err(|e| error!("get relative path failed: {}", e))?; + .map_err(|e| FSError::from_message(path, FSOperation::Redirect, format!("{e}")))?; Ok(dist.join(relative_path)) } -async fn clean_scope(scope: &PackScope, fs: Arc) -> Result<()> { +async fn try_remove_scope_files( + scope: &PackScope, + fs: Arc, +) -> BatchHandleFileResult<()> { let scope_root = &scope.path; let scope_meta_file = ScopeMeta::get_path(scope_root); let mut scope_files = scope @@ -288,26 +286,23 @@ async fn clean_scope(scope: &PackScope, fs: Arc) -> Result<()> { Ok(()) } -pub async fn clean_scopes(scopes: &HashMap, fs: Arc) -> Result<()> { - let clean_scope_tasks = scopes.values().map(|scope| clean_scope(scope, fs.clone())); - - let res = join_all(clean_scope_tasks).await; - - let mut errors = vec![]; - for task_result in res { - if let Err(e) = task_result { - errors.push(format!("- {}", e)); - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(error!("clean scopes failed:\n{}", errors.join("\n"))) - } +pub async fn remove_unused_scope_files( + scopes: &HashMap, + fs: Arc, +) -> BatchHandleFileResult<()> { + let clean_scope_tasks = scopes + .values() + .map(|scope| try_remove_scope_files(scope, fs.clone())); + + BatchFSError::try_from_results("clean scopes failed", join_all(clean_scope_tasks).await) + .map(|_| ()) } -async fn remove_unused_scope(name: &str, dir: &Utf8Path, fs: Arc) -> Result<()> { +async fn try_remove_scope( + name: &str, + dir: &Utf8Path, + fs: Arc, +) -> HandleFileResult<()> { // do not remove hidden dirs if name.starts_with(".") { return Ok(()); @@ -323,39 +318,34 @@ async fn remove_unused_scope(name: &str, dir: &Utf8Path, fs: Arc) -> Ok(()) } -pub async fn clean_root(root: &Utf8Path, root_meta: &RootMeta, fs: Arc) -> Result<()> { +pub async fn remove_unused_scopes( + root: &Utf8Path, + root_meta: &RootMeta, + fs: Arc, +) -> BatchHandleFileResult<()> { let dirs = fs.read_dir(root).await?; let tasks = dirs.difference(&root_meta.scopes).map(|name| { let fs = fs.clone(); let scope_dir = root.join(name); let scope_name = name.clone(); - tokio::spawn(async move { remove_unused_scope(&scope_name, &scope_dir, fs).await }) - .map_err(|e| error!("{e}")) + tokio::spawn(async move { try_remove_scope(&scope_name, &scope_dir, fs).await }) }); - let res = join_all(tasks) - .await - .into_iter() - .collect::>>()?; - - let mut errors = vec![]; - for task_result in res { - if let Err(e) = task_result { - errors.push(format!("- {}", e)); - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(error!( - "remove unused scopes failed:\n{}", - errors.join("\n") - )) - } + BatchFSError::try_from_joined_result( + "remove unused scopes failed", + join_all(tasks) + .await + .into_iter() + .collect::, JoinError>>(), + ) + .map(|_| ()) } -async fn remove_expired_version(version: &str, dir: &Utf8Path, fs: Arc) -> Result<()> { +async fn try_remove_version( + version: &str, + dir: &Utf8Path, + fs: Arc, +) -> BatchHandleFileResult<()> { // do not remove hidden dirs and lock files if version.starts_with(".") || version.contains(".lock") { return Ok(()); @@ -369,30 +359,34 @@ async fn remove_expired_version(version: &str, dir: &Utf8Path, fs: Arc() - .map_err(|e| error!("parse option meta failed: {}", e))?; + let expire_time = reader.read_line().await?.parse::().map_err(|e| { + FSError::from_message( + &meta, + FSOperation::Read, + format!("parse option meta failed: {}", e), + ) + })?; let current = current_time(); if current > expire_time { - fs.remove_dir(dir).await + fs.remove_dir(dir).await?; + Ok(()) } else { Ok(()) } } -pub async fn clean_versions( +pub async fn remove_expired_versions( root: &Utf8Path, root_options: &RootOptions, - fs: Arc, -) -> Result<()> { + fs: Arc, +) -> BatchHandleFileResult<()> { let dirs = fs.read_dir(&root_options.root).await?; let tasks = dirs.into_iter().filter_map(|version| { let version_dir = root_options.root.join(&version); @@ -400,31 +394,18 @@ pub async fn clean_versions( None } else { let fs = fs.clone(); - Some( - tokio::spawn(async move { remove_expired_version(&version, &version_dir, fs).await }) - .map_err(|e| error!("{e}")), - ) + Some(tokio::spawn(async move { + try_remove_version(&version, &version_dir, fs).await + })) } }); - let res = join_all(tasks) - .await - .into_iter() - .collect::>>()?; - - let mut errors = vec![]; - for task_result in res { - if let Err(e) = task_result { - errors.push(format!("- {}", e)); - } - } - - if errors.is_empty() { - Ok(()) - } else { - Err(error!( - "remove expired versions failed:\n{}", - errors.join("\n") - )) - } + BatchFSError::try_from_joined_result( + "remove expired versions failed", + join_all(tasks) + .await + .into_iter() + .collect::, JoinError>>(), + ) + .map(|_| ()) } diff --git a/crates/rspack_storage/src/pack/strategy/split/mod.rs b/crates/rspack_storage/src/pack/strategy/split/mod.rs index a4a7dd289b3..4fbfb7b63d1 100644 --- a/crates/rspack_storage/src/pack/strategy/split/mod.rs +++ b/crates/rspack_storage/src/pack/strategy/split/mod.rs @@ -9,29 +9,33 @@ mod write_scope; use std::{hash::Hasher, sync::Arc}; use handle_file::{ - clean_root, clean_scopes, clean_versions, recovery_move_lock, recovery_remove_lock, + recovery_move_lock, recovery_remove_lock, remove_expired_versions, remove_unused_scope_files, + remove_unused_scopes, }; use itertools::Itertools; -use rspack_error::{error, Result}; use rspack_paths::{Utf8Path, Utf8PathBuf}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet, FxHasher}; use util::get_name; -use super::{RootStrategy, ScopeStrategy, ValidateResult}; -use crate::pack::{ - data::{current_time, PackContents, PackKeys, PackScope, RootMeta, RootMetaFrom, RootOptions}, - fs::PackFS, +use super::{RootStrategy, ScopeStrategy}; +use crate::{ + error::{Result, ValidateResult}, + fs::{FSError, FSOperation}, + pack::data::{ + current_time, PackContents, PackKeys, PackScope, RootMeta, RootMetaFrom, RootOptions, + }, + FileSystem, }; #[derive(Debug, Clone)] pub struct SplitPackStrategy { - pub fs: Arc, + pub fs: Arc, pub root: Arc, pub temp_root: Arc, } impl SplitPackStrategy { - pub fn new(root: Utf8PathBuf, temp_root: Utf8PathBuf, fs: Arc) -> Self { + pub fn new(root: Utf8PathBuf, temp_root: Utf8PathBuf, fs: Arc) -> Self { Self { fs, root: Arc::new(root), @@ -71,11 +75,13 @@ impl RootStrategy for SplitPackStrategy { } let mut reader = self.fs.read_file(&meta_path).await?; - let expire_time = reader - .read_line() - .await? - .parse::() - .map_err(|e| error!("parse option meta failed: {}", e))?; + let expire_time = reader.read_line().await?.parse::().map_err(|e| { + FSError::from_message( + &meta_path, + FSOperation::Read, + format!("parse root meta failed: {}", e), + ) + })?; let scopes = reader .read_line() .await? @@ -91,7 +97,6 @@ impl RootStrategy for SplitPackStrategy { } async fn write_root_meta(&self, root_meta: &RootMeta) -> Result<()> { let meta_path = RootMeta::get_path(&self.root); - let mut writer = self.fs.write_file(&meta_path).await?; writer @@ -103,6 +108,7 @@ impl RootStrategy for SplitPackStrategy { .await?; writer.flush().await?; + Ok(()) } async fn validate_root(&self, root_meta: &RootMeta) -> Result { @@ -111,7 +117,7 @@ impl RootStrategy for SplitPackStrategy { } else { let now = current_time(); if now > root_meta.expire_time { - Ok(ValidateResult::invalid("cache expired")) + Ok(ValidateResult::invalid("expiration")) } else { Ok(ValidateResult::Valid) } @@ -129,9 +135,9 @@ impl RootStrategy for SplitPackStrategy { } let _ = tokio::try_join!( - clean_scopes(scopes, self.fs.clone()), - clean_root(&self.root, root_meta, self.fs.clone()), - clean_versions(&self.root, root_options, self.fs.clone()) + remove_unused_scope_files(scopes, self.fs.clone()), + remove_unused_scopes(&self.root, root_meta, self.fs.clone()), + remove_expired_versions(&self.root, root_options, self.fs.clone()) ); Ok(()) diff --git a/crates/rspack_storage/src/pack/strategy/split/read_pack.rs b/crates/rspack_storage/src/pack/strategy/split/read_pack.rs index 7d7a5a91927..5072071c29b 100644 --- a/crates/rspack_storage/src/pack/strategy/split/read_pack.rs +++ b/crates/rspack_storage/src/pack/strategy/split/read_pack.rs @@ -1,13 +1,16 @@ use std::sync::Arc; use async_trait::async_trait; -use rspack_error::Result; use rspack_paths::Utf8Path; use super::SplitPackStrategy; -use crate::pack::{ - data::{PackContents, PackKeys}, - strategy::PackReadStrategy, +use crate::{ + error::Result, + pack::{ + data::{PackContents, PackKeys}, + strategy::PackReadStrategy, + }, + FSError, FSOperation, }; #[async_trait] @@ -18,12 +21,20 @@ impl PackReadStrategy for SplitPackStrategy { } let mut reader = self.fs.read_file(path).await?; - let key_lengths: Vec = reader + let key_lengths = reader .read_line() .await? .split(" ") - .map(|item| item.parse::().expect("should have meta info")) - .collect(); + .map(|item| { + item.parse::().map_err(|e| { + FSError::from_message( + path, + FSOperation::Read, + format!("parse pack key lengths failed: {}", e), + ) + }) + }) + .collect::, FSError>>()?; reader.read_line().await?; @@ -44,15 +55,33 @@ impl PackReadStrategy for SplitPackStrategy { .read_line() .await? .split(" ") - .map(|item| item.parse::().expect("should have meta info")) + .map(|item| { + item.parse::().map_err(|e| { + FSError::from_message( + path, + FSOperation::Read, + format!("parse pack key lengths failed: {}", e), + ) + }) + }) + .collect::, FSError>>()? + .iter() .sum::(); let content_lengths: Vec = reader .read_line() .await? .split(" ") - .map(|item| item.parse::().expect("should have meta info")) - .collect(); + .map(|item| { + item.parse::().map_err(|e| { + FSError::from_message( + path, + FSOperation::Read, + format!("parse pack content lengths failed: {}", e), + ) + }) + }) + .collect::, FSError>>()?; reader.skip(total_key_length).await?; @@ -68,13 +97,15 @@ impl PackReadStrategy for SplitPackStrategy { #[cfg(test)] mod tests { - use rspack_error::Result; use rspack_paths::Utf8PathBuf; use rustc_hash::FxHashSet as HashSet; - use crate::pack::strategy::{ - split::util::test_pack_utils::{clean_strategy, create_strategies, mock_pack_file}, - PackReadStrategy, ScopeReadStrategy, SplitPackStrategy, + use crate::{ + error::Result, + pack::strategy::{ + split::util::test_pack_utils::{clean_strategy, create_strategies, mock_pack_file}, + PackReadStrategy, ScopeReadStrategy, SplitPackStrategy, + }, }; async fn test_read_keys_non_exists(strategy: &SplitPackStrategy) -> Result<()> { diff --git a/crates/rspack_storage/src/pack/strategy/split/read_scope.rs b/crates/rspack_storage/src/pack/strategy/split/read_scope.rs index 580e220d7a5..8a64c003e9c 100644 --- a/crates/rspack_storage/src/pack/strategy/split/read_scope.rs +++ b/crates/rspack_storage/src/pack/strategy/split/read_scope.rs @@ -3,14 +3,16 @@ use std::sync::Arc; use async_trait::async_trait; use futures::{future::join_all, TryFutureExt}; use itertools::Itertools; -use rspack_error::{error, Result}; use rspack_paths::{Utf8Path, Utf8PathBuf}; use super::{util::get_indexed_packs, SplitPackStrategy}; -use crate::pack::{ - data::{Pack, PackContents, PackFileMeta, PackKeys, PackScope, ScopeMeta}, - fs::PackFS, - strategy::{PackReadStrategy, ScopeReadStrategy}, +use crate::{ + error::{Error, ErrorType, Result}, + pack::{ + data::{Pack, PackContents, PackFileMeta, PackKeys, PackScope, ScopeMeta}, + strategy::{PackReadStrategy, ScopeReadStrategy}, + }, + FileSystem, }; #[async_trait] @@ -18,7 +20,7 @@ impl ScopeReadStrategy for SplitPackStrategy { async fn ensure_meta(&self, scope: &mut PackScope) -> Result<()> { if !scope.meta.loaded() { let meta_path = ScopeMeta::get_path(&scope.path); - let meta = read_scope_meta(&meta_path, self.fs.clone()) + let meta = read_scope_meta(scope.name, &meta_path, self.fs.clone()) .await? .unwrap_or_else(|| ScopeMeta::new(&scope.path, &scope.options)); scope.meta.set_value(meta); @@ -87,7 +89,11 @@ impl ScopeReadStrategy for SplitPackStrategy { } } -async fn read_scope_meta(path: &Utf8Path, fs: Arc) -> Result> { +async fn read_scope_meta( + scope: &'static str, + path: &Utf8Path, + fs: Arc, +) -> Result> { if !fs.exists(path).await? { return Ok(None); } @@ -99,14 +105,22 @@ async fn read_scope_meta(path: &Utf8Path, fs: Arc) -> Result() - .map_err(|e| error!("parse option meta failed: {}", e)) + item.parse::().map_err(|e| { + Error::from_reason( + Some(ErrorType::Load), + Some(scope), + format!("parse option meta failed: {}", e), + ) + }) }) .collect::>>()?; if option_items.len() < 2 { - return Err(error!("option meta not match")); + return Err(Error::from_reason( + Some(ErrorType::Load), + Some(scope), + "option meta not match".to_string(), + )); } let bucket_size = option_items[0]; @@ -123,14 +137,22 @@ async fn read_scope_meta(path: &Utf8Path, fs: Arc) -> Result>()) .map(|i| { if i.len() < 3 { - Err(error!("file meta not match")) + Err(Error::from_reason( + Some(ErrorType::Load), + Some(scope), + "file meta not match".to_string(), + )) } else { Ok(PackFileMeta { name: i[0].to_owned(), hash: i[1].to_owned(), - size: i[2] - .parse::() - .map_err(|e| error!("parse file meta failed: {}", e))?, + size: i[2].parse::().map_err(|e| { + Error::from_reason( + Some(ErrorType::Load), + Some(scope), + format!("parse file meta failed: {}", e), + ) + })?, wrote: true, }) } @@ -140,7 +162,11 @@ async fn read_scope_meta(path: &Utf8Path, fs: Arc) -> Result bool { } async fn read_keys(scope: &PackScope, strategy: &SplitPackStrategy) -> Result> { - let (pack_indexes, packs) = get_indexed_packs(scope, Some(&read_keys_filter))?; + let (pack_indexes, packs) = get_indexed_packs(scope, Some(&read_keys_filter)); let tasks = packs .into_iter() .map(|i| { let strategy = strategy.clone(); let path = i.1.path.clone(); - tokio::spawn(async move { strategy.read_pack_keys(&path).await }).map_err(|e| error!("{}", e)) + tokio::spawn(async move { strategy.read_pack_keys(&path).await }) + .map_err(|e| Error::from_error(Some(ErrorType::Load), Some(scope.name), Box::new(e))) }) .collect_vec(); @@ -208,14 +235,14 @@ async fn read_contents( scope: &PackScope, strategy: &SplitPackStrategy, ) -> Result> { - let (pack_indexes, packs) = get_indexed_packs(scope, Some(&read_contents_filter))?; + let (pack_indexes, packs) = get_indexed_packs(scope, Some(&read_contents_filter)); let tasks = packs .into_iter() .map(|i| { let strategy = strategy.to_owned(); let path = i.1.path.to_owned(); tokio::spawn(async move { strategy.read_pack_contents(&path).await }) - .map_err(|e| error!("{}", e)) + .map_err(|e| Error::from_error(Some(ErrorType::Load), Some(scope.name), Box::new(e))) }) .collect_vec(); let pack_contents = join_all(tasks).await.into_iter().process_results(|iter| { @@ -243,21 +270,23 @@ mod tests { use std::{collections::HashSet, sync::Arc}; use itertools::Itertools; - use rspack_error::Result; use rspack_paths::Utf8Path; - use crate::pack::{ - data::{PackOptions, PackScope, ScopeMeta}, - fs::PackFS, - strategy::{ - split::util::test_pack_utils::{ - clean_strategy, create_strategies, mock_pack_file, mock_scope_meta_file, + use crate::{ + error::Result, + pack::{ + data::{PackOptions, PackScope, ScopeMeta}, + strategy::{ + split::util::test_pack_utils::{ + clean_strategy, create_strategies, mock_pack_file, mock_scope_meta_file, + }, + ScopeReadStrategy, SplitPackStrategy, }, - ScopeReadStrategy, SplitPackStrategy, }, + FileSystem, }; - async fn mock_scope(path: &Utf8Path, fs: &dyn PackFS, options: &PackOptions) -> Result<()> { + async fn mock_scope(path: &Utf8Path, fs: &dyn FileSystem, options: &PackOptions) -> Result<()> { mock_scope_meta_file(&ScopeMeta::get_path(path), fs, options, 3).await?; for bucket_id in 0..options.bucket_size { for pack_no in 0..3 { @@ -335,7 +364,11 @@ mod tests { bucket_size: 1, pack_size: 16, }); - let mut scope = PackScope::new(strategy.get_path("scope_name"), options.clone()); + let mut scope = PackScope::new( + "scope_name", + strategy.get_path("scope_name"), + options.clone(), + ); mock_scope(&scope.path, strategy.fs.as_ref(), &scope.options) .await diff --git a/crates/rspack_storage/src/pack/strategy/split/util.rs b/crates/rspack_storage/src/pack/strategy/split/util.rs index a74bbf7069c..092e83d95a7 100644 --- a/crates/rspack_storage/src/pack/strategy/split/util.rs +++ b/crates/rspack_storage/src/pack/strategy/split/util.rs @@ -1,7 +1,6 @@ use std::hash::Hasher; use itertools::Itertools; -use rspack_error::Result; use rustc_hash::FxHasher; use crate::pack::data::{Pack, PackContents, PackFileMeta, PackKeys, PackScope}; @@ -21,40 +20,38 @@ pub fn flag_scope_wrote(scope: &mut PackScope) { pub fn get_indexed_packs<'a>( scope: &'a PackScope, filter: Option<&dyn Fn(&'a Pack, &'a PackFileMeta) -> bool>, -) -> Result<(PackIndexList, PackInfoList<'a>)> { +) -> (PackIndexList, PackInfoList<'a>) { let meta = scope.meta.expect_value(); let packs = scope.packs.expect_value(); - Ok( - meta - .packs - .iter() - .enumerate() - .flat_map(|(bucket_id, pack_meta_list)| { - let bucket_packs = packs.get(bucket_id).expect("should have bucket packs"); - pack_meta_list - .iter() - .enumerate() - .map(|(pack_pos, pack_meta)| { + meta + .packs + .iter() + .enumerate() + .flat_map(|(bucket_id, pack_meta_list)| { + let bucket_packs = packs.get(bucket_id).expect("should have bucket packs"); + pack_meta_list + .iter() + .enumerate() + .map(|(pack_pos, pack_meta)| { + ( + (bucket_id, pack_pos), ( - (bucket_id, pack_pos), - ( - pack_meta, - bucket_packs.get(pack_pos).expect("should have bucket pack"), - ), - ) - }) - .collect_vec() - }) - .filter(|(_, (pack_meta, pack))| { - if let Some(filter) = filter { - filter(pack, pack_meta) - } else { - true - } - }) - .unzip(), - ) + pack_meta, + bucket_packs.get(pack_pos).expect("should have bucket pack"), + ), + ) + }) + .collect_vec() + }) + .filter(|(_, (pack_meta, pack))| { + if let Some(filter) = filter { + filter(pack, pack_meta) + } else { + true + } + }) + .unzip() } pub fn get_name(keys: &PackKeys, _: &PackContents) -> String { @@ -77,25 +74,24 @@ pub mod test_pack_utils { use std::sync::Arc; use itertools::Itertools; - use rspack_error::Result; use rspack_fs::{MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf}; use rustc_hash::FxHashMap as HashMap; use super::flag_scope_wrote; use crate::{ + error::Result, pack::{ data::{current_time, PackOptions, PackScope}, - fs::PackFS, strategy::{ split::handle_file::prepare_scope, ScopeUpdate, ScopeWriteStrategy, SplitPackStrategy, WriteScopeResult, }, }, - PackBridgeFS, + BridgeFileSystem, FileSystem, }; - pub async fn mock_root_meta_file(path: &Utf8Path, fs: &dyn PackFS) -> Result<()> { + pub async fn mock_root_meta_file(path: &Utf8Path, fs: &dyn FileSystem) -> Result<()> { fs.ensure_dir(path.parent().expect("should have parent")) .await?; let mut writer = fs.write_file(path).await?; @@ -108,7 +104,7 @@ pub mod test_pack_utils { pub async fn mock_scope_meta_file( path: &Utf8Path, - fs: &dyn PackFS, + fs: &dyn FileSystem, options: &PackOptions, pack_count: usize, ) -> Result<()> { @@ -138,7 +134,7 @@ pub mod test_pack_utils { path: &Utf8Path, unique_id: &str, item_count: usize, - fs: &dyn PackFS, + fs: &dyn FileSystem, ) -> Result<()> { fs.ensure_dir(path.parent().expect("should have parent")) .await?; @@ -228,7 +224,7 @@ pub mod test_pack_utils { .expect("should remove dir"); } - pub async fn flush_file_mtime(path: &Utf8Path, fs: Arc) -> Result<()> { + pub async fn flush_file_mtime(path: &Utf8Path, fs: Arc) -> Result<()> { let content = fs.read_file(path).await?.read_to_end().await?; fs.write_file(path).await?.write_all(&content).await?; @@ -270,11 +266,11 @@ pub mod test_pack_utils { pub fn create_strategies(case: &str) -> Vec { let fs = [ ( - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), get_memory_path(case), ), ( - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), get_native_path(case), ), ]; diff --git a/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs b/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs index 14b12ab9b7a..4e5f1b0eb42 100644 --- a/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs +++ b/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs @@ -1,12 +1,11 @@ use async_trait::async_trait; use futures::{future::join_all, TryFutureExt}; use itertools::Itertools; -use rspack_error::{error, Result}; use super::{util::get_indexed_packs, SplitPackStrategy}; -use crate::pack::{ - data::PackScope, - strategy::{ScopeValidateStrategy, ValidateResult}, +use crate::{ + error::{Error, ErrorType, Result, ValidateResult}, + pack::{data::PackScope, strategy::ScopeValidateStrategy}, }; #[async_trait] @@ -26,7 +25,7 @@ impl ScopeValidateStrategy for SplitPackStrategy { } async fn validate_packs(&self, scope: &mut PackScope) -> Result { - let (_, pack_list) = get_indexed_packs(scope, None)?; + let (_, pack_list) = get_indexed_packs(scope, None); let tasks = pack_list .iter() @@ -45,7 +44,7 @@ impl ScopeValidateStrategy for SplitPackStrategy { Err(_) => false, } }) - .map_err(|e| error!("{}", e)) + .map_err(|e| Error::from_error(Some(ErrorType::Validate), Some(scope.name), e.into())) }); let validate_results = join_all(tasks) @@ -79,27 +78,28 @@ impl ScopeValidateStrategy for SplitPackStrategy { mod tests { use std::sync::Arc; - use rspack_error::Result; use rspack_paths::Utf8PathBuf; use rustc_hash::FxHashSet as HashSet; - use crate::pack::{ - data::{PackOptions, PackScope, RootMeta, ScopeMeta}, - fs::PackFS, - strategy::{ - split::{ - handle_file::prepare_scope, - util::{ - flag_scope_wrote, - test_pack_utils::{ - clean_strategy, create_strategies, flush_file_mtime, mock_root_meta_file, - mock_scope_meta_file, mock_updates, save_scope, UpdateVal, + use crate::{ + error::{Error, ErrorType, Result, ValidateResult}, + pack::{ + data::{PackOptions, PackScope, RootMeta, ScopeMeta}, + strategy::{ + split::{ + handle_file::prepare_scope, + util::{ + flag_scope_wrote, + test_pack_utils::{ + clean_strategy, create_strategies, flush_file_mtime, mock_root_meta_file, + mock_scope_meta_file, mock_updates, save_scope, UpdateVal, + }, }, }, + ScopeReadStrategy, ScopeValidateStrategy, ScopeWriteStrategy, SplitPackStrategy, }, - ScopeReadStrategy, ScopeValidateStrategy, ScopeWriteStrategy, SplitPackStrategy, - ValidateResult, }, + FileSystem, }; async fn test_valid_meta(scope_path: Utf8PathBuf, strategy: &SplitPackStrategy) -> Result<()> { @@ -107,7 +107,7 @@ mod tests { bucket_size: 10, pack_size: 100, }); - let mut scope = PackScope::new(scope_path, same_options); + let mut scope = PackScope::new("scope_name", scope_path, same_options); strategy.ensure_meta(&mut scope).await?; let validated = strategy.validate_meta(&mut scope).await?; assert!(validated.is_valid()); @@ -123,25 +123,41 @@ mod tests { bucket_size: 1, pack_size: 100, }); - let mut scope = PackScope::new(scope_path.clone(), bucket_changed_options.clone()); - strategy.ensure_meta(&mut scope).await?; - let validated: ValidateResult = strategy.validate_meta(&mut scope).await?; - assert_eq!( - validated.to_string(), - "validation failed due to `options.bucketSize` changed" + let mut scope = PackScope::new( + "scope_name", + scope_path.clone(), + bucket_changed_options.clone(), ); + strategy.ensure_meta(&mut scope).await?; + if let ValidateResult::Invalid(detail) = strategy.validate_meta(&mut scope).await? { + let error = Error::from_detail(Some(ErrorType::Validate), Some("test_scope"), detail); + assert_eq!( + error.to_string(), + "validate scope `test_scope` failed due to `options.bucketSize` changed" + ); + } else { + panic!("should be invalid"); + } let max_size_changed_options = Arc::new(PackOptions { bucket_size: 10, pack_size: 99, }); - let mut scope = PackScope::new(scope_path.clone(), max_size_changed_options.clone()); - strategy.ensure_meta(&mut scope).await?; - let validated: ValidateResult = strategy.validate_meta(&mut scope).await?; - assert_eq!( - validated.to_string(), - "validation failed due to `options.packSize` changed" + let mut scope = PackScope::new( + "scope_name", + scope_path.clone(), + max_size_changed_options.clone(), ); + strategy.ensure_meta(&mut scope).await?; + if let ValidateResult::Invalid(detail) = strategy.validate_meta(&mut scope).await? { + let error = Error::from_detail(Some(ErrorType::Validate), Some("test_scope"), detail); + assert_eq!( + error.to_string(), + "validate scope `test_scope` failed due to `options.packSize` changed" + ); + } else { + panic!("should be invalid"); + } Ok(()) } @@ -151,7 +167,7 @@ mod tests { strategy: &SplitPackStrategy, options: Arc, ) -> Result<()> { - let mut scope = PackScope::new(scope_path, options); + let mut scope = PackScope::new("scope_name", scope_path, options); strategy.ensure_keys(&mut scope).await?; let validated = strategy.validate_packs(&mut scope).await?; assert!(validated.is_valid()); @@ -162,11 +178,11 @@ mod tests { async fn test_invalid_packs_changed( scope_path: Utf8PathBuf, strategy: &SplitPackStrategy, - fs: Arc, + fs: Arc, options: Arc, files: HashSet, ) -> Result<()> { - let mut scope = PackScope::new(scope_path, options); + let mut scope = PackScope::new("scope_name", scope_path, options); for file in files { if !file.to_string().contains("scope_meta") { flush_file_mtime(&file, fs.clone()).await?; @@ -174,11 +190,14 @@ mod tests { } strategy.ensure_keys(&mut scope).await?; - let validated = strategy.validate_packs(&mut scope).await?; - assert!(validated - .to_string() - .starts_with("validation failed due to some packs are modified:")); - assert_eq!(validated.to_string().split("\n").count(), 7); + if let ValidateResult::Invalid(detail) = strategy.validate_packs(&mut scope).await? { + let error = + Error::from_detail(Some(ErrorType::Validate), Some("scope_name"), detail).to_string(); + assert!(error.contains("validate scope `scope_name` failed due to some packs are modified")); + assert_eq!(error.split("\n").count(), 7); + } else { + panic!("should be invalid"); + } Ok(()) } @@ -236,7 +255,7 @@ mod tests { bucket_size: 10, pack_size: 100, }); - let mut mock_scope = PackScope::empty(scope_path.clone(), pack_options.clone()); + let mut mock_scope = PackScope::empty("scope_name", scope_path.clone(), pack_options.clone()); let updates = mock_updates(0, 100, 30, UpdateVal::Value("val".to_string())); strategy .update_scope(&mut mock_scope, updates) diff --git a/crates/rspack_storage/src/pack/strategy/split/write_pack.rs b/crates/rspack_storage/src/pack/strategy/split/write_pack.rs index e7c6aba024c..6ac2befe860 100644 --- a/crates/rspack_storage/src/pack/strategy/split/write_pack.rs +++ b/crates/rspack_storage/src/pack/strategy/split/write_pack.rs @@ -2,18 +2,18 @@ use std::sync::Arc; use async_trait::async_trait; use itertools::Itertools; -use rspack_error::{error, Result}; use rspack_paths::{Utf8Path, Utf8PathBuf}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use super::{handle_file::redirect_to_path, SplitPackStrategy}; use crate::{ + error::Result, pack::{ data::{Pack, PackContents, PackFileMeta, PackKeys, PackOptions}, strategy::{split::util::get_name, PackWriteStrategy, UpdatePacksResult}, ScopeUpdate, }, - StorageItemKey, StorageItemValue, + ItemKey, ItemValue, }; #[async_trait] @@ -137,7 +137,7 @@ impl PackWriteStrategy for SplitPackStrategy { let keys = pack.keys.expect_value(); let contents = pack.contents.expect_value(); if keys.len() != contents.len() { - return Err(error!("pack keys and contents length not match")); + panic!("pack keys and contents length not match"); } let mut writer = self.fs.write_file(&path).await?; @@ -183,7 +183,7 @@ impl PackWriteStrategy for SplitPackStrategy { fn create( dir: &Utf8Path, options: &PackOptions, - items: HashMap, Arc>, + items: HashMap, Arc>, ) -> Vec<(PackFileMeta, Pack)> { let mut items = items.into_iter().collect_vec(); items.sort_unstable_by(|a, b| a.1.len().cmp(&b.1.len())); @@ -261,17 +261,19 @@ mod tests { use std::sync::Arc; use itertools::Itertools; - use rspack_error::Result; use rustc_hash::FxHashMap as HashMap; - use crate::pack::{ - data::{Pack, PackFileMeta, PackOptions}, - strategy::{ - split::{ - handle_file::redirect_to_path, - util::test_pack_utils::{clean_strategy, create_strategies, mock_updates, UpdateVal}, + use crate::{ + error::Result, + pack::{ + data::{Pack, PackFileMeta, PackOptions}, + strategy::{ + split::{ + handle_file::redirect_to_path, + util::test_pack_utils::{clean_strategy, create_strategies, mock_updates, UpdateVal}, + }, + PackWriteStrategy, SplitPackStrategy, UpdatePacksResult, }, - PackWriteStrategy, SplitPackStrategy, UpdatePacksResult, }, }; diff --git a/crates/rspack_storage/src/pack/strategy/split/write_scope.rs b/crates/rspack_storage/src/pack/strategy/split/write_scope.rs index 33e4f68c25e..cbf47805d0e 100644 --- a/crates/rspack_storage/src/pack/strategy/split/write_scope.rs +++ b/crates/rspack_storage/src/pack/strategy/split/write_scope.rs @@ -1,10 +1,9 @@ use async_trait::async_trait; use futures::future::join_all; -use futures::TryFutureExt; use itertools::Itertools; use rayon::iter::{IntoParallelIterator, ParallelBridge, ParallelIterator}; -use rspack_error::{error, Result}; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; +use tokio::task::JoinError; use super::{ handle_file::{ @@ -13,9 +12,13 @@ use super::{ util::{choose_bucket, flag_scope_wrote}, SplitPackStrategy, }; -use crate::pack::{ - data::{Pack, PackScope}, - strategy::{PackWriteStrategy, ScopeUpdate, ScopeWriteStrategy, WriteScopeResult}, +use crate::{ + error::Result, + fs::BatchFSError, + pack::{ + data::{Pack, PackScope}, + strategy::{PackWriteStrategy, ScopeUpdate, ScopeWriteStrategy, WriteScopeResult}, + }, }; #[async_trait] @@ -70,7 +73,7 @@ impl ScopeWriteStrategy for SplitPackStrategy { fn update_scope(&self, scope: &mut PackScope, updates: ScopeUpdate) -> Result<()> { if !scope.loaded() { - return Err(error!("scope not loaded, run `load` first")); + panic!("scope not loaded, run `load` first"); } let mut scope_meta = scope.meta.take_value().expect("should have scope meta"); let mut scope_packs = scope.packs.take_value().expect("should have scope packs"); @@ -241,7 +244,7 @@ async fn save_pack(pack: &Pack, strategy: &SplitPackStrategy) -> Result let keys = pack.keys.expect_value(); let contents = pack.contents.expect_value(); if keys.len() != contents.len() { - return Err(error!("pack keys and contents length not match")); + panic!("pack keys and contents length not match"); } strategy.write_pack(pack).await?; let hash = strategy @@ -260,19 +263,17 @@ async fn batch_write_packs( ) -> Result> { let tasks = packs.into_iter().map(|pack| { let strategy = strategy.to_owned(); - tokio::spawn(async move { (save_pack(&pack, &strategy).await, pack) }) - .map_err(|e| error!("{}", e)) + tokio::spawn(async move { save_pack(&pack, &strategy).await.map(|hash| (hash, pack)) }) }); - let task_result = join_all(tasks) - .await - .into_iter() - .collect::, Pack)>>>()?; + let res = BatchFSError::try_from_joined_result( + "write packs failed", + join_all(tasks) + .await + .into_iter() + .collect::, JoinError>>(), + )?; - let mut res = vec![]; - for (hash, pack) in task_result { - res.push((hash?, pack)); - } Ok(res) } @@ -280,18 +281,19 @@ async fn batch_write_packs( mod tests { use std::{collections::HashMap, sync::Arc}; - use rspack_error::Result; - - use crate::pack::{ - data::{PackOptions, PackScope}, - strategy::{ - split::util::test_pack_utils::{ - clean_strategy, count_bucket_packs, count_scope_packs, create_strategies, - get_bucket_pack_sizes, mock_updates, save_scope, UpdateVal, + use crate::{ + error::Result, + pack::{ + data::{PackOptions, PackScope}, + strategy::{ + split::util::test_pack_utils::{ + clean_strategy, count_bucket_packs, count_scope_packs, create_strategies, + get_bucket_pack_sizes, mock_updates, save_scope, UpdateVal, + }, + ScopeReadStrategy, ScopeWriteStrategy, }, - ScopeReadStrategy, ScopeWriteStrategy, + SplitPackStrategy, }, - SplitPackStrategy, }; async fn test_short_value( @@ -489,7 +491,11 @@ mod tests { bucket_size: 1, pack_size: 32, }); - let mut scope = PackScope::empty(strategy.get_path("scope_name"), options.clone()); + let mut scope = PackScope::empty( + "scope_name", + strategy.get_path("scope_name"), + options.clone(), + ); clean_strategy(&strategy).await; let _ = test_single_bucket(&mut scope, &strategy) @@ -508,7 +514,11 @@ mod tests { bucket_size: 10, pack_size: 32, }); - let mut scope = PackScope::empty(strategy.get_path("scope_name"), options.clone()); + let mut scope = PackScope::empty( + "scope_name", + strategy.get_path("scope_name"), + options.clone(), + ); clean_strategy(&strategy).await; let _ = test_multi_bucket(&mut scope, &strategy).await.map_err(|e| { @@ -525,7 +535,11 @@ mod tests { bucket_size: 1, pack_size: 2000, }); - let mut scope = PackScope::empty(strategy.get_path("scope_name"), options.clone()); + let mut scope = PackScope::empty( + "scope_name", + strategy.get_path("scope_name"), + options.clone(), + ); clean_strategy(&strategy).await; let _ = test_big_bucket(&mut scope, &strategy).await.map_err(|e| { diff --git a/crates/rspack_storage/tests/build.rs b/crates/rspack_storage/tests/build.rs index 6b262afc3e8..f9d2871d0ad 100644 --- a/crates/rspack_storage/tests/build.rs +++ b/crates/rspack_storage/tests/build.rs @@ -2,10 +2,11 @@ mod test_storage_build { use std::{collections::HashMap, path::PathBuf, sync::Arc}; - use rspack_error::Result; use rspack_fs::{MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8PathBuf}; - use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + use rspack_storage::{ + BridgeFileSystem, FileSystem, PackStorage, PackStorageOptions, Result, Storage, + }; pub fn get_native_path(p: &str) -> (PathBuf, PathBuf) { let base = std::env::temp_dir() @@ -23,7 +24,7 @@ mod test_storage_build { root: &Utf8PathBuf, temp_root: &Utf8PathBuf, version: &str, - fs: Arc, + fs: Arc, ) -> PackStorageOptions { PackStorageOptions { version: version.to_string(), @@ -39,7 +40,7 @@ mod test_storage_build { async fn test_initial_build( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -60,7 +61,7 @@ mod test_storage_build { async fn test_recovery_modify( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -80,7 +81,7 @@ mod test_storage_build { async fn test_recovery_final( _root: &Utf8PathBuf, - _fs: Arc, + _fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -111,11 +112,11 @@ mod test_storage_build { let cases = [ ( get_native_path("test_build_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_build_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; let version = "xxx".to_string(); diff --git a/crates/rspack_storage/tests/dev.rs b/crates/rspack_storage/tests/dev.rs index 8e3fca96631..def2700a0e1 100644 --- a/crates/rspack_storage/tests/dev.rs +++ b/crates/rspack_storage/tests/dev.rs @@ -2,10 +2,11 @@ mod test_storage_dev { use std::{collections::HashMap, path::PathBuf, sync::Arc}; - use rspack_error::Result; use rspack_fs::{MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8PathBuf}; - use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + use rspack_storage::{ + BridgeFileSystem, FileSystem, PackStorage, PackStorageOptions, Result, Storage, + }; pub fn get_native_path(p: &str) -> (PathBuf, PathBuf) { let base = std::env::temp_dir() @@ -23,7 +24,7 @@ mod test_storage_dev { root: &Utf8PathBuf, temp_root: &Utf8PathBuf, version: &str, - fs: Arc, + fs: Arc, ) -> PackStorageOptions { PackStorageOptions { version: version.to_string(), @@ -39,7 +40,7 @@ mod test_storage_dev { async fn test_initial_dev( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -83,7 +84,7 @@ mod test_storage_dev { async fn test_recovery_modify( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -114,7 +115,7 @@ mod test_storage_dev { async fn test_recovery_final( _root: &Utf8PathBuf, - _fs: Arc, + _fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -146,11 +147,11 @@ mod test_storage_dev { let cases = [ ( get_native_path("test_dev_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_dev_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; let version = "xxx".to_string(); diff --git a/crates/rspack_storage/tests/error.rs b/crates/rspack_storage/tests/error.rs index 1af967e1149..0baff6b9322 100644 --- a/crates/rspack_storage/tests/error.rs +++ b/crates/rspack_storage/tests/error.rs @@ -2,10 +2,11 @@ mod test_storage_error { use std::{path::PathBuf, sync::Arc}; - use rspack_error::Result; use rspack_fs::{MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf}; - use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + use rspack_storage::{ + BridgeFileSystem, FileSystem, PackStorage, PackStorageOptions, Result, Storage, + }; pub fn get_native_path(p: &str) -> (PathBuf, PathBuf) { let base = std::env::temp_dir() @@ -23,7 +24,7 @@ mod test_storage_error { root: &Utf8PathBuf, temp_root: &Utf8PathBuf, version: &str, - fs: Arc, + fs: Arc, ) -> PackStorageOptions { PackStorageOptions { version: version.to_string(), @@ -39,7 +40,7 @@ mod test_storage_error { async fn test_initial_error( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -62,7 +63,7 @@ mod test_storage_error { async fn test_recovery_invalid_meta( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -92,7 +93,7 @@ mod test_storage_error { async fn get_first_pack( scope_name: &str, meta_path: &Utf8Path, - fs: &dyn PackFS, + fs: &dyn FileSystem, ) -> Result { let mut reader = fs.read_file(meta_path).await?; reader.read_line().await?; @@ -112,7 +113,7 @@ mod test_storage_error { async fn test_recovery_remove_pack( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -126,7 +127,7 @@ mod test_storage_error { // test assert!(storage.load("test_scope").await.is_err_and(|e| { e.to_string() - .contains("validation failed due to some packs are modified") + .contains("validate scope `test_scope` failed due to some packs are modified") })); // resume @@ -139,7 +140,7 @@ mod test_storage_error { async fn test_recovery_modified_pack( _root: &Utf8PathBuf, - _fs: Arc, + _fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -147,7 +148,7 @@ mod test_storage_error { // test assert!(storage.load("test_scope").await.is_err_and(|e| { e.to_string() - .contains("validation failed due to some packs are modified") + .contains("validate scope `test_scope` failed due to some packs are modified") })); Ok(()) @@ -159,11 +160,11 @@ mod test_storage_error { let cases = [ ( get_native_path("test_error_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_error_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; let version = "xxx".to_string(); diff --git a/crates/rspack_storage/tests/expire.rs b/crates/rspack_storage/tests/expire.rs index 7bc65917500..18bbefaebdf 100644 --- a/crates/rspack_storage/tests/expire.rs +++ b/crates/rspack_storage/tests/expire.rs @@ -2,10 +2,11 @@ mod test_storage_expire { use std::{path::PathBuf, sync::Arc}; - use rspack_error::Result; use rspack_fs::{MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8PathBuf}; - use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + use rspack_storage::{ + BridgeFileSystem, FileSystem, PackStorage, PackStorageOptions, Result, Storage, + }; pub fn get_native_path(p: &str) -> (PathBuf, PathBuf) { let base = std::env::temp_dir() @@ -23,7 +24,7 @@ mod test_storage_expire { version: &str, root: &Utf8PathBuf, temp_root: &Utf8PathBuf, - fs: Arc, + fs: Arc, ) -> Result<()> { let storage = PackStorage::new(PackStorageOptions { version: version.to_string(), @@ -59,7 +60,7 @@ mod test_storage_expire { version: &str, root: &Utf8PathBuf, temp_root: &Utf8PathBuf, - fs: Arc, + fs: Arc, ) -> Result<()> { let storage = PackStorage::new(PackStorageOptions { version: version.to_string(), @@ -73,7 +74,7 @@ mod test_storage_expire { }); assert!(storage.load("test_scope").await.is_err_and(|e| { e.to_string() - .contains("validation failed due to cache expired") + .contains("validate scope `test_scope` failed due to expiration") })); Ok(()) @@ -84,7 +85,7 @@ mod test_storage_expire { version: &str, root: &Utf8PathBuf, temp_root: &Utf8PathBuf, - fs: Arc, + fs: Arc, ) -> Result<()> { let storage = PackStorage::new(PackStorageOptions { version: version.to_string(), @@ -124,11 +125,11 @@ mod test_storage_expire { let cases = [ ( get_native_path("test_expire_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_expire_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; diff --git a/crates/rspack_storage/tests/lock.rs b/crates/rspack_storage/tests/lock.rs index 113e5f1b689..7990beee191 100644 --- a/crates/rspack_storage/tests/lock.rs +++ b/crates/rspack_storage/tests/lock.rs @@ -5,57 +5,63 @@ mod test_storage_lock { sync::{atomic::AtomicUsize, Arc}, }; - use rspack_error::{error, Result}; - use rspack_fs::{FileMetadata, MemoryFileSystem, NativeFileSystem, ReadStream, WriteStream}; + use rspack_fs::{FileMetadata, MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf}; - use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + use rspack_storage::{ + BridgeFileSystem, FSError, FSOperation, FSResult, FileSystem, PackStorage, PackStorageOptions, + Reader, Result, Storage, Writer, + }; use rustc_hash::FxHashSet as HashSet; #[derive(Debug)] - pub struct MockPackFS { - pub fs: Arc, + pub struct MockFileSystem { + pub fs: Arc, pub moved: AtomicUsize, pub break_on: usize, } #[async_trait::async_trait] - impl PackFS for MockPackFS { - async fn exists(&self, path: &Utf8Path) -> Result { + impl FileSystem for MockFileSystem { + async fn exists(&self, path: &Utf8Path) -> FSResult { self.fs.exists(path).await } - async fn remove_dir(&self, path: &Utf8Path) -> Result<()> { + async fn remove_dir(&self, path: &Utf8Path) -> FSResult<()> { self.fs.remove_dir(path).await } - async fn ensure_dir(&self, path: &Utf8Path) -> Result<()> { + async fn ensure_dir(&self, path: &Utf8Path) -> FSResult<()> { self.fs.ensure_dir(path).await } - async fn write_file(&self, path: &Utf8Path) -> Result> { + async fn write_file(&self, path: &Utf8Path) -> FSResult { self.fs.write_file(path).await } - async fn read_file(&self, path: &Utf8Path) -> Result> { + async fn read_file(&self, path: &Utf8Path) -> FSResult { self.fs.read_file(path).await } - async fn read_dir(&self, path: &Utf8Path) -> Result> { + async fn read_dir(&self, path: &Utf8Path) -> FSResult> { self.fs.read_dir(path).await } - async fn metadata(&self, path: &Utf8Path) -> Result { + async fn metadata(&self, path: &Utf8Path) -> FSResult { self.fs.metadata(path).await } - async fn remove_file(&self, path: &Utf8Path) -> Result<()> { + async fn remove_file(&self, path: &Utf8Path) -> FSResult<()> { self.fs.remove_file(path).await } - async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> Result<()> { + async fn move_file(&self, from: &Utf8Path, to: &Utf8Path) -> FSResult<()> { let moved = self.moved.load(std::sync::atomic::Ordering::Relaxed); if moved == self.break_on { - Err(error!("move failed")) + Err(FSError::from_message( + from, + FSOperation::Move, + "move failed".to_string(), + )) } else { self .moved @@ -81,7 +87,7 @@ mod test_storage_lock { version: &str, root: &Utf8PathBuf, temp_root: &Utf8PathBuf, - fs: Arc, + fs: Arc, ) -> Result<()> { let storage = PackStorage::new(PackStorageOptions { version: version.to_string(), @@ -117,7 +123,7 @@ mod test_storage_lock { version: &str, root: &Utf8PathBuf, temp_root: &Utf8PathBuf, - fs: Arc, + fs: Arc, ) -> Result<()> { let storage = PackStorage::new(PackStorageOptions { version: version.to_string(), @@ -137,7 +143,7 @@ mod test_storage_lock { version: &str, root: &Utf8PathBuf, temp_root: &Utf8PathBuf, - fs: Arc, + fs: Arc, ) -> Result<()> { let storage = PackStorage::new(PackStorageOptions { version: version.to_string(), @@ -162,11 +168,11 @@ mod test_storage_lock { let cases = [ ( get_native_path("test_lock_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_lock_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; @@ -182,7 +188,7 @@ mod test_storage_lock { "xxx", &root, &temp_root, - Arc::new(MockPackFS { + Arc::new(MockFileSystem { fs: fs.clone(), moved: AtomicUsize::new(0), break_on: 3, @@ -195,7 +201,7 @@ mod test_storage_lock { "xxx", &root, &temp_root, - Arc::new(MockPackFS { + Arc::new(MockFileSystem { fs: fs.clone(), moved: AtomicUsize::new(0), break_on: 9999, @@ -212,11 +218,11 @@ mod test_storage_lock { let cases = [ ( get_native_path("test_lock_fail_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_lock_fail_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; @@ -232,7 +238,7 @@ mod test_storage_lock { "xxx", &root, &temp_root, - Arc::new(MockPackFS { + Arc::new(MockFileSystem { fs: fs.clone(), moved: AtomicUsize::new(0), break_on: 3, @@ -245,7 +251,7 @@ mod test_storage_lock { "xxx", &root, &temp_root.join("other"), - Arc::new(MockPackFS { + Arc::new(MockFileSystem { fs: fs.clone(), moved: AtomicUsize::new(0), break_on: 9999, diff --git a/crates/rspack_storage/tests/multi.rs b/crates/rspack_storage/tests/multi.rs index 498a857f990..713f1308ab7 100644 --- a/crates/rspack_storage/tests/multi.rs +++ b/crates/rspack_storage/tests/multi.rs @@ -2,10 +2,11 @@ mod test_storage_multi { use std::{collections::HashMap, path::PathBuf, sync::Arc}; - use rspack_error::Result; use rspack_fs::{MemoryFileSystem, NativeFileSystem}; use rspack_paths::{AssertUtf8, Utf8PathBuf}; - use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + use rspack_storage::{ + BridgeFileSystem, FileSystem, PackStorage, PackStorageOptions, Result, Storage, + }; pub fn get_native_path(p: &str) -> (PathBuf, PathBuf) { let base = std::env::temp_dir() @@ -23,7 +24,7 @@ mod test_storage_multi { root: &Utf8PathBuf, temp_root: &Utf8PathBuf, version: &str, - fs: Arc, + fs: Arc, ) -> PackStorageOptions { PackStorageOptions { version: version.to_string(), @@ -39,7 +40,7 @@ mod test_storage_multi { async fn test_initial_build( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -68,7 +69,7 @@ mod test_storage_multi { async fn test_recovery_modify( root: &Utf8PathBuf, - fs: Arc, + fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -104,7 +105,7 @@ mod test_storage_multi { async fn test_recovery_final( _root: &Utf8PathBuf, - _fs: Arc, + _fs: Arc, options: PackStorageOptions, ) -> Result<()> { let storage = PackStorage::new(options); @@ -156,11 +157,11 @@ mod test_storage_multi { let cases = [ ( get_native_path("test_multi_native"), - Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + Arc::new(BridgeFileSystem(Arc::new(NativeFileSystem {}))), ), ( get_memory_path("test_multi_memory"), - Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + Arc::new(BridgeFileSystem(Arc::new(MemoryFileSystem::default()))), ), ]; let version = "xxx".to_string();