From da44eabce980a348e7edd73cb82e34e6d09fe367 Mon Sep 17 00:00:00 2001 From: harpsealjs Date: Tue, 17 Dec 2024 11:43:29 +0800 Subject: [PATCH] fix: duplicate move lock files (#8732) --- crates/rspack_storage/src/pack/manager/mod.rs | 47 +--- .../rspack_storage/src/pack/strategy/mod.rs | 14 +- .../src/pack/strategy/split/handle_file.rs | 129 ++++++++++- .../src/pack/strategy/split/mod.rs | 5 +- .../src/pack/strategy/split/util.rs | 39 +++- .../src/pack/strategy/split/validate_scope.rs | 38 ++-- .../src/pack/strategy/split/write_scope.rs | 65 +++--- crates/rspack_storage/tests/build.rs | 2 +- crates/rspack_storage/tests/multi.rs | 201 ++++++++++++++++++ 9 files changed, 430 insertions(+), 110 deletions(-) create mode 100644 crates/rspack_storage/tests/multi.rs diff --git a/crates/rspack_storage/src/pack/manager/mod.rs b/crates/rspack_storage/src/pack/manager/mod.rs index 53f4b17f983..b6dae30869f 100644 --- a/crates/rspack_storage/src/pack/manager/mod.rs +++ b/crates/rspack_storage/src/pack/manager/mod.rs @@ -230,21 +230,9 @@ async fn save_scopes( ) -> Result { scopes.retain(|_, scope| scope.loaded()); - for (_, scope) in scopes.iter_mut() { - strategy.before_all(scope)?; - } + strategy.before_all(&mut scopes).await?; - join_all( - scopes - .values() - .map(|scope| async move { strategy.before_write(scope).await }) - .collect_vec(), - ) - .await - .into_iter() - .collect::>>()?; - - let wrote_results = join_all( + let changed = join_all( scopes .values_mut() .map(|scope| async move { @@ -261,33 +249,14 @@ async fn save_scopes( .into_iter() .collect::>>()? .into_iter() - .collect_vec(); + .fold(WriteScopeResult::default(), |mut acc, res| { + acc.extend(res); + acc + }); strategy.write_root_meta(root_meta).await?; - - join_all( - scopes - .values() - .zip(wrote_results) - .map(|(scope, scope_wrote_result)| async move { - strategy - .after_write( - scope, - scope_wrote_result.wrote_files, - scope_wrote_result.removed_files, - ) - .await - }) - .collect_vec(), - ) - .await - .into_iter() - .collect::>>()?; - - for (_, scope) in scopes.iter_mut() { - strategy.after_all(scope)?; - } - + strategy.merge_changed(changed).await?; + strategy.after_all(&mut scopes).await?; strategy .clean_unused(root_meta, &scopes, root_options) .await?; diff --git a/crates/rspack_storage/src/pack/strategy/mod.rs b/crates/rspack_storage/src/pack/strategy/mod.rs index 60a30cb1fa8..03de839a4bf 100644 --- a/crates/rspack_storage/src/pack/strategy/mod.rs +++ b/crates/rspack_storage/src/pack/strategy/mod.rs @@ -137,7 +137,7 @@ pub trait ScopeValidateStrategy { async fn validate_packs(&self, scope: &mut PackScope) -> Result; } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct WriteScopeResult { pub wrote_files: HashSet, pub removed_files: HashSet, @@ -154,15 +154,9 @@ pub type ScopeUpdate = HashMap>; #[async_trait] pub trait ScopeWriteStrategy { fn update_scope(&self, scope: &mut PackScope, updates: ScopeUpdate) -> Result<()>; - fn before_all(&self, scope: &mut PackScope) -> Result<()>; - async fn before_write(&self, scope: &PackScope) -> Result<()>; + async fn before_all(&self, scopes: &mut HashMap) -> Result<()>; async fn write_packs(&self, scope: &mut PackScope) -> Result; async fn write_meta(&self, scope: &mut PackScope) -> Result; - async fn after_write( - &self, - scope: &PackScope, - wrote_files: HashSet, - removed_files: HashSet, - ) -> Result<()>; - fn after_all(&self, scope: &mut PackScope) -> Result<()>; + async fn merge_changed(&self, changed: WriteScopeResult) -> Result<()>; + async fn after_all(&self, scopes: &mut HashMap) -> Result<()>; } diff --git a/crates/rspack_storage/src/pack/strategy/split/handle_file.rs b/crates/rspack_storage/src/pack/strategy/split/handle_file.rs index f90b04d2c09..ea50952e8a4 100644 --- a/crates/rspack_storage/src/pack/strategy/split/handle_file.rs +++ b/crates/rspack_storage/src/pack/strategy/split/handle_file.rs @@ -10,6 +10,56 @@ use crate::{ PackFS, }; +pub async fn prepare_scope( + scope_path: &Utf8Path, + root: &Utf8Path, + temp_root: &Utf8Path, + fs: Arc, +) -> Result<()> { + let temp_path = redirect_to_path(scope_path, root, temp_root)?; + fs.remove_dir(&temp_path).await?; + fs.ensure_dir(&temp_path).await?; + fs.ensure_dir(scope_path).await?; + Ok(()) +} + +pub async fn prepare_scope_dirs( + scopes: &HashMap, + root: &Utf8Path, + temp_root: &Utf8Path, + fs: Arc, +) -> Result<()> { + let tasks = scopes.values().map(|scope| { + let fs = fs.clone(); + let scope_path = scope.path.clone(); + let root_path = root.to_path_buf(); + let temp_root_path = temp_root.to_path_buf(); + tokio::spawn(async move { prepare_scope(&scope_path, &root_path, &temp_root_path, fs).await }) + .map_err(|e| error!("{e}")) + }); + + let res = join_all(tasks) + .await + .into_iter() + .collect::>>()?; + + let mut errors = vec![]; + for task_result in res { + if let Err(e) = task_result { + errors.push(format!("- {}", e)); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(error!( + "prepare scopes directories failed:\n{}", + errors.join("\n") + )) + } +} + pub async fn remove_files(files: HashSet, fs: Arc) -> Result<()> { let tasks = files.into_iter().map(|path| { let fs = fs.clone(); @@ -35,7 +85,32 @@ pub async fn remove_files(files: HashSet, fs: Arc) -> R } } -pub async fn move_temp_files( +pub async fn write_lock( + lock_file: &str, + files: &HashSet, + root: &Utf8Path, + temp_root: &Utf8Path, + fs: Arc, +) -> Result<()> { + let lock_file = root.join(lock_file); + let mut lock_writer = fs.write_file(&lock_file).await?; + let mut contents = vec![temp_root.to_string()]; + contents.extend(files.iter().map(|path| path.to_string())); + + lock_writer + .write_all(contents.join("\n").as_bytes()) + .await?; + lock_writer.flush().await?; + Ok(()) +} + +pub async fn remove_lock(lock_file: &str, root: &Utf8Path, fs: Arc) -> Result<()> { + let lock_file = root.join(lock_file); + fs.remove_file(&lock_file).await?; + Ok(()) +} + +pub async fn move_files( files: HashSet, root: &Utf8Path, temp_root: &Utf8Path, @@ -75,27 +150,29 @@ pub async fn move_temp_files( } if errors.is_empty() { - fs.remove_file(&lock_file).await?; - Ok(()) } else { Err(error!("move temp files failed:\n{}", errors.join("\n"))) } } -pub async fn recovery_move_lock( +async fn recovery_lock( + lock: &str, root: &Utf8Path, temp_root: &Utf8Path, fs: Arc, -) -> Result<()> { - let lock_file = root.join("move.lock"); +) -> Result> { + let lock_file = root.join(lock); if !fs.exists(&lock_file).await? { - return Ok(()); + return Ok(vec![]); } let mut lock_reader = fs.read_file(&lock_file).await?; let lock_file_content = String::from_utf8(lock_reader.read_to_end().await?) .map_err(|e| error!("parse utf8 failed: {}", e))?; - let files = lock_file_content.split("\n").collect::>(); + let files = lock_file_content + .split("\n") + .map(|i| i.to_owned()) + .collect::>(); fs.remove_file(&lock_file).await?; if files.is_empty() { @@ -106,8 +183,20 @@ pub async fn recovery_move_lock( "incomplete storage due to `move.lock` from an unexpected directory" )); } - move_temp_files( - files[1..] + Ok(files[1..].to_vec()) +} + +pub async fn recovery_move_lock( + root: &Utf8Path, + temp_root: &Utf8Path, + fs: Arc, +) -> Result<()> { + let moving_files = recovery_lock("move.lock", root, temp_root, fs.clone()).await?; + if moving_files.is_empty() { + return Ok(()); + } + move_files( + moving_files .iter() .map(Utf8PathBuf::from) .collect::>(), @@ -119,6 +208,26 @@ pub async fn recovery_move_lock( Ok(()) } +pub async fn recovery_remove_lock( + root: &Utf8Path, + temp_root: &Utf8Path, + fs: Arc, +) -> Result<()> { + let removing_files = recovery_lock("remove.lock", root, temp_root, fs.clone()).await?; + if removing_files.is_empty() { + return Ok(()); + } + remove_files( + removing_files + .iter() + .map(Utf8PathBuf::from) + .collect::>(), + fs, + ) + .await?; + Ok(()) +} + pub async fn walk_dir(root: &Utf8Path, fs: Arc) -> Result> { let mut files = HashSet::default(); let mut stack = vec![root.to_owned()]; diff --git a/crates/rspack_storage/src/pack/strategy/split/mod.rs b/crates/rspack_storage/src/pack/strategy/split/mod.rs index 293e7973571..a4a7dd289b3 100644 --- a/crates/rspack_storage/src/pack/strategy/split/mod.rs +++ b/crates/rspack_storage/src/pack/strategy/split/mod.rs @@ -8,7 +8,9 @@ mod write_scope; use std::{hash::Hasher, sync::Arc}; -use handle_file::{clean_root, clean_scopes, clean_versions, recovery_move_lock}; +use handle_file::{ + clean_root, clean_scopes, clean_versions, recovery_move_lock, recovery_remove_lock, +}; use itertools::Itertools; use rspack_error::{error, Result}; use rspack_paths::{Utf8Path, Utf8PathBuf}; @@ -58,6 +60,7 @@ impl SplitPackStrategy { #[async_trait::async_trait] impl RootStrategy for SplitPackStrategy { async fn before_load(&self) -> Result<()> { + recovery_remove_lock(&self.root, &self.temp_root, self.fs.clone()).await?; recovery_move_lock(&self.root, &self.temp_root, self.fs.clone()).await?; Ok(()) } diff --git a/crates/rspack_storage/src/pack/strategy/split/util.rs b/crates/rspack_storage/src/pack/strategy/split/util.rs index a531d0fe6fd..a74bbf7069c 100644 --- a/crates/rspack_storage/src/pack/strategy/split/util.rs +++ b/crates/rspack_storage/src/pack/strategy/split/util.rs @@ -9,6 +9,15 @@ use crate::pack::data::{Pack, PackContents, PackFileMeta, PackKeys, PackScope}; pub type PackIndexList = Vec<(usize, usize)>; pub type PackInfoList<'a> = Vec<(&'a PackFileMeta, &'a Pack)>; +pub fn flag_scope_wrote(scope: &mut PackScope) { + let scope_meta = scope.meta.expect_value_mut(); + for bucket in scope_meta.packs.iter_mut() { + for pack in bucket { + pack.wrote = true; + } + } +} + pub fn get_indexed_packs<'a>( scope: &'a PackScope, filter: Option<&dyn Fn(&'a Pack, &'a PackFileMeta) -> bool>, @@ -73,11 +82,15 @@ pub mod test_pack_utils { use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf}; use rustc_hash::FxHashMap as HashMap; + use super::flag_scope_wrote; use crate::{ pack::{ data::{current_time, PackOptions, PackScope}, fs::PackFS, - strategy::{ScopeUpdate, ScopeWriteStrategy, SplitPackStrategy, WriteScopeResult}, + strategy::{ + split::handle_file::prepare_scope, ScopeUpdate, ScopeWriteStrategy, SplitPackStrategy, + WriteScopeResult, + }, }, PackBridgeFS, }; @@ -226,15 +239,21 @@ pub mod test_pack_utils { scope: &mut PackScope, strategy: &SplitPackStrategy, ) -> Result { - let mut res = WriteScopeResult::default(); - strategy.before_write(scope).await?; - res.extend(strategy.write_packs(scope).await?); - res.extend(strategy.write_meta(scope).await?); - strategy - .after_write(scope, res.wrote_files.clone(), res.removed_files.clone()) - .await?; - strategy.after_all(scope)?; - Ok(res) + prepare_scope( + &scope.path, + &strategy.root, + &strategy.temp_root, + strategy.fs.clone(), + ) + .await?; + + let mut changed = WriteScopeResult::default(); + changed.extend(strategy.write_packs(scope).await?); + changed.extend(strategy.write_meta(scope).await?); + strategy.merge_changed(changed.clone()).await?; + flag_scope_wrote(scope); + + Ok(changed) } pub fn get_native_path(p: &str) -> Utf8PathBuf { diff --git a/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs b/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs index cb44ea7cf06..14b12ab9b7a 100644 --- a/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs +++ b/crates/rspack_storage/src/pack/strategy/split/validate_scope.rs @@ -87,9 +87,15 @@ mod tests { data::{PackOptions, PackScope, RootMeta, ScopeMeta}, fs::PackFS, strategy::{ - split::util::test_pack_utils::{ - clean_strategy, create_strategies, flush_file_mtime, mock_root_meta_file, - mock_scope_meta_file, mock_updates, save_scope, UpdateVal, + split::{ + handle_file::prepare_scope, + util::{ + flag_scope_wrote, + test_pack_utils::{ + clean_strategy, create_strategies, flush_file_mtime, mock_root_meta_file, + mock_scope_meta_file, mock_updates, save_scope, UpdateVal, + }, + }, }, ScopeReadStrategy, ScopeValidateStrategy, ScopeWriteStrategy, SplitPackStrategy, ValidateResult, @@ -235,20 +241,24 @@ mod tests { strategy .update_scope(&mut mock_scope, updates) .expect("should update scope"); - strategy - .before_write(&mock_scope) - .await - .expect("should prepare dirs"); - let files = save_scope(&mut mock_scope, &strategy) + + prepare_scope( + &mock_scope.path, + &strategy.root, + &strategy.temp_root, + strategy.fs.clone(), + ) + .await + .expect("should prepare dirs"); + let changed = save_scope(&mut mock_scope, &strategy) .await .expect("should write scope"); strategy - .after_write(&mock_scope, files.wrote_files.clone(), files.removed_files) + .merge_changed(changed.clone()) .await - .expect("should clean dirs"); - strategy - .after_all(&mut mock_scope) - .expect("should modify wrote flags"); + .expect("should merge changed"); + + flag_scope_wrote(&mut mock_scope); let _ = test_valid_packs(scope_path.clone(), &strategy, pack_options.clone()) .await @@ -259,7 +269,7 @@ mod tests { &strategy, strategy.fs.clone(), pack_options.clone(), - files.wrote_files, + changed.wrote_files, ) .await .map_err(|e| panic!("{}", e)); diff --git a/crates/rspack_storage/src/pack/strategy/split/write_scope.rs b/crates/rspack_storage/src/pack/strategy/split/write_scope.rs index b285cff38ed..33e4f68c25e 100644 --- a/crates/rspack_storage/src/pack/strategy/split/write_scope.rs +++ b/crates/rspack_storage/src/pack/strategy/split/write_scope.rs @@ -4,12 +4,13 @@ use futures::TryFutureExt; use itertools::Itertools; use rayon::iter::{IntoParallelIterator, ParallelBridge, ParallelIterator}; use rspack_error::{error, Result}; -use rspack_paths::Utf8PathBuf; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; use super::{ - handle_file::{move_temp_files, redirect_to_path, remove_files}, - util::choose_bucket, + handle_file::{ + move_files, prepare_scope_dirs, redirect_to_path, remove_files, remove_lock, write_lock, + }, + util::{choose_bucket, flag_scope_wrote}, SplitPackStrategy, }; use crate::pack::{ @@ -19,37 +20,51 @@ use crate::pack::{ #[async_trait] impl ScopeWriteStrategy for SplitPackStrategy { - fn before_all(&self, _: &mut PackScope) -> Result<()> { + async fn before_all(&self, scopes: &mut HashMap) -> Result<()> { + prepare_scope_dirs(scopes, &self.root, &self.temp_root, self.fs.clone()).await?; Ok(()) } - async fn before_write(&self, scope: &PackScope) -> Result<()> { - let temp_path = redirect_to_path(&scope.path, &self.root, &self.temp_root)?; - self.fs.remove_dir(&temp_path).await?; - self.fs.ensure_dir(&temp_path).await?; - self.fs.ensure_dir(&scope.path).await?; - Ok(()) - } + async fn merge_changed(&self, changed: WriteScopeResult) -> Result<()> { + // remove files with `.lock` + write_lock( + "remove.lock", + &changed.wrote_files, + &self.root, + &self.temp_root, + self.fs.clone(), + ) + .await?; + remove_files(changed.removed_files, self.fs.clone()).await?; + remove_lock("remove.lock", &self.root, self.fs.clone()).await?; + + // move files with `.lock` + write_lock( + "move.lock", + &changed.wrote_files, + &self.root, + &self.temp_root, + self.fs.clone(), + ) + .await?; + move_files( + changed.wrote_files, + &self.root, + &self.temp_root, + self.fs.clone(), + ) + .await?; + remove_lock("move.lock", &self.root, self.fs.clone()).await?; - async fn after_write( - &self, - _scope: &PackScope, - wrote_files: HashSet, - removed_files: HashSet, - ) -> Result<()> { - remove_files(removed_files, self.fs.clone()).await?; - move_temp_files(wrote_files, &self.root, &self.temp_root, self.fs.clone()).await?; self.fs.remove_dir(&self.temp_root).await?; Ok(()) } - fn after_all(&self, scope: &mut PackScope) -> Result<()> { - let scope_meta = scope.meta.expect_value_mut(); - for bucket in scope_meta.packs.iter_mut() { - for pack in bucket { - pack.wrote = true; - } + async fn after_all(&self, scopes: &mut HashMap) -> Result<()> { + for scope in scopes.values_mut() { + flag_scope_wrote(scope); } + Ok(()) } diff --git a/crates/rspack_storage/tests/build.rs b/crates/rspack_storage/tests/build.rs index 180020d7512..6b262afc3e8 100644 --- a/crates/rspack_storage/tests/build.rs +++ b/crates/rspack_storage/tests/build.rs @@ -107,7 +107,7 @@ mod test_storage_build { #[tokio::test] #[cfg_attr(miri, ignore)] - async fn test_dev() { + async fn test_build() { let cases = [ ( get_native_path("test_build_native"), diff --git a/crates/rspack_storage/tests/multi.rs b/crates/rspack_storage/tests/multi.rs new file mode 100644 index 00000000000..498a857f990 --- /dev/null +++ b/crates/rspack_storage/tests/multi.rs @@ -0,0 +1,201 @@ +#[cfg(test)] +mod test_storage_multi { + use std::{collections::HashMap, path::PathBuf, sync::Arc}; + + use rspack_error::Result; + use rspack_fs::{MemoryFileSystem, NativeFileSystem}; + use rspack_paths::{AssertUtf8, Utf8PathBuf}; + use rspack_storage::{PackBridgeFS, PackFS, PackStorage, PackStorageOptions, Storage}; + + pub fn get_native_path(p: &str) -> (PathBuf, PathBuf) { + let base = std::env::temp_dir() + .join("rspack_test/storage/test_storage_build") + .join(p); + (base.join("cache"), base.join("temp")) + } + + pub fn get_memory_path(p: &str) -> (PathBuf, PathBuf) { + let base = PathBuf::from("/rspack_test/storage/test_storage_build/").join(p); + (base.join("cache"), base.join("temp")) + } + + fn create_pack_options( + root: &Utf8PathBuf, + temp_root: &Utf8PathBuf, + version: &str, + fs: Arc, + ) -> PackStorageOptions { + PackStorageOptions { + version: version.to_string(), + root: root.into(), + temp_root: temp_root.into(), + fs, + bucket_size: 5, + pack_size: 200, + expire: 7 * 24 * 60 * 60 * 1000, + clean: true, + } + } + + async fn test_initial_build( + root: &Utf8PathBuf, + fs: Arc, + options: PackStorageOptions, + ) -> Result<()> { + let storage = PackStorage::new(options); + let scope_data_1 = storage.load("scope_1").await?; + let scope_data_2 = storage.load("scope_2").await?; + assert!(scope_data_1.is_empty()); + assert!(scope_data_2.is_empty()); + for i in 0..500 { + storage.set( + "scope_1", + format!("scope_1_key_{:0>3}", i).as_bytes().to_vec(), + format!("scope_1_val_{:0>3}", i).as_bytes().to_vec(), + ); + storage.set( + "scope_2", + format!("scope_2_key_{:0>3}", i).as_bytes().to_vec(), + format!("scope_2_val_{:0>3}", i).as_bytes().to_vec(), + ); + } + let rx = storage.trigger_save()?; + rx.await.expect("should save")?; + assert!(fs.exists(&root.join("scope_1/scope_meta")).await?); + assert!(fs.exists(&root.join("scope_2/scope_meta")).await?); + Ok(()) + } + + async fn test_recovery_modify( + root: &Utf8PathBuf, + fs: Arc, + options: PackStorageOptions, + ) -> Result<()> { + let storage = PackStorage::new(options); + let scope_data_1 = storage.load("scope_1").await?; + let scope_data_2 = storage.load("scope_2").await?; + assert_eq!(scope_data_1.len(), 500); + assert_eq!(scope_data_2.len(), 500); + storage.set( + "scope_1", + format!("scope_1_key_{:0>3}", 111).as_bytes().to_vec(), + format!("scope_1_new_{:0>3}", 111).as_bytes().to_vec(), + ); + storage.remove( + "scope_1", + format!("scope_1_key_{:0>3}", 222).as_bytes().as_ref(), + ); + + storage.set( + "scope_2", + format!("scope_2_key_{:0>3}", 333).as_bytes().to_vec(), + format!("scope_2_new_{:0>3}", 333).as_bytes().to_vec(), + ); + storage.remove( + "scope_2", + format!("scope_2_key_{:0>3}", 444).as_bytes().as_ref(), + ); + let rx = storage.trigger_save()?; + rx.await.expect("should save")?; + assert!(fs.exists(&root.join("scope_1/scope_meta")).await?); + assert!(fs.exists(&root.join("scope_2/scope_meta")).await?); + Ok(()) + } + + async fn test_recovery_final( + _root: &Utf8PathBuf, + _fs: Arc, + options: PackStorageOptions, + ) -> Result<()> { + let storage = PackStorage::new(options); + let scope_data_1 = storage + .load("scope_1") + .await? + .into_iter() + .map(|(k, v)| { + ( + String::from_utf8(k.to_vec()).expect("should be utf8"), + String::from_utf8(v.to_vec()).expect("should be utf8"), + ) + }) + .collect::>(); + assert_eq!(scope_data_1.len(), 499); + assert_eq!( + *scope_data_1 + .get(&format!("scope_1_key_{:0>3}", 111)) + .expect("should get modified value"), + format!("scope_1_new_{:0>3}", 111) + ); + assert!(!scope_data_1.contains_key(&format!("scope_1_key_{:0>3}", 222))); + + let scope_data_2 = storage + .load("scope_2") + .await? + .into_iter() + .map(|(k, v)| { + ( + String::from_utf8(k.to_vec()).expect("should be utf8"), + String::from_utf8(v.to_vec()).expect("should be utf8"), + ) + }) + .collect::>(); + assert_eq!(scope_data_2.len(), 499); + assert_eq!( + *scope_data_2 + .get(&format!("scope_2_key_{:0>3}", 333)) + .expect("should get modified value"), + format!("scope_2_new_{:0>3}", 333) + ); + assert!(!scope_data_2.contains_key(&format!("scope_2_key_{:0>3}", 444))); + Ok(()) + } + + #[tokio::test] + #[cfg_attr(miri, ignore)] + async fn test_multi() { + let cases = [ + ( + get_native_path("test_multi_native"), + Arc::new(PackBridgeFS(Arc::new(NativeFileSystem {}))), + ), + ( + get_memory_path("test_multi_memory"), + Arc::new(PackBridgeFS(Arc::new(MemoryFileSystem::default()))), + ), + ]; + let version = "xxx".to_string(); + + for ((root, temp_root), fs) in cases { + let root = root.assert_utf8(); + let temp_root = temp_root.assert_utf8(); + fs.remove_dir(&root).await.expect("should remove root"); + fs.remove_dir(&temp_root) + .await + .expect("should remove temp root"); + + let _ = test_initial_build( + &root.join(&version), + fs.clone(), + create_pack_options(&root, &temp_root, &version, fs.clone()), + ) + .await + .map_err(|e| panic!("{}", e)); + + let _ = test_recovery_modify( + &root.join(&version), + fs.clone(), + create_pack_options(&root, &temp_root, &version, fs.clone()), + ) + .await + .map_err(|e| panic!("{}", e)); + + let _ = test_recovery_final( + &root.join(&version), + fs.clone(), + create_pack_options(&root, &temp_root, &version, fs.clone()), + ) + .await + .map_err(|e| panic!("{}", e)); + } + } +}