Skip to content

Commit

Permalink
feat: implement output.clean.keep: Option<string> (#8479)
Browse files Browse the repository at this point in the history
  • Loading branch information
ClSlaid authored Dec 3, 2024
1 parent 08ddcdd commit 044e8e3
Show file tree
Hide file tree
Showing 23 changed files with 470 additions and 45 deletions.
13 changes: 12 additions & 1 deletion crates/node_binding/binding.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,17 @@ export interface JsChunkGroupOrigin {
request?: string
}

/**
* File clean options
*
* This matches with:
* - keep:
* - If a string, keep the files under this path
*/
export interface JsCleanOptions {
keep?: string
}

export interface JsCodegenerationResult {
sources: Record<string, string>
}
Expand Down Expand Up @@ -1744,7 +1755,7 @@ export interface RawOptions {
export interface RawOutputOptions {
path: string
pathinfo: boolean | "verbose"
clean: boolean
clean: boolean | JsCleanOptions
publicPath: "auto" | JsFilename
assetModuleFilename: JsFilename
wasmLoading: string
Expand Down
13 changes: 9 additions & 4 deletions crates/rspack_binding_options/src/options/raw_output.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use napi::Either;
use napi_derive::napi;
use rspack_binding_values::library::JsLibraryOptions;
use rspack_binding_values::JsFilename;
use rspack_core::{CrossOriginLoading, Environment, PathInfo};
use rspack_binding_values::{JsCleanOptions, JsFilename};
use rspack_core::{CleanOptions, CrossOriginLoading, Environment, PathInfo};
use rspack_core::{OutputOptions, TrustedTypes};

#[derive(Debug)]
Expand Down Expand Up @@ -66,7 +66,7 @@ pub struct RawOutputOptions {
pub path: String,
#[napi(ts_type = "boolean | \"verbose\"")]
pub pathinfo: Either<bool, String>,
pub clean: bool,
pub clean: Either<bool, JsCleanOptions>,
#[napi(ts_type = "\"auto\" | JsFilename")]
pub public_path: JsFilename,
pub asset_module_filename: JsFilename,
Expand Down Expand Up @@ -120,10 +120,15 @@ impl TryFrom<RawOutputOptions> for OutputOptions {
Either::B(s) => PathInfo::String(s),
};

let clean = match value.clean {
Either::A(b) => CleanOptions::CleanAll(b),
Either::B(cop) => cop.to_clean_options(),
};

Ok(OutputOptions {
path: value.path.into(),
pathinfo,
clean: value.clean,
clean,
public_path: value.public_path.into(),
asset_module_filename: value.asset_module_filename.into(),
wasm_loading: value.wasm_loading.as_str().into(),
Expand Down
31 changes: 31 additions & 0 deletions crates/rspack_binding_values/src/clean_options.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
use napi_derive::napi;
use rspack_core::CleanOptions;
use rspack_napi::napi;

/// File clean options
///
/// This matches with:
/// - keep:
/// - If a string, keep the files under this path
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct JsCleanOptions {
pub keep: Option<String>,
// todo:
// - support RegExp type
// if path match the RegExp, keep the file
// - support function type
// if the fn returns true on path str, keep the file
}

impl JsCleanOptions {
pub fn to_clean_options(&self) -> CleanOptions {
let keep = self.keep.as_ref();
if let Some(path) = keep {
let p = path.as_str();
CleanOptions::from(p)
} else {
CleanOptions::CleanAll(false)
}
}
}
2 changes: 2 additions & 0 deletions crates/rspack_binding_values/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ mod asset_condition;
mod chunk;
mod chunk_graph;
mod chunk_group;
mod clean_options;
mod codegen_result;
mod compilation;
mod context_module_factory;
Expand All @@ -30,6 +31,7 @@ pub use asset_condition::*;
pub use chunk::*;
pub use chunk_graph::*;
pub use chunk_group::*;
pub use clean_options::*;
pub use codegen_result::*;
pub use compilation::*;
pub use context_module_factory::*;
Expand Down
85 changes: 56 additions & 29 deletions crates/rspack_core/src/compiler/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ use crate::cache::{new_cache, Cache};
use crate::incremental::IncrementalPasses;
use crate::old_cache::Cache as OldCache;
use crate::{
fast_set, include_hash, BoxPlugin, CompilerOptions, Logger, PluginDriver, ResolverFactory,
SharedPluginDriver,
fast_set, include_hash, trim_dir, BoxPlugin, CleanOptions, CompilerOptions, Logger, PluginDriver,
ResolverFactory, SharedPluginDriver,
};
use crate::{ContextModuleFactory, NormalModuleFactory};

Expand Down Expand Up @@ -73,7 +73,7 @@ impl Compiler {
options: CompilerOptions,
plugins: Vec<BoxPlugin>,
buildtime_plugins: Vec<BoxPlugin>,
output_filesystem: Option<Box<dyn WritableFileSystem + Send + Sync>>,
output_filesystem: Option<Box<dyn WritableFileSystem>>,
// only supports passing input_filesystem in rust api, no support for js api
input_filesystem: Option<Arc<dyn FileSystem + Send + Sync>>,
// no need to pass resolve_factory in rust api
Expand Down Expand Up @@ -264,32 +264,7 @@ impl Compiler {

#[instrument(name = "emit_assets", skip_all)]
pub async fn emit_assets(&mut self) -> Result<()> {
if self.options.output.clean {
if self.emitted_asset_versions.is_empty() {
self
.output_filesystem
.remove_dir_all(&self.options.output.path)
.await?;
} else {
// clean unused file
let assets = self.compilation.assets();
let _ = self
.emitted_asset_versions
.iter()
.filter_map(|(filename, _version)| {
if !assets.contains_key(filename) {
let filename = filename.to_owned();
Some(async {
let filename = Utf8Path::new(&self.options.output.path).join(filename);
let _ = self.output_filesystem.remove_file(&filename).await;
})
} else {
None
}
})
.collect::<FuturesResults<_>>();
}
}
self.run_clean_options().await?;

self
.plugin_driver
Expand Down Expand Up @@ -421,6 +396,58 @@ impl Compiler {
Ok(())
}

async fn run_clean_options(&mut self) -> Result<()> {
let clean_options = &self.options.output.clean;

// keep all
if let CleanOptions::CleanAll(false) = clean_options {
return Ok(());
}

if self.emitted_asset_versions.is_empty() {
if let CleanOptions::KeepPath(p) = clean_options {
let path_to_keep = self.options.output.path.join(Utf8Path::new(p));
trim_dir(
&*self.output_filesystem,
&self.options.output.path,
&path_to_keep,
)
.await?;
return Ok(());
}

// CleanOptions::CleanAll(true) only
debug_assert!(matches!(clean_options, CleanOptions::CleanAll(true)));

self
.output_filesystem
.remove_dir_all(&self.options.output.path)
.await?;
return Ok(());
}

let assets = self.compilation.assets();
let _ = self
.emitted_asset_versions
.iter()
.filter_map(|(filename, _version)| {
if !assets.contains_key(filename) {
let filename = filename.to_owned();
Some(async {
if !clean_options.keep(filename.as_str()) {
let filename = Utf8Path::new(&self.options.output.path).join(filename);
let _ = self.output_filesystem.remove_file(&filename).await;
}
})
} else {
None
}
})
.collect::<FuturesResults<_>>();

Ok(())
}

fn new_compilation_params(&self) -> CompilationParams {
CompilationParams {
normal_module_factory: Arc::new(NormalModuleFactory::new(
Expand Down
51 changes: 51 additions & 0 deletions crates/rspack_core/src/options/clean_options.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
use std::{path::PathBuf, str::FromStr};

use rspack_paths::Utf8PathBuf;

/// rust representation of the clean options
// TODO: support RegExp and function type
#[derive(Debug)]
pub enum CleanOptions {
// if true, clean all files
CleanAll(bool),
// keep the files under this path
KeepPath(Utf8PathBuf),
}

impl CleanOptions {
pub fn keep(&self, path: &str) -> bool {
match self {
Self::CleanAll(value) => !*value,
Self::KeepPath(value) => {
let path = PathBuf::from(path);
path.starts_with(value)
}
}
}
}

impl From<bool> for CleanOptions {
fn from(value: bool) -> Self {
Self::CleanAll(value)
}
}

impl From<&'_ str> for CleanOptions {
fn from(value: &str) -> Self {
let pb = Utf8PathBuf::from_str(value).expect("should be a valid path");
Self::KeepPath(pb)
}
}
impl From<&String> for CleanOptions {
fn from(value: &String) -> Self {
let pb = Utf8PathBuf::from_str(value).expect("should be a valid path");
Self::KeepPath(pb)
}
}

impl From<String> for CleanOptions {
fn from(value: String) -> Self {
let pb = Utf8PathBuf::from_str(&value).expect("should be a valid path");
Self::KeepPath(pb)
}
}
2 changes: 2 additions & 0 deletions crates/rspack_core/src/options/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,5 @@ mod node;
pub use node::*;
mod filename;
pub use filename::*;
mod clean_options;
pub use clean_options::*;
3 changes: 2 additions & 1 deletion crates/rspack_core/src/options/output.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use rspack_macros::MergeFrom;
use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf};
use sugar_path::SugarPath;

use super::CleanOptions;
use crate::{Chunk, ChunkGroupByUkey, ChunkKind, Compilation, Filename, FilenameTemplate};

#[derive(Debug)]
Expand All @@ -21,7 +22,7 @@ pub enum PathInfo {
pub struct OutputOptions {
pub path: Utf8PathBuf,
pub pathinfo: PathInfo,
pub clean: bool,
pub clean: CleanOptions,
pub public_path: PublicPath,
pub asset_module_filename: Filename,
pub wasm_loading: WasmLoading,
Expand Down
Loading

2 comments on commit 044e8e3

@rspack-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Benchmark detail: Open

Name Base (2024-12-03 2ec1cf5) Current Change
10000_big_production-mode_disable-minimize + exec 42.1 s ± 1.03 s 42.4 s ± 677 ms +0.75 %
10000_development-mode + exec 1.79 s ± 11 ms 1.81 s ± 25 ms +1.04 %
10000_development-mode_hmr + exec 647 ms ± 8.7 ms 648 ms ± 2.3 ms +0.23 %
10000_production-mode + exec 2.4 s ± 44 ms 2.4 s ± 28 ms -0.35 %
arco-pro_development-mode + exec 1.76 s ± 92 ms 1.77 s ± 82 ms +0.44 %
arco-pro_development-mode_hmr + exec 425 ms ± 1.9 ms 426 ms ± 2.1 ms +0.27 %
arco-pro_production-mode + exec 3.1 s ± 67 ms 3.11 s ± 84 ms +0.25 %
arco-pro_production-mode_generate-package-json-webpack-plugin + exec 3.15 s ± 71 ms 3.16 s ± 92 ms +0.39 %
threejs_development-mode_10x + exec 1.62 s ± 15 ms 1.62 s ± 14 ms +0.46 %
threejs_development-mode_10x_hmr + exec 808 ms ± 7.7 ms 808 ms ± 19 ms +0.01 %
threejs_production-mode_10x + exec 4.9 s ± 17 ms 4.91 s ± 50 ms +0.09 %
10000_big_production-mode_disable-minimize + rss memory 13327 MiB ± 247 MiB 13464 MiB ± 120 MiB +1.03 %
10000_development-mode + rss memory 778 MiB ± 24.5 MiB 769 MiB ± 14.4 MiB -1.18 %
10000_development-mode_hmr + rss memory 2088 MiB ± 261 MiB 2047 MiB ± 284 MiB -1.95 %
10000_production-mode + rss memory 665 MiB ± 30.3 MiB 663 MiB ± 40 MiB -0.35 %
arco-pro_development-mode + rss memory 731 MiB ± 23.7 MiB 721 MiB ± 41.3 MiB -1.40 %
arco-pro_development-mode_hmr + rss memory 904 MiB ± 67.8 MiB 905 MiB ± 155 MiB +0.17 %
arco-pro_production-mode + rss memory 849 MiB ± 52.5 MiB 881 MiB ± 55.2 MiB +3.81 %
arco-pro_production-mode_generate-package-json-webpack-plugin + rss memory 848 MiB ± 54.4 MiB 853 MiB ± 78.6 MiB +0.56 %
threejs_development-mode_10x + rss memory 817 MiB ± 47.2 MiB 797 MiB ± 45.9 MiB -2.49 %
threejs_development-mode_10x_hmr + rss memory 2170 MiB ± 202 MiB 2125 MiB ± 206 MiB -2.07 %
threejs_production-mode_10x + rss memory 1033 MiB ± 67.6 MiB 1045 MiB ± 47.9 MiB +1.11 %

@rspack-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📝 Ran ecosystem CI: Open

suite result
modernjs ✅ success
_selftest ✅ success
rsdoctor ✅ success
rspress ✅ success
rslib ❌ failure
rsbuild ✅ success
examples ✅ success
devserver ✅ success
nuxt ✅ success

Please sign in to comment.