Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement output.clean.keep: Option<string> #8479

Merged
merged 28 commits into from
Dec 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
fcc669e
feat: implement output.clean.keep: Option<string>
ClSlaid Nov 19, 2024
be23764
Merge remote-tracking branch 'origin/main' into feat/webpack/output-c…
ClSlaid Nov 19, 2024
6ac7cf0
Merge branch 'web-infra-dev:main' into feat/webpack/output-clean
ClSlaid Nov 23, 2024
69103e2
patch: add test files
ClSlaid Nov 23, 2024
9b0827b
patch: recover test files
ClSlaid Nov 23, 2024
74d39be
patch: rewrite datatype for clean
ClSlaid Nov 23, 2024
29ab514
patch: fix ts types
ClSlaid Nov 24, 2024
fe07604
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 24, 2024
c9ae7c6
patch: fix-up simple clean test
ClSlaid Nov 24, 2024
703e5a5
patch: remove debug remains
ClSlaid Nov 25, 2024
b16e52c
patch: rerun test, hope ./x could fix it
ClSlaid Nov 25, 2024
a37072e
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 26, 2024
48e4468
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 26, 2024
c0d25af
patch: improve clean performance
ClSlaid Nov 26, 2024
05ee6e3
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 27, 2024
7902a17
patch: fix format
ClSlaid Nov 27, 2024
1ff245f
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 27, 2024
d09e782
fix: add path checks to writable fs extension
ClSlaid Nov 27, 2024
29bfe79
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 27, 2024
e76c8e5
patch: remove unnecessary clone
ClSlaid Nov 28, 2024
49d5540
patch: move Fs modification to utils
ClSlaid Nov 28, 2024
fea0fee
patch: rename structs and trim unneeded trait implementations
ClSlaid Nov 28, 2024
9eccf95
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Nov 28, 2024
f222a59
fix: binding.d.ts update
ClSlaid Nov 28, 2024
5a5c21e
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Dec 2, 2024
e285a33
patch: simplify rust structs
ClSlaid Dec 3, 2024
8bfe5b0
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Dec 3, 2024
d2335fc
Merge branch 'main' into feat/webpack/output-clean
ClSlaid Dec 3, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion crates/node_binding/binding.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -518,6 +518,17 @@ export interface JsChunkGroupOrigin {
request?: string
}

/**
* File clean options
*
* This matches with:
* - keep:
* - If a string, keep the files under this path
*/
export interface JsCleanOptions {
keep?: string
}

export interface JsCodegenerationResult {
sources: Record<string, string>
}
Expand Down Expand Up @@ -1744,7 +1755,7 @@ export interface RawOptions {
export interface RawOutputOptions {
path: string
pathinfo: boolean | "verbose"
clean: boolean
clean: boolean | JsCleanOptions
publicPath: "auto" | JsFilename
assetModuleFilename: JsFilename
wasmLoading: string
Expand Down
13 changes: 9 additions & 4 deletions crates/rspack_binding_options/src/options/raw_output.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
use napi::Either;
use napi_derive::napi;
use rspack_binding_values::library::JsLibraryOptions;
use rspack_binding_values::JsFilename;
use rspack_core::{CrossOriginLoading, Environment, PathInfo};
use rspack_binding_values::{JsCleanOptions, JsFilename};
use rspack_core::{CleanOptions, CrossOriginLoading, Environment, PathInfo};
use rspack_core::{OutputOptions, TrustedTypes};

#[derive(Debug)]
Expand Down Expand Up @@ -66,7 +66,7 @@ pub struct RawOutputOptions {
pub path: String,
#[napi(ts_type = "boolean | \"verbose\"")]
pub pathinfo: Either<bool, String>,
pub clean: bool,
pub clean: Either<bool, JsCleanOptions>,
#[napi(ts_type = "\"auto\" | JsFilename")]
pub public_path: JsFilename,
pub asset_module_filename: JsFilename,
Expand Down Expand Up @@ -120,10 +120,15 @@ impl TryFrom<RawOutputOptions> for OutputOptions {
Either::B(s) => PathInfo::String(s),
};

let clean = match value.clean {
Either::A(b) => CleanOptions::CleanAll(b),
Either::B(cop) => cop.to_clean_options(),
};

Ok(OutputOptions {
path: value.path.into(),
pathinfo,
clean: value.clean,
clean,
public_path: value.public_path.into(),
asset_module_filename: value.asset_module_filename.into(),
wasm_loading: value.wasm_loading.as_str().into(),
Expand Down
31 changes: 31 additions & 0 deletions crates/rspack_binding_values/src/clean_options.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
use napi_derive::napi;
use rspack_core::CleanOptions;
use rspack_napi::napi;

/// File clean options
///
/// This matches with:
/// - keep:
/// - If a string, keep the files under this path
#[napi(object, object_to_js = false)]
#[derive(Debug)]
pub struct JsCleanOptions {
pub keep: Option<String>,
// todo:
// - support RegExp type
// if path match the RegExp, keep the file
// - support function type
// if the fn returns true on path str, keep the file
}

impl JsCleanOptions {
pub fn to_clean_options(&self) -> CleanOptions {
let keep = self.keep.as_ref();
if let Some(path) = keep {
let p = path.as_str();
CleanOptions::from(p)
} else {
CleanOptions::CleanAll(false)
}
}
}
2 changes: 2 additions & 0 deletions crates/rspack_binding_values/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ mod asset_condition;
mod chunk;
mod chunk_graph;
mod chunk_group;
mod clean_options;
mod codegen_result;
mod compilation;
mod context_module_factory;
Expand All @@ -30,6 +31,7 @@ pub use asset_condition::*;
pub use chunk::*;
pub use chunk_graph::*;
pub use chunk_group::*;
pub use clean_options::*;
pub use codegen_result::*;
pub use compilation::*;
pub use context_module_factory::*;
Expand Down
85 changes: 56 additions & 29 deletions crates/rspack_core/src/compiler/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ use crate::cache::{new_cache, Cache};
use crate::incremental::IncrementalPasses;
use crate::old_cache::Cache as OldCache;
use crate::{
fast_set, include_hash, BoxPlugin, CompilerOptions, Logger, PluginDriver, ResolverFactory,
SharedPluginDriver,
fast_set, include_hash, trim_dir, BoxPlugin, CleanOptions, CompilerOptions, Logger, PluginDriver,
ResolverFactory, SharedPluginDriver,
};
use crate::{ContextModuleFactory, NormalModuleFactory};

Expand Down Expand Up @@ -73,7 +73,7 @@ impl Compiler {
options: CompilerOptions,
plugins: Vec<BoxPlugin>,
buildtime_plugins: Vec<BoxPlugin>,
output_filesystem: Option<Box<dyn WritableFileSystem + Send + Sync>>,
output_filesystem: Option<Box<dyn WritableFileSystem>>,
// only supports passing input_filesystem in rust api, no support for js api
input_filesystem: Option<Arc<dyn FileSystem + Send + Sync>>,
// no need to pass resolve_factory in rust api
Expand Down Expand Up @@ -264,32 +264,7 @@ impl Compiler {

#[instrument(name = "emit_assets", skip_all)]
pub async fn emit_assets(&mut self) -> Result<()> {
if self.options.output.clean {
if self.emitted_asset_versions.is_empty() {
self
.output_filesystem
.remove_dir_all(&self.options.output.path)
.await?;
} else {
// clean unused file
let assets = self.compilation.assets();
let _ = self
.emitted_asset_versions
.iter()
.filter_map(|(filename, _version)| {
if !assets.contains_key(filename) {
let filename = filename.to_owned();
Some(async {
let filename = Utf8Path::new(&self.options.output.path).join(filename);
let _ = self.output_filesystem.remove_file(&filename).await;
})
} else {
None
}
})
.collect::<FuturesResults<_>>();
}
}
self.run_clean_options().await?;

self
.plugin_driver
Expand Down Expand Up @@ -421,6 +396,58 @@ impl Compiler {
Ok(())
}

async fn run_clean_options(&mut self) -> Result<()> {
let clean_options = &self.options.output.clean;

// keep all
if let CleanOptions::CleanAll(false) = clean_options {
return Ok(());
}

if self.emitted_asset_versions.is_empty() {
if let CleanOptions::KeepPath(p) = clean_options {
let path_to_keep = self.options.output.path.join(Utf8Path::new(p));
trim_dir(
&*self.output_filesystem,
&self.options.output.path,
&path_to_keep,
)
.await?;
return Ok(());
}

// CleanOptions::CleanAll(true) only
debug_assert!(matches!(clean_options, CleanOptions::CleanAll(true)));

self
.output_filesystem
.remove_dir_all(&self.options.output.path)
.await?;
return Ok(());
}

let assets = self.compilation.assets();
let _ = self
.emitted_asset_versions
.iter()
.filter_map(|(filename, _version)| {
if !assets.contains_key(filename) {
let filename = filename.to_owned();
Some(async {
if !clean_options.keep(filename.as_str()) {
let filename = Utf8Path::new(&self.options.output.path).join(filename);
let _ = self.output_filesystem.remove_file(&filename).await;
}
})
} else {
None
}
})
.collect::<FuturesResults<_>>();

Ok(())
}

fn new_compilation_params(&self) -> CompilationParams {
CompilationParams {
normal_module_factory: Arc::new(NormalModuleFactory::new(
Expand Down
51 changes: 51 additions & 0 deletions crates/rspack_core/src/options/clean_options.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
use std::{path::PathBuf, str::FromStr};

use rspack_paths::Utf8PathBuf;

/// rust representation of the clean options
// TODO: support RegExp and function type
#[derive(Debug)]
pub enum CleanOptions {
// if true, clean all files
CleanAll(bool),
// keep the files under this path
KeepPath(Utf8PathBuf),
}

impl CleanOptions {
pub fn keep(&self, path: &str) -> bool {
match self {
Self::CleanAll(value) => !*value,
Self::KeepPath(value) => {
let path = PathBuf::from(path);
path.starts_with(value)
}
}
}
}

impl From<bool> for CleanOptions {
fn from(value: bool) -> Self {
Self::CleanAll(value)
}
}

impl From<&'_ str> for CleanOptions {
fn from(value: &str) -> Self {
let pb = Utf8PathBuf::from_str(value).expect("should be a valid path");
Self::KeepPath(pb)
}
}
impl From<&String> for CleanOptions {
fn from(value: &String) -> Self {
let pb = Utf8PathBuf::from_str(value).expect("should be a valid path");
Self::KeepPath(pb)
}
}

impl From<String> for CleanOptions {
fn from(value: String) -> Self {
let pb = Utf8PathBuf::from_str(&value).expect("should be a valid path");
Self::KeepPath(pb)
}
}
2 changes: 2 additions & 0 deletions crates/rspack_core/src/options/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,5 @@ mod node;
pub use node::*;
mod filename;
pub use filename::*;
mod clean_options;
pub use clean_options::*;
3 changes: 2 additions & 1 deletion crates/rspack_core/src/options/output.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use rspack_macros::MergeFrom;
use rspack_paths::{AssertUtf8, Utf8Path, Utf8PathBuf};
use sugar_path::SugarPath;

use super::CleanOptions;
use crate::{Chunk, ChunkGroupByUkey, ChunkKind, Compilation, Filename, FilenameTemplate};

#[derive(Debug)]
Expand All @@ -21,7 +22,7 @@ pub enum PathInfo {
pub struct OutputOptions {
pub path: Utf8PathBuf,
pub pathinfo: PathInfo,
pub clean: bool,
pub clean: CleanOptions,
pub public_path: PublicPath,
pub asset_module_filename: Filename,
pub wasm_loading: WasmLoading,
Expand Down
Loading
Loading