Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add progress bar for uv cache clean #8857

Merged
merged 2 commits into from
Nov 6, 2024
Merged

Add progress bar for uv cache clean #8857

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8769858
Add progress bar for `uv cache clean`
j178 Nov 6, 2024
9ea8186
Fix tests
j178 Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

51 changes: 30 additions & 21 deletions crates/uv-cache/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,8 +320,8 @@ impl Cache {
}

/// Clear the cache, removing all entries.
pub fn clear(&self) -> Result<Removal, io::Error> {
rm_rf(&self.root)
pub fn clear(&self, reporter: Option<&dyn CleanReporter>) -> Result<Removal, io::Error> {
rm_rf(&self.root, reporter)
}

/// Remove a package from the cache.
Expand Down Expand Up @@ -379,7 +379,7 @@ impl Cache {
let path = fs_err::canonicalize(entry.path())?;
if !after.contains(&path) && before.contains(&path) {
debug!("Removing dangling cache entry: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}
}
Expand Down Expand Up @@ -409,13 +409,13 @@ impl Cache {
if CacheBucket::iter().all(|bucket| entry.file_name() != bucket.to_str()) {
let path = entry.path();
debug!("Removing dangling cache bucket: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
} else {
// If the file is not a marker file, remove it.
let path = entry.path();
debug!("Removing dangling cache bucket: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}

Expand All @@ -427,7 +427,7 @@ impl Cache {
let entry = entry?;
let path = fs_err::canonicalize(entry.path())?;
debug!("Removing dangling cache environment: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}
Err(err) if err.kind() == io::ErrorKind::NotFound => (),
Expand All @@ -444,7 +444,7 @@ impl Cache {
let path = fs_err::canonicalize(entry.path())?;
if path.is_dir() {
debug!("Removing unzipped wheel entry: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}
}
Expand Down Expand Up @@ -472,10 +472,10 @@ impl Cache {

if path.is_dir() {
debug!("Removing unzipped built wheel entry: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
} else if path.is_symlink() {
debug!("Removing unzipped built wheel entry: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}
}
Expand Down Expand Up @@ -505,7 +505,7 @@ impl Cache {
let path = fs_err::canonicalize(entry.path())?;
if !references.contains(&path) {
debug!("Removing dangling cache archive: {}", path.display());
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}
}
Expand All @@ -517,6 +517,15 @@ impl Cache {
}
}

pub trait CleanReporter: Send + Sync {
/// Called after one file or directory is removed.
fn on_clean(&self);
/// Called after a package is cleaned.
fn on_clean_package(&self, _package: &str, _removal: &Removal) {}
/// Called after all files and directories are removed.
fn on_complete(&self);
}

/// The different kinds of data in the cache are stored in different bucket, which in our case
/// are subdirectories of the cache root.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)]
Expand Down Expand Up @@ -800,32 +809,32 @@ impl CacheBucket {
Self::Wheels => {
// For `pypi` wheels, we expect a directory per package (indexed by name).
let root = cache.bucket(self).join(WheelCacheKind::Pypi);
summary += rm_rf(root.join(name.to_string()))?;
summary += rm_rf(root.join(name.to_string()), None)?;

// For alternate indices, we expect a directory for every index (under an `index`
// subdirectory), followed by a directory per package (indexed by name).
let root = cache.bucket(self).join(WheelCacheKind::Index);
for directory in directories(root) {
summary += rm_rf(directory.join(name.to_string()))?;
summary += rm_rf(directory.join(name.to_string()), None)?;
}

// For direct URLs, we expect a directory for every URL, followed by a
// directory per package (indexed by name).
let root = cache.bucket(self).join(WheelCacheKind::Url);
for directory in directories(root) {
summary += rm_rf(directory.join(name.to_string()))?;
summary += rm_rf(directory.join(name.to_string()), None)?;
}
}
Self::SourceDistributions => {
// For `pypi` wheels, we expect a directory per package (indexed by name).
let root = cache.bucket(self).join(WheelCacheKind::Pypi);
summary += rm_rf(root.join(name.to_string()))?;
summary += rm_rf(root.join(name.to_string()), None)?;

// For alternate indices, we expect a directory for every index (under an `index`
// subdirectory), followed by a directory per package (indexed by name).
let root = cache.bucket(self).join(WheelCacheKind::Index);
for directory in directories(root) {
summary += rm_rf(directory.join(name.to_string()))?;
summary += rm_rf(directory.join(name.to_string()), None)?;
}

// For direct URLs, we expect a directory for every URL, followed by a
Expand All @@ -834,7 +843,7 @@ impl CacheBucket {
let root = cache.bucket(self).join(WheelCacheKind::Url);
for url in directories(root) {
if directories(&url).any(|version| is_match(&version, name)) {
summary += rm_rf(url)?;
summary += rm_rf(url, None)?;
}
}

Expand All @@ -844,7 +853,7 @@ impl CacheBucket {
let root = cache.bucket(self).join(WheelCacheKind::Path);
for path in directories(root) {
if directories(&path).any(|version| is_match(&version, name)) {
summary += rm_rf(path)?;
summary += rm_rf(path, None)?;
}
}

Expand All @@ -855,28 +864,28 @@ impl CacheBucket {
for repository in directories(root) {
for sha in directories(repository) {
if is_match(&sha, name) {
summary += rm_rf(sha)?;
summary += rm_rf(sha, None)?;
}
}
}
}
Self::Simple => {
// For `pypi` wheels, we expect a rkyv file per package, indexed by name.
let root = cache.bucket(self).join(WheelCacheKind::Pypi);
summary += rm_rf(root.join(format!("{name}.rkyv")))?;
summary += rm_rf(root.join(format!("{name}.rkyv")), None)?;

// For alternate indices, we expect a directory for every index (under an `index`
// subdirectory), followed by a directory per package (indexed by name).
let root = cache.bucket(self).join(WheelCacheKind::Index);
for directory in directories(root) {
summary += rm_rf(directory.join(format!("{name}.rkyv")))?;
summary += rm_rf(directory.join(format!("{name}.rkyv")), None)?;
}
}
Self::FlatIndex => {
// We can't know if the flat index includes a package, so we just remove the entire
// cache entry.
let root = cache.bucket(self);
summary += rm_rf(root)?;
summary += rm_rf(root, None)?;
}
Self::Git => {
// Nothing to do.
Expand Down
16 changes: 12 additions & 4 deletions crates/uv-cache/src/removal.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
use std::io;
use std::path::Path;

use crate::CleanReporter;

/// Remove a file or directory and all its contents, returning a [`Removal`] with
/// the number of files and directories removed, along with a total byte count.
pub fn rm_rf(path: impl AsRef<Path>) -> io::Result<Removal> {
pub fn rm_rf(path: impl AsRef<Path>, reporter: Option<&dyn CleanReporter>) -> io::Result<Removal> {
let mut removal = Removal::default();
removal.rm_rf(path.as_ref())?;
removal.rm_rf(path.as_ref(), reporter)?;
Ok(removal)
}

Expand All @@ -28,7 +30,7 @@ pub struct Removal {

impl Removal {
/// Recursively remove a file or directory and all its contents.
fn rm_rf(&mut self, path: &Path) -> io::Result<()> {
fn rm_rf(&mut self, path: &Path, reporter: Option<&dyn CleanReporter>) -> io::Result<()> {
let metadata = match fs_err::symlink_metadata(path) {
Ok(metadata) => metadata,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
Expand All @@ -47,6 +49,8 @@ impl Removal {
remove_file(path)?;
}

reporter.map(CleanReporter::on_clean);

return Ok(());
}

Expand All @@ -61,7 +65,7 @@ impl Removal {
if set_readable(dir).unwrap_or(false) {
// Retry the operation; if we _just_ `self.rm_rf(dir)` and continue,
// `walkdir` may give us duplicate entries for the directory.
return self.rm_rf(path);
return self.rm_rf(path, reporter);
}
}
}
Expand All @@ -88,8 +92,12 @@ impl Removal {
}
remove_file(entry.path())?;
}

reporter.map(CleanReporter::on_clean);
}

reporter.map(CleanReporter::on_complete);

Ok(())
}
}
Expand Down
8 changes: 4 additions & 4 deletions crates/uv-distribution/src/source/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1995,8 +1995,8 @@ pub fn prune(cache: &Cache) -> Result<Removal, Error> {
"Removing dangling source revision: {}",
sibling.path().display()
);
removal +=
uv_cache::rm_rf(sibling.path()).map_err(Error::CacheWrite)?;
removal += uv_cache::rm_rf(sibling.path(), None)
.map_err(Error::CacheWrite)?;
}
}
}
Expand All @@ -2020,8 +2020,8 @@ pub fn prune(cache: &Cache) -> Result<Removal, Error> {
"Removing dangling source revision: {}",
sibling.path().display()
);
removal +=
uv_cache::rm_rf(sibling.path()).map_err(Error::CacheWrite)?;
removal += uv_cache::rm_rf(sibling.path(), None)
.map_err(Error::CacheWrite)?;
}
}
}
Expand Down
1 change: 1 addition & 0 deletions crates/uv/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ tracing-subscriber = { workspace = true, features = ["json"] }
tracing-tree = { workspace = true }
unicode-width = { workspace = true }
url = { workspace = true }
walkdir = { workspace = true }
which = { workspace = true }
zip = { workspace = true }

Expand Down
Loading
Loading