Skip to content

Commit

Permalink
Remove source distribution filename from cache (#8907)
Browse files Browse the repository at this point in the history 
## Summary

In the example outlined in #8884,
this removes an unnecessary `jupyter_contrib_nbextensions-0.7.0.tar.gz`
segment (replacing it with `src`), thereby saving 39 characters and
getting that build working on my Windows machine.

This should _not_ require a version bump because we already have logic
in place to "heal" partial cache entries that lack an unzipped
distribution.

Closes #8884.

Closes #7376.
  • Loading branch information
@charliermarsh
charliermarsh authored Nov 8, 2024
1 parent 8a1b581 commit 8803361
Showing 1 changed file with 18 additions and 50 deletions.
68 changes: 18 additions & 50 deletions crates/uv-distribution/src/source/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ use uv_configuration::{BuildKind, BuildOutput, SourceStrategy};
use uv_distribution_filename::{SourceDistExtension, WheelFilename};
use uv_distribution_types::{
BuildableSource, DirectorySourceUrl, FileLocation, GitSourceUrl, HashPolicy, Hashed,
PathSourceUrl, RemoteSource, SourceDist, SourceUrl,
PathSourceUrl, SourceDist, SourceUrl,
};
use uv_extract::hash::Hasher;
use uv_fs::{rename_with_retry, write_atomic, LockedFile};
Expand Down Expand Up @@ -58,6 +58,9 @@ pub(crate) const LOCAL_REVISION: &str = "revision.rev";
/// The name of the file that contains the cached distribution metadata, encoded via `MsgPack`.
pub(crate) const METADATA: &str = "metadata.msgpack";

/// The directory within each entry under which to store the unpacked source distribution.
pub(crate) const SOURCE: &str = "src";

impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
/// Initialize a [`SourceDistributionBuilder`] from a [`BuildContext`].
pub(crate) fn new(build_context: &'a T) -> Self {
Expand Down Expand Up @@ -125,7 +128,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url(
source,
&dist.file.filename,
&url,
&cache_shard,
None,
Expand All @@ -138,8 +140,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await?
}
BuildableSource::Dist(SourceDist::DirectUrl(dist)) => {
let filename = dist.filename().expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -148,7 +148,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url(
source,
&filename,
&dist.url,
&cache_shard,
dist.subdirectory.as_deref(),
Expand Down Expand Up @@ -186,11 +185,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await?
}
BuildableSource::Url(SourceUrl::Direct(resource)) => {
let filename = resource
.url
.filename()
.expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -199,7 +193,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url(
source,
&filename,
resource.url,
&cache_shard,
resource.subdirectory,
Expand Down Expand Up @@ -281,22 +274,11 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await;
}

self.url_metadata(
source,
&dist.file.filename,
&url,
&cache_shard,
None,
dist.ext,
hashes,
client,
)
.boxed_local()
.await?
self.url_metadata(source, &url, &cache_shard, None, dist.ext, hashes, client)
.boxed_local()
.await?
}
BuildableSource::Dist(SourceDist::DirectUrl(dist)) => {
let filename = dist.filename().expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -305,7 +287,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url_metadata(
source,
&filename,
&dist.url,
&cache_shard,
dist.subdirectory.as_deref(),
Expand Down Expand Up @@ -336,11 +317,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
.await?
}
BuildableSource::Url(SourceUrl::Direct(resource)) => {
let filename = resource
.url
.filename()
.expect("Distribution must have a filename");

// For direct URLs, cache directly under the hash of the URL itself.
let cache_shard = self.build_context.cache().shard(
CacheBucket::SourceDistributions,
Expand All @@ -349,7 +325,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

self.url_metadata(
source,
&filename,
resource.url,
&cache_shard,
resource.subdirectory,
Expand Down Expand Up @@ -403,7 +378,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url<'data>(
&self,
source: &BuildableSource<'data>,
filename: &'data str,
url: &'data Url,
cache_shard: &CacheShard,
subdirectory: Option<&'data Path>,
Expand All @@ -416,7 +390,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Fetch the revision for the source distribution.
let revision = self
.url_revision(source, filename, ext, url, cache_shard, hashes, client)
.url_revision(source, ext, url, cache_shard, hashes, client)
.await?;

// Before running the build, check that the hashes match.
Expand All @@ -431,7 +405,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_dist_entry = cache_shard.entry(filename);
let source_dist_entry = cache_shard.entry(SOURCE);

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
Expand All @@ -452,7 +426,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} else {
self.heal_url_revision(
source,
filename,
ext,
url,
&source_dist_entry,
Expand Down Expand Up @@ -507,7 +480,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url_metadata<'data>(
&self,
source: &BuildableSource<'data>,
filename: &'data str,
url: &'data Url,
cache_shard: &CacheShard,
subdirectory: Option<&'data Path>,
Expand All @@ -519,7 +491,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Fetch the revision for the source distribution.
let revision = self
.url_revision(source, filename, ext, url, cache_shard, hashes, client)
.url_revision(source, ext, url, cache_shard, hashes, client)
.await?;

// Before running the build, check that the hashes match.
Expand All @@ -534,7 +506,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_dist_entry = cache_shard.entry(filename);
let source_dist_entry = cache_shard.entry(SOURCE);

// If the metadata is static, return it.
if let Some(metadata) =
Expand Down Expand Up @@ -562,7 +534,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
} else {
self.heal_url_revision(
source,
filename,
ext,
url,
&source_dist_entry,
Expand Down Expand Up @@ -644,7 +615,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn url_revision(
&self,
source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension,
url: &Url,
cache_shard: &CacheShard,
Expand All @@ -670,9 +640,9 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Download the source distribution.
debug!("Downloading source distribution: {source}");
let entry = cache_shard.shard(revision.id()).entry(filename);
let entry = cache_shard.shard(revision.id()).entry(SOURCE);
let hashes = self
.download_archive(response, source, filename, ext, entry.path(), hashes)
.download_archive(response, source, ext, entry.path(), hashes)
.await?;

Ok(revision.with_hashes(hashes))
Expand Down Expand Up @@ -743,7 +713,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_entry = cache_shard.entry("source");
let source_entry = cache_shard.entry(SOURCE);

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
Expand Down Expand Up @@ -832,7 +802,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// Scope all operations to the revision. Within the revision, there's no need to check for
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());
let source_entry = cache_shard.entry("source");
let source_entry = cache_shard.entry(SOURCE);

// If the metadata is static, return it.
if let Some(metadata) =
Expand Down Expand Up @@ -957,7 +927,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

// Unzip the archive to a temporary directory.
debug!("Unpacking source distribution: {source}");
let entry = cache_shard.shard(revision.id()).entry("source");
let entry = cache_shard.shard(revision.id()).entry(SOURCE);
let hashes = self
.persist_archive(&resource.path, resource.ext, entry.path(), hashes)
.await?;
Expand Down Expand Up @@ -1568,7 +1538,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
async fn heal_url_revision(
&self,
source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension,
url: &Url,
entry: &CacheEntry,
Expand All @@ -1581,7 +1550,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let download = |response| {
async {
let hashes = self
.download_archive(response, source, filename, ext, entry.path(), hashes)
.download_archive(response, source, ext, entry.path(), hashes)
.await?;
if hashes != revision.hashes() {
return Err(Error::CacheHeal(source.to_string()));
Expand Down Expand Up @@ -1610,7 +1579,6 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
&self,
response: Response,
source: &BuildableSource<'_>,
filename: &str,
ext: SourceDistExtension,
target: &Path,
hashes: HashPolicy<'_>,
Expand All @@ -1632,7 +1600,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
let mut hasher = uv_extract::hash::HashReader::new(reader.compat(), &mut hashers);

// Download and unzip the source distribution into a temporary directory.
let span = info_span!("download_source_dist", filename = filename, source_dist = %source);
let span = info_span!("download_source_dist", source_dist = %source);
uv_extract::stream::archive(&mut hasher, ext, temp_dir.path()).await?;
drop(span);

Expand Down

0 comments on commit 8803361

Please sign in to comment.