Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Invalidate cache when --config-settings change #7139

Merged
merged 1 commit into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/uv-dispatch/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
&BuildOptions::default(),
self.hasher,
self.index_locations,
self.config_settings,
self.cache(),
venv,
&markers,
Expand Down
1 change: 1 addition & 0 deletions crates/uv-distribution/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ license = { workspace = true }
workspace = true

[dependencies]
cache-key = { workspace = true }
distribution-filename = { workspace = true }
distribution-types = { workspace = true }
install-wheel-rs = { workspace = true }
Expand Down
38 changes: 37 additions & 1 deletion crates/uv-distribution/src/index/built_wheel_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use distribution_types::{
use platform_tags::Tags;
use uv_cache::{Cache, CacheBucket, CacheShard, WheelCache};
use uv_cache_info::CacheInfo;
use uv_configuration::ConfigSettings;
use uv_fs::symlinks;
use uv_types::HashStrategy;

Expand All @@ -16,15 +17,22 @@ pub struct BuiltWheelIndex<'a> {
cache: &'a Cache,
tags: &'a Tags,
hasher: &'a HashStrategy,
build_configuration: &'a ConfigSettings,
}

impl<'a> BuiltWheelIndex<'a> {
/// Initialize an index of built distributions.
pub fn new(cache: &'a Cache, tags: &'a Tags, hasher: &'a HashStrategy) -> Self {
pub fn new(
cache: &'a Cache,
tags: &'a Tags,
hasher: &'a HashStrategy,
build_configuration: &'a ConfigSettings,
) -> Self {
Self {
cache,
tags,
hasher,
build_configuration,
}
}

Expand Down Expand Up @@ -53,6 +61,13 @@ impl<'a> BuiltWheelIndex<'a> {

let cache_shard = cache_shard.shard(revision.id());

// If there are build settings, we need to scope to a cache shard.
let cache_shard = if self.build_configuration.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(self.build_configuration))
};

Ok(self.find(&cache_shard))
}
/// Return the most compatible [`CachedWheel`] for a given source distribution at a local path.
Expand Down Expand Up @@ -83,6 +98,13 @@ impl<'a> BuiltWheelIndex<'a> {

let cache_shard = cache_shard.shard(revision.id());

// If there are build settings, we need to scope to a cache shard.
let cache_shard = if self.build_configuration.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(self.build_configuration))
};

Ok(self
.find(&cache_shard)
.map(|wheel| wheel.with_cache_info(cache_info)))
Expand Down Expand Up @@ -125,6 +147,13 @@ impl<'a> BuiltWheelIndex<'a> {

let cache_shard = cache_shard.shard(revision.id());

// If there are build settings, we need to scope to a cache shard.
let cache_shard = if self.build_configuration.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(self.build_configuration))
};

Ok(self
.find(&cache_shard)
.map(|wheel| wheel.with_cache_info(cache_info)))
Expand All @@ -144,6 +173,13 @@ impl<'a> BuiltWheelIndex<'a> {
WheelCache::Git(&source_dist.url, &git_sha.to_short_string()).root(),
);

// If there are build settings, we need to scope to a cache shard.
let cache_shard = if self.build_configuration.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(self.build_configuration))
};

self.find(&cache_shard)
}

Expand Down
64 changes: 64 additions & 0 deletions crates/uv-distribution/src/source/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(tags, &cache_shard) {
return Ok(built_wheel.with_hashes(revision.into_hashes()));
Expand Down Expand Up @@ -519,6 +527,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
});
}

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// Otherwise, we either need to build the metadata.
// If the backend supports `prepare_metadata_for_build_wheel`, use it.
if let Some(metadata) = self
Expand Down Expand Up @@ -671,6 +687,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(tags, &cache_shard) {
return Ok(built_wheel);
Expand Down Expand Up @@ -781,6 +805,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
});
}

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// Otherwise, we need to build a wheel.
let task = self
.reporter
Expand Down Expand Up @@ -897,6 +929,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
// freshness, since entries have to be fresher than the revision itself.
let cache_shard = cache_shard.shard(revision.id());

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(tags, &cache_shard) {
return Ok(built_wheel);
Expand Down Expand Up @@ -1020,6 +1060,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
));
}

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// Otherwise, we need to build a wheel.
let task = self
.reporter
Expand Down Expand Up @@ -1131,6 +1179,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {

let _lock = lock_shard(&cache_shard).await?;

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// If the cache contains a compatible wheel, return it.
if let Some(built_wheel) = BuiltWheelMetadata::find_in_cache(tags, &cache_shard) {
return Ok(built_wheel);
Expand Down Expand Up @@ -1257,6 +1313,14 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
));
}

// If there are build settings, we need to scope to a cache shard.
let config_settings = self.build_context.config_settings();
let cache_shard = if config_settings.is_empty() {
cache_shard
} else {
cache_shard.shard(cache_key::cache_digest(config_settings))
};

// Otherwise, we need to build a wheel.
let task = self
.reporter
Expand Down
5 changes: 3 additions & 2 deletions crates/uv-installer/src/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use platform_tags::Tags;
use pypi_types::{Requirement, RequirementSource, ResolverMarkerEnvironment};
use uv_cache::{Cache, CacheBucket, WheelCache};
use uv_cache_info::{CacheInfo, Timestamp};
use uv_configuration::{BuildOptions, Reinstall};
use uv_configuration::{BuildOptions, ConfigSettings, Reinstall};
use uv_distribution::{
BuiltWheelIndex, HttpArchivePointer, LocalArchivePointer, RegistryWheelIndex,
};
Expand Down Expand Up @@ -57,14 +57,15 @@ impl<'a> Planner<'a> {
build_options: &BuildOptions,
hasher: &HashStrategy,
index_locations: &IndexLocations,
config_settings: &ConfigSettings,
cache: &Cache,
venv: &PythonEnvironment,
markers: &ResolverMarkerEnvironment,
tags: &Tags,
) -> Result<Plan> {
// Index all the already-downloaded wheels in the cache.
let mut registry_index = RegistryWheelIndex::new(cache, tags, index_locations, hasher);
let built_index = BuiltWheelIndex::new(cache, tags, hasher);
let built_index = BuiltWheelIndex::new(cache, tags, hasher, config_settings);

let mut cached = vec![];
let mut remote = vec![];
Expand Down
1 change: 1 addition & 0 deletions crates/uv/src/commands/pip/install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,7 @@ pub(crate) async fn pip_install(
link_mode,
compile,
&index_locations,
config_settings,
&hasher,
&markers,
&tags,
Expand Down
5 changes: 4 additions & 1 deletion crates/uv/src/commands/pip/operations.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,8 @@ use pypi_types::ResolverMarkerEnvironment;
use uv_cache::Cache;
use uv_client::{BaseClientBuilder, RegistryClient};
use uv_configuration::{
BuildOptions, Concurrency, Constraints, ExtrasSpecification, Overrides, Reinstall, Upgrade,
BuildOptions, Concurrency, ConfigSettings, Constraints, ExtrasSpecification, Overrides,
Reinstall, Upgrade,
};
use uv_dispatch::BuildDispatch;
use uv_distribution::DistributionDatabase;
Expand Down Expand Up @@ -349,6 +350,7 @@ pub(crate) async fn install(
link_mode: LinkMode,
compile: bool,
index_urls: &IndexLocations,
config_settings: &ConfigSettings,
hasher: &HashStrategy,
markers: &ResolverMarkerEnvironment,
tags: &Tags,
Expand Down Expand Up @@ -376,6 +378,7 @@ pub(crate) async fn install(
build_options,
hasher,
index_urls,
config_settings,
cache,
venv,
markers,
Expand Down
1 change: 1 addition & 0 deletions crates/uv/src/commands/pip/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,7 @@ pub(crate) async fn pip_sync(
link_mode,
compile,
&index_locations,
config_settings,
&hasher,
&markers,
&tags,
Expand Down
2 changes: 2 additions & 0 deletions crates/uv/src/commands/project/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -892,6 +892,7 @@ pub(crate) async fn sync_environment(
link_mode,
compile_bytecode,
index_locations,
config_setting,
&hasher,
&markers,
tags,
Expand Down Expand Up @@ -1125,6 +1126,7 @@ pub(crate) async fn update_environment(
*link_mode,
*compile_bytecode,
index_locations,
config_setting,
&hasher,
&markers,
tags,
Expand Down
1 change: 1 addition & 0 deletions crates/uv/src/commands/project/sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,7 @@ pub(super) async fn do_sync(
link_mode,
compile_bytecode,
index_locations,
config_setting,
&hasher,
&markers,
tags,
Expand Down
37 changes: 32 additions & 5 deletions crates/uv/tests/pip_install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2764,9 +2764,37 @@ fn config_settings() {
.join("__editable___setuptools_editable_0_1_0_finder.py");
assert!(finder.exists());

// Install the editable package with `--editable_mode=compat`.
let context = TestContext::new("3.12");
// Reinstalling with `--editable_mode=compat` should be a no-op; changes in build configuration
// don't invalidate the environment.
uv_snapshot!(context.filters(), context.pip_install()
.arg("-e")
.arg(context.workspace_root.join("scripts/packages/setuptools_editable"))
.arg("-C")
.arg("editable_mode=compat")
, @r###"
success: true
exit_code: 0
----- stdout -----

----- stderr -----
Audited 1 package in [TIME]
"###
);

// Uninstall the package.
uv_snapshot!(context.filters(), context.pip_uninstall()
.arg("setuptools-editable"), @r###"
success: true
exit_code: 0
----- stdout -----

----- stderr -----
Uninstalled 1 package in [TIME]
- setuptools-editable==0.1.0 (from file://[WORKSPACE]/scripts/packages/setuptools_editable)
"###);

// Install the editable package with `--editable_mode=compat`. We should ignore the cached
// build configuration and rebuild.
uv_snapshot!(context.filters(), context.pip_install()
.arg("-e")
.arg(context.workspace_root.join("scripts/packages/setuptools_editable"))
Expand All @@ -2779,9 +2807,8 @@ fn config_settings() {

----- stderr -----
Resolved 2 packages in [TIME]
Prepared 2 packages in [TIME]
Installed 2 packages in [TIME]
+ iniconfig==2.0.0
Prepared 1 package in [TIME]
Installed 1 package in [TIME]
+ setuptools-editable==0.1.0 (from file://[WORKSPACE]/scripts/packages/setuptools_editable)
"###
);
Expand Down
Loading