Skip to content

Commit 6ed6fc1

Browse files
konstinkonstin
authored
Build backend: Add direct builds to the resolver and installer (#9621)
This is like #9556, but at the level of all other builds, including the resolver and installer. Going through PEP 517 to build a package is slow, so when building a package with the uv build backend, we can call into the uv build backend directly instead: No temporary virtual env, no temp venv sync, no python subprocess calls, no uv subprocess calls. This fast path is gated through preview. Since the uv wheel is not available at test time, I've manually confirmed the feature by comparing `uv venv && cargo run pip install . -v --preview --reinstall .` and `uv venv && cargo run pip install . -v --reinstall .`. When hacking the preview so that the python uv build backend works without the setting the direct build also (wheel built with `maturin build --profile profiling`), we can see the perfomance difference: ``` $ hyperfine --prepare "uv venv" --warmup 3 \ "UV_PREVIEW=1 target/profiling/uv pip install --no-deps --reinstall scripts/packages/built-by-uv --preview" \ "target/profiling/uv pip install --no-deps --reinstall scripts/packages/built-by-uv --find-links target/wheels/" Benchmark 1: UV_PREVIEW=1 target/profiling/uv pip install --no-deps --reinstall scripts/packages/built-by-uv --preview Time (mean ± σ): 33.1 ms ± 2.5 ms [User: 25.7 ms, System: 13.0 ms] Range (min … max): 29.8 ms … 47.3 ms 73 runs Benchmark 2: target/profiling/uv pip install --no-deps --reinstall scripts/packages/built-by-uv --find-links target/wheels/ Time (mean ± σ): 115.1 ms ± 4.3 ms [User: 54.0 ms, System: 27.0 ms] Range (min … max): 109.2 ms … 123.8 ms 25 runs Summary UV_PREVIEW=1 target/profiling/uv pip install --no-deps --reinstall scripts/packages/built-by-uv --preview ran 3.48 ± 0.29 times faster than target/profiling/uv pip install --no-deps --reinstall scripts/packages/built-by-uv --find-links target/wheels/ ``` Do we need a global option to disable the fast path? There is one for `uv build` because `--force-pep517` moves `uv build` much closer to a `pip install` from source that a user of a library would experience (See discussion at #9610), but uv overall doesn't really make guarantees around the build env of dependencies, so I consider the direct build a valid option. Best reviewed commit-by-commit, only the last commit is the actual implementation, while the preview mode introduction is just a refactoring touching too many files.
1 parent 566c178 commit 6ed6fc1

File tree

29 files changed

+304
-86
lines changed

29 files changed

+304
-86
lines changed

‎Cargo.lock

+6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/uv-bench/benches/uv.rs

+2-1
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ mod resolver {
8787
use uv_client::RegistryClient;
8888
use uv_configuration::{
8989
BuildOptions, Concurrency, ConfigSettings, Constraints, IndexStrategy, LowerBound,
90-
SourceStrategy,
90+
PreviewMode, SourceStrategy,
9191
};
9292
use uv_dispatch::{BuildDispatch, SharedState};
9393
use uv_distribution::DistributionDatabase;
@@ -190,6 +190,7 @@ mod resolver {
190190
LowerBound::default(),
191191
sources,
192192
concurrency,
193+
PreviewMode::Enabled,
193194
);
194195

195196
let markers = if universal {

‎crates/uv-build-backend/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ uv-normalize = { workspace = true }
2020
uv-pep440 = { workspace = true }
2121
uv-pep508 = { workspace = true }
2222
uv-pypi-types = { workspace = true }
23+
uv-version = { workspace = true }
2324
uv-warnings = { workspace = true }
2425

2526
csv = { workspace = true }

‎crates/uv-build-backend/src/lib.rs

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ mod metadata;
22
mod source_dist;
33
mod wheel;
44

5-
pub use metadata::PyProjectToml;
5+
pub use metadata::{check_direct_build, PyProjectToml};
66
pub use source_dist::{build_source_dist, list_source_dist};
77
pub use wheel::{build_editable, build_wheel, list_wheel, metadata};
88

‎crates/uv-build-backend/src/metadata.rs

+36
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ use itertools::Itertools;
33
use serde::Deserialize;
44
use std::collections::{BTreeMap, Bound};
55
use std::ffi::OsStr;
6+
use std::fmt::Display;
67
use std::path::{Path, PathBuf};
78
use std::str::FromStr;
89
use tracing::{debug, trace};
@@ -50,6 +51,41 @@ pub enum ValidationError {
5051
InvalidSpdx(String, #[source] spdx::error::ParseError),
5152
}
5253

54+
/// Check if the build backend is matching the currently running uv version.
55+
pub fn check_direct_build(source_tree: &Path, name: impl Display) -> bool {
56+
let pyproject_toml: PyProjectToml =
57+
match fs_err::read_to_string(source_tree.join("pyproject.toml"))
58+
.map_err(|err| err.to_string())
59+
.and_then(|pyproject_toml| {
60+
toml::from_str(&pyproject_toml).map_err(|err| err.to_string())
61+
}) {
62+
Ok(pyproject_toml) => pyproject_toml,
63+
Err(err) => {
64+
debug!(
65+
"Not using uv build backend direct build of {name}, no pyproject.toml: {err}"
66+
);
67+
return false;
68+
}
69+
};
70+
match pyproject_toml
71+
.check_build_system(uv_version::version())
72+
.as_slice()
73+
{
74+
// No warnings -> match
75+
[] => true,
76+
// Any warning -> no match
77+
[first, others @ ..] => {
78+
debug!(
79+
"Not using uv build backend direct build of {name}, pyproject.toml does not match: {first}"
80+
);
81+
for other in others {
82+
trace!("Further uv build backend direct build of {name} mismatch: {other}");
83+
}
84+
false
85+
}
86+
}
87+
}
88+
5389
/// A `pyproject.toml` as specified in PEP 517.
5490
#[derive(Deserialize, Debug, Clone)]
5591
#[serde(

‎crates/uv-build-frontend/src/lib.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ impl SourceBuild {
251251
interpreter: &Interpreter,
252252
build_context: &impl BuildContext,
253253
source_build_context: SourceBuildContext,
254-
version_id: Option<String>,
254+
version_id: Option<&str>,
255255
locations: &IndexLocations,
256256
source_strategy: SourceStrategy,
257257
config_settings: ConfigSettings,
@@ -376,7 +376,7 @@ impl SourceBuild {
376376
build_context,
377377
package_name.as_ref(),
378378
package_version.as_ref(),
379-
version_id.as_deref(),
379+
version_id,
380380
locations,
381381
source_strategy,
382382
build_kind,
@@ -401,7 +401,7 @@ impl SourceBuild {
401401
metadata_directory: None,
402402
package_name,
403403
package_version,
404-
version_id,
404+
version_id: version_id.map(ToString::to_string),
405405
environment_variables,
406406
modified_path,
407407
runner,

‎crates/uv-dispatch/Cargo.toml

+4
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,13 @@ doctest = false
1717
workspace = true
1818

1919
[dependencies]
20+
uv-build-backend = { workspace = true }
2021
uv-build-frontend = { workspace = true }
2122
uv-cache = { workspace = true }
2223
uv-client = { workspace = true }
2324
uv-configuration = { workspace = true }
2425
uv-distribution = { workspace = true }
26+
uv-distribution-filename = { workspace = true }
2527
uv-distribution-types = { workspace = true }
2628
uv-git = { workspace = true }
2729
uv-install-wheel = { workspace = true }
@@ -30,9 +32,11 @@ uv-pypi-types = { workspace = true }
3032
uv-python = { workspace = true }
3133
uv-resolver = { workspace = true }
3234
uv-types = { workspace = true }
35+
uv-version = { workspace = true }
3336

3437
anyhow = { workspace = true }
3538
futures = { workspace = true }
3639
itertools = { workspace = true }
3740
rustc-hash = { workspace = true }
41+
tokio = { workspace = true }
3842
tracing = { workspace = true }

‎crates/uv-dispatch/src/lib.rs

+76-5
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,18 @@ use anyhow::{anyhow, Context, Result};
99
use futures::FutureExt;
1010
use itertools::Itertools;
1111
use rustc_hash::FxHashMap;
12-
use tracing::{debug, instrument};
13-
12+
use tracing::{debug, instrument, trace};
13+
use uv_build_backend::check_direct_build;
1414
use uv_build_frontend::{SourceBuild, SourceBuildContext};
1515
use uv_cache::Cache;
1616
use uv_client::RegistryClient;
1717
use uv_configuration::{
18-
BuildKind, BuildOptions, ConfigSettings, Constraints, IndexStrategy, LowerBound, Reinstall,
19-
SourceStrategy,
18+
BuildKind, BuildOptions, ConfigSettings, Constraints, IndexStrategy, LowerBound, PreviewMode,
19+
Reinstall, SourceStrategy,
2020
};
2121
use uv_configuration::{BuildOutput, Concurrency};
2222
use uv_distribution::DistributionDatabase;
23+
use uv_distribution_filename::DistFilename;
2324
use uv_distribution_types::{
2425
CachedDist, DependencyMetadata, IndexCapabilities, IndexLocations, Name, Resolution,
2526
SourceDist, VersionOrUrlRef,
@@ -57,6 +58,7 @@ pub struct BuildDispatch<'a> {
5758
bounds: LowerBound,
5859
sources: SourceStrategy,
5960
concurrency: Concurrency,
61+
preview: PreviewMode,
6062
}
6163

6264
impl<'a> BuildDispatch<'a> {
@@ -79,6 +81,7 @@ impl<'a> BuildDispatch<'a> {
7981
bounds: LowerBound,
8082
sources: SourceStrategy,
8183
concurrency: Concurrency,
84+
preview: PreviewMode,
8285
) -> Self {
8386
Self {
8487
client,
@@ -101,6 +104,7 @@ impl<'a> BuildDispatch<'a> {
101104
bounds,
102105
sources,
103106
concurrency,
107+
preview,
104108
}
105109
}
106110

@@ -340,7 +344,7 @@ impl<'a> BuildContext for BuildDispatch<'a> {
340344
source: &'data Path,
341345
subdirectory: Option<&'data Path>,
342346
install_path: &'data Path,
343-
version_id: Option<String>,
347+
version_id: Option<&'data str>,
344348
dist: Option<&'data SourceDist>,
345349
sources: SourceStrategy,
346350
build_kind: BuildKind,
@@ -394,6 +398,73 @@ impl<'a> BuildContext for BuildDispatch<'a> {
394398
.await?;
395399
Ok(builder)
396400
}
401+
402+
async fn direct_build<'data>(
403+
&'data self,
404+
source: &'data Path,
405+
subdirectory: Option<&'data Path>,
406+
output_dir: &'data Path,
407+
build_kind: BuildKind,
408+
version_id: Option<&'data str>,
409+
) -> Result<Option<DistFilename>> {
410+
// Direct builds are a preview feature with the uv build backend.
411+
if self.preview.is_disabled() {
412+
trace!("Preview is disabled, not checking for direct build");
413+
return Ok(None);
414+
}
415+
416+
let source_tree = if let Some(subdir) = subdirectory {
417+
source.join(subdir)
418+
} else {
419+
source.to_path_buf()
420+
};
421+
422+
// Only perform the direct build if the backend is uv in a compatible version.
423+
let source_tree_str = source_tree.display().to_string();
424+
let identifier = version_id.unwrap_or_else(|| &source_tree_str);
425+
if !check_direct_build(&source_tree, identifier) {
426+
trace!("Requirements for direct build not matched: {identifier}");
427+
return Ok(None);
428+
}
429+
430+
debug!("Performing direct build for {identifier}");
431+
432+
let output_dir = output_dir.to_path_buf();
433+
let filename = tokio::task::spawn_blocking(move || -> Result<_> {
434+
let filename = match build_kind {
435+
BuildKind::Wheel => {
436+
let wheel = uv_build_backend::build_wheel(
437+
&source_tree,
438+
&output_dir,
439+
None,
440+
uv_version::version(),
441+
)?;
442+
DistFilename::WheelFilename(wheel)
443+
}
444+
BuildKind::Sdist => {
445+
let source_dist = uv_build_backend::build_source_dist(
446+
&source_tree,
447+
&output_dir,
448+
uv_version::version(),
449+
)?;
450+
DistFilename::SourceDistFilename(source_dist)
451+
}
452+
BuildKind::Editable => {
453+
let wheel = uv_build_backend::build_editable(
454+
&source_tree,
455+
&output_dir,
456+
None,
457+
uv_version::version(),
458+
)?;
459+
DistFilename::WheelFilename(wheel)
460+
}
461+
};
462+
Ok(filename)
463+
})
464+
.await??;
465+
466+
Ok(Some(filename))
467+
}
397468
}
398469

399470
/// Shared state used during resolution and installation.

‎crates/uv-distribution/src/source/mod.rs

+31-11
Original file line numberDiff line numberDiff line change
@@ -1803,27 +1803,47 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
18031803
fs::create_dir_all(&cache_shard)
18041804
.await
18051805
.map_err(Error::CacheWrite)?;
1806-
let disk_filename = self
1806+
// Try a direct build if that isn't disabled and the uv build backend is used.
1807+
let disk_filename = if let Some(name) = self
18071808
.build_context
1808-
.setup_build(
1809+
.direct_build(
18091810
source_root,
18101811
subdirectory,
1811-
source_root,
1812-
Some(source.to_string()),
1813-
source.as_dist(),
1814-
source_strategy,
1812+
temp_dir.path(),
18151813
if source.is_editable() {
18161814
BuildKind::Editable
18171815
} else {
18181816
BuildKind::Wheel
18191817
},
1820-
BuildOutput::Debug,
1818+
Some(&source.to_string()),
18211819
)
18221820
.await
18231821
.map_err(Error::Build)?
1824-
.wheel(temp_dir.path())
1825-
.await
1826-
.map_err(Error::Build)?;
1822+
{
1823+
// In the uv build backend, the normalized filename and the disk filename are the same.
1824+
name.to_string()
1825+
} else {
1826+
self.build_context
1827+
.setup_build(
1828+
source_root,
1829+
subdirectory,
1830+
source_root,
1831+
Some(&source.to_string()),
1832+
source.as_dist(),
1833+
source_strategy,
1834+
if source.is_editable() {
1835+
BuildKind::Editable
1836+
} else {
1837+
BuildKind::Wheel
1838+
},
1839+
BuildOutput::Debug,
1840+
)
1841+
.await
1842+
.map_err(Error::Build)?
1843+
.wheel(temp_dir.path())
1844+
.await
1845+
.map_err(Error::Build)?
1846+
};
18271847

18281848
// Read the metadata from the wheel.
18291849
let filename = WheelFilename::from_str(&disk_filename)?;
@@ -1884,7 +1904,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> {
18841904
source_root,
18851905
subdirectory,
18861906
source_root,
1887-
Some(source.to_string()),
1907+
Some(&source.to_string()),
18881908
source.as_dist(),
18891909
source_strategy,
18901910
if source.is_editable() {

‎crates/uv-types/Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ workspace = true
1818
[dependencies]
1919
uv-cache = { workspace = true }
2020
uv-configuration = { workspace = true }
21+
uv-distribution-filename = { workspace = true }
2122
uv-distribution-types = { workspace = true }
2223
uv-git = { workspace = true }
2324
uv-normalize = { workspace = true }

0 commit comments

Comments
 (0)