From f67347e72cda8e6a46a09941f0ac67063ba2cf84 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 30 Sep 2024 17:16:44 -0400 Subject: [PATCH] Allow multiple source entries for each package in `tool.uv.sources` (#7745) ## Summary This PR enables users to provide multiple source entries in `tool.uv.sources`, e.g.: ```toml [tool.uv.sources] httpx = [ { git = "https://github.com/encode/httpx", tag = "0.27.2", marker = "sys_platform == 'darwin'" }, { git = "https://github.com/encode/httpx", tag = "0.24.1", marker = "sys_platform == 'linux'" }, ] ``` The implementation is relatively straightforward: when we lower the requirement, we now return an iterator rather than a single requirement. In other words, the above is transformed into two requirements: ```txt httpx @ git+https://github.com/encode/httpx@0.27.2 ; sys_platform == 'darwin' httpx @ git+https://github.com/encode/httpx@0.24.1 ; sys_platform == 'linux' ``` We verify (at deserialization time) that the markers are non-overlapping. Closes https://github.com/astral-sh/uv/issues/3397. --- Cargo.lock | 2 + crates/pep508-rs/src/marker/tree.rs | 25 +- crates/uv-distribution/Cargo.toml | 1 + .../uv-distribution/src/metadata/lowering.rs | 442 ++++++++++------- crates/uv-distribution/src/metadata/mod.rs | 2 + .../src/metadata/requires_dist.rs | 37 +- crates/uv-scripts/src/lib.rs | 4 +- crates/uv-workspace/Cargo.toml | 3 +- crates/uv-workspace/src/pyproject.rs | 172 ++++++- crates/uv-workspace/src/workspace.rs | 24 +- crates/uv/src/commands/project/add.rs | 2 + crates/uv/src/commands/project/lock.rs | 6 +- crates/uv/src/commands/project/run.rs | 4 +- crates/uv/src/commands/project/sync.rs | 4 +- crates/uv/tests/lock.rs | 465 ++++++++++++++++++ docs/concepts/dependencies.md | 40 ++ uv.schema.json | 40 +- 17 files changed, 1065 insertions(+), 208 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dccf26845d00..cc78efaecc45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4789,6 +4789,7 @@ dependencies = [ "cache-key", "distribution-filename", "distribution-types", + "either", "fs-err", "futures", "indoc", @@ -5319,6 +5320,7 @@ dependencies = [ "glob", "insta", "itertools 0.13.0", + "owo-colors", "pep440_rs", "pep508_rs", "pypi-types", diff --git a/crates/pep508-rs/src/marker/tree.rs b/crates/pep508-rs/src/marker/tree.rs index 550c85acb0c0..b6fc58df0030 100644 --- a/crates/pep508-rs/src/marker/tree.rs +++ b/crates/pep508-rs/src/marker/tree.rs @@ -5,12 +5,11 @@ use std::ops::{Bound, Deref}; use std::str::FromStr; use itertools::Itertools; +use pep440_rs::{Version, VersionParseError, VersionSpecifier}; use pubgrub::Range; #[cfg(feature = "pyo3")] use pyo3::{basic::CompareOp, pyclass, pymethods}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; - -use pep440_rs::{Version, VersionParseError, VersionSpecifier}; use uv_normalize::ExtraName; use crate::cursor::Cursor; @@ -1667,6 +1666,28 @@ impl Display for MarkerTreeContents { } } +#[cfg(feature = "schemars")] +impl schemars::JsonSchema for MarkerTree { + fn schema_name() -> String { + "MarkerTree".to_string() + } + + fn json_schema(_gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { + schemars::schema::SchemaObject { + instance_type: Some(schemars::schema::InstanceType::String.into()), + metadata: Some(Box::new(schemars::schema::Metadata { + description: Some( + "A PEP 508-compliant marker expression, e.g., `sys_platform == 'Darwin'`" + .to_string(), + ), + ..schemars::schema::Metadata::default() + })), + ..schemars::schema::SchemaObject::default() + } + .into() + } +} + #[cfg(test)] mod test { use std::ops::Bound; diff --git a/crates/uv-distribution/Cargo.toml b/crates/uv-distribution/Cargo.toml index 1845a60951fd..868673c10e95 100644 --- a/crates/uv-distribution/Cargo.toml +++ b/crates/uv-distribution/Cargo.toml @@ -34,6 +34,7 @@ uv-warnings = { workspace = true } uv-workspace = { workspace = true } anyhow = { workspace = true } +either = { workspace = true } fs-err = { workspace = true } futures = { workspace = true } nanoid = { workspace = true } diff --git a/crates/uv-distribution/src/metadata/lowering.rs b/crates/uv-distribution/src/metadata/lowering.rs index 378277546222..564cd6f82f9f 100644 --- a/crates/uv-distribution/src/metadata/lowering.rs +++ b/crates/uv-distribution/src/metadata/lowering.rs @@ -1,18 +1,18 @@ +use either::Either; use std::collections::BTreeMap; use std::io; use std::path::{Path, PathBuf}; - use thiserror::Error; use url::Url; use distribution_filename::DistExtension; use pep440_rs::VersionSpecifiers; -use pep508_rs::{VerbatimUrl, VersionOrUrl}; +use pep508_rs::{MarkerTree, VerbatimUrl, VersionOrUrl}; use pypi_types::{ParsedUrlError, Requirement, RequirementSource, VerbatimParsedUrl}; use uv_git::GitReference; use uv_normalize::PackageName; use uv_warnings::warn_user_once; -use uv_workspace::pyproject::{PyProjectToml, Source}; +use uv_workspace::pyproject::{PyProjectToml, Source, Sources}; use uv_workspace::Workspace; #[derive(Debug, Clone)] @@ -28,13 +28,13 @@ enum Origin { impl LoweredRequirement { /// Combine `project.dependencies` or `project.optional-dependencies` with `tool.uv.sources`. - pub(crate) fn from_requirement( + pub(crate) fn from_requirement<'data>( requirement: pep508_rs::Requirement, - project_name: &PackageName, - project_dir: &Path, - project_sources: &BTreeMap, - workspace: &Workspace, - ) -> Result { + project_name: &'data PackageName, + project_dir: &'data Path, + project_sources: &'data BTreeMap, + workspace: &'data Workspace, + ) -> impl Iterator> + 'data { let (source, origin) = if let Some(source) = project_sources.get(&requirement.name) { (Some(source), Origin::Project) } else if let Some(source) = workspace.sources().get(&requirement.name) { @@ -48,18 +48,16 @@ impl LoweredRequirement { // We require that when you use a package that's part of the workspace, ... !workspace.packages().contains_key(&requirement.name) // ... it must be declared as a workspace dependency (`workspace = true`), ... - || matches!( - source, - Some(Source::Workspace { - // By using toml, we technically support `workspace = false`. - workspace: true - }) - ) + || source.as_ref().filter(|sources| !sources.is_empty()).is_some_and(|source| source.iter().all(|source| { + matches!(source, Source::Workspace { workspace: true, .. }) + })) // ... except for recursive self-inclusion (extras that activate other extras), e.g. // `framework[machine_learning]` depends on `framework[cuda]`. || &requirement.name == project_name; if !workspace_package_declared { - return Err(LoweringError::UndeclaredWorkspacePackage); + return Either::Left(std::iter::once(Err( + LoweringError::UndeclaredWorkspacePackage, + ))); } let Some(source) = source else { @@ -74,172 +72,282 @@ impl LoweredRequirement { requirement.name ); } - return Ok(Self(Requirement::from(requirement))); + return Either::Left(std::iter::once(Ok(Self(Requirement::from(requirement))))); }; - let source = match source { - Source::Git { - git, - subdirectory, - rev, - tag, - branch, - } => { - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - git_source(&git, subdirectory.map(PathBuf::from), rev, tag, branch)? - } - Source::Url { url, subdirectory } => { - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - url_source(url, subdirectory.map(PathBuf::from))? - } - Source::Path { path, editable } => { - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - path_source( - PathBuf::from(path), - origin, - project_dir, - workspace.install_path(), - editable.unwrap_or(false), - )? - } - Source::Registry { index } => registry_source(&requirement, index)?, - Source::Workspace { - workspace: is_workspace, - } => { - if !is_workspace { - return Err(LoweringError::WorkspaceFalse); - } - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - let member = workspace - .packages() - .get(&requirement.name) - .ok_or(LoweringError::UndeclaredWorkspacePackage)? - .clone(); - - // Say we have: - // ``` - // root - // ├── main_workspace <- We want to the path from here ... - // │ ├── pyproject.toml - // │ └── uv.lock - // └──current_workspace - // └── packages - // └── current_package <- ... to here. - // └── pyproject.toml - // ``` - // The path we need in the lockfile: `../current_workspace/packages/current_project` - // member root: `/root/current_workspace/packages/current_project` - // workspace install root: `/root/current_workspace` - // relative to workspace: `packages/current_project` - // workspace lock root: `../current_workspace` - // relative to main workspace: `../current_workspace/packages/current_project` - let url = VerbatimUrl::from_absolute_path(member.root())?; - let install_path = url.to_file_path().map_err(|()| { - LoweringError::RelativeTo(io::Error::new( - io::ErrorKind::Other, - "Invalid path in file URL", - )) - })?; - - if member.pyproject_toml().is_package() { - RequirementSource::Directory { - install_path, - url, - editable: true, - r#virtual: false, - } - } else { - RequirementSource::Directory { - install_path, - url, - editable: false, - r#virtual: true, - } - } - } - Source::CatchAll { .. } => { - // Emit a dedicated error message, which is an improvement over Serde's default error. - return Err(LoweringError::InvalidEntry); + // Determine whether the markers cover the full space for the requirement. If not, fill the + // remaining space with the negation of the sources. + let remaining = { + // Determine the space covered by the sources. + let mut total = MarkerTree::FALSE; + for source in source.iter() { + total.or(source.marker()); } + + // Determine the space covered by the requirement. + let mut remaining = total.negate(); + remaining.and(requirement.marker.clone()); + + LoweredRequirement(Requirement { + marker: remaining, + ..Requirement::from(requirement.clone()) + }) }; - Ok(Self(Requirement { - name: requirement.name, - extras: requirement.extras, - marker: requirement.marker, - source, - origin: requirement.origin, - })) + + Either::Right( + source + .into_iter() + .map(move |source| { + let (source, mut marker) = match source { + Source::Git { + git, + subdirectory, + rev, + tag, + branch, + marker, + } => { + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let source = git_source( + &git, + subdirectory.map(PathBuf::from), + rev, + tag, + branch, + )?; + (source, marker) + } + Source::Url { + url, + subdirectory, + marker, + } => { + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let source = url_source(url, subdirectory.map(PathBuf::from))?; + (source, marker) + } + Source::Path { + path, + editable, + marker, + } => { + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let source = path_source( + PathBuf::from(path), + origin, + project_dir, + workspace.install_path(), + editable.unwrap_or(false), + )?; + (source, marker) + } + Source::Registry { index, marker } => { + let source = registry_source(&requirement, index)?; + (source, marker) + } + Source::Workspace { + workspace: is_workspace, + marker, + } => { + if !is_workspace { + return Err(LoweringError::WorkspaceFalse); + } + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let member = workspace + .packages() + .get(&requirement.name) + .ok_or(LoweringError::UndeclaredWorkspacePackage)? + .clone(); + + // Say we have: + // ``` + // root + // ├── main_workspace <- We want to the path from here ... + // │ ├── pyproject.toml + // │ └── uv.lock + // └──current_workspace + // └── packages + // └── current_package <- ... to here. + // └── pyproject.toml + // ``` + // The path we need in the lockfile: `../current_workspace/packages/current_project` + // member root: `/root/current_workspace/packages/current_project` + // workspace install root: `/root/current_workspace` + // relative to workspace: `packages/current_project` + // workspace lock root: `../current_workspace` + // relative to main workspace: `../current_workspace/packages/current_project` + let url = VerbatimUrl::from_absolute_path(member.root())?; + let install_path = url.to_file_path().map_err(|()| { + LoweringError::RelativeTo(io::Error::new( + io::ErrorKind::Other, + "Invalid path in file URL", + )) + })?; + + let source = if member.pyproject_toml().is_package() { + RequirementSource::Directory { + install_path, + url, + editable: true, + r#virtual: false, + } + } else { + RequirementSource::Directory { + install_path, + url, + editable: false, + r#virtual: true, + } + }; + (source, marker) + } + Source::CatchAll { .. } => { + // Emit a dedicated error message, which is an improvement over Serde's default error. + return Err(LoweringError::InvalidEntry); + } + }; + + marker.and(requirement.marker.clone()); + + Ok(Self(Requirement { + name: requirement.name.clone(), + extras: requirement.extras.clone(), + marker, + source, + origin: requirement.origin.clone(), + })) + }) + .chain(std::iter::once(Ok(remaining))) + .filter(|requirement| match requirement { + Ok(requirement) => !requirement.0.marker.is_false(), + Err(_) => true, + }), + ) } /// Lower a [`pep508_rs::Requirement`] in a non-workspace setting (for example, in a PEP 723 /// script, which runs in an isolated context). - pub fn from_non_workspace_requirement( + pub fn from_non_workspace_requirement<'data>( requirement: pep508_rs::Requirement, - dir: &Path, - sources: &BTreeMap, - ) -> Result { + dir: &'data Path, + sources: &'data BTreeMap, + ) -> impl Iterator> + 'data { let source = sources.get(&requirement.name).cloned(); let Some(source) = source else { - return Ok(Self(Requirement::from(requirement))); + return Either::Left(std::iter::once(Ok(Self(Requirement::from(requirement))))); }; - let source = match source { - Source::Git { - git, - subdirectory, - rev, - tag, - branch, - } => { - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - git_source(&git, subdirectory.map(PathBuf::from), rev, tag, branch)? - } - Source::Url { url, subdirectory } => { - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - url_source(url, subdirectory.map(PathBuf::from))? - } - Source::Path { path, editable } => { - if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { - return Err(LoweringError::ConflictingUrls); - } - path_source( - PathBuf::from(path), - Origin::Project, - dir, - dir, - editable.unwrap_or(false), - )? - } - Source::Registry { index } => registry_source(&requirement, index)?, - Source::Workspace { .. } => { - return Err(LoweringError::WorkspaceMember); - } - Source::CatchAll { .. } => { - // Emit a dedicated error message, which is an improvement over Serde's default - // error. - return Err(LoweringError::InvalidEntry); + // Determine whether the markers cover the full space for the requirement. If not, fill the + // remaining space with the negation of the sources. + let remaining = { + // Determine the space covered by the sources. + let mut total = MarkerTree::FALSE; + for source in source.iter() { + total.or(source.marker()); } + + // Determine the space covered by the requirement. + let mut remaining = total.negate(); + remaining.and(requirement.marker.clone()); + + LoweredRequirement(Requirement { + marker: remaining, + ..Requirement::from(requirement.clone()) + }) }; - Ok(Self(Requirement { - name: requirement.name, - extras: requirement.extras, - marker: requirement.marker, - source, - origin: requirement.origin, - })) + + Either::Right( + source + .into_iter() + .map(move |source| { + let (source, mut marker) = match source { + Source::Git { + git, + subdirectory, + rev, + tag, + branch, + marker, + } => { + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let source = git_source( + &git, + subdirectory.map(PathBuf::from), + rev, + tag, + branch, + )?; + (source, marker) + } + Source::Url { + url, + subdirectory, + marker, + } => { + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let source = url_source(url, subdirectory.map(PathBuf::from))?; + (source, marker) + } + Source::Path { + path, + editable, + marker, + } => { + if matches!(requirement.version_or_url, Some(VersionOrUrl::Url(_))) { + return Err(LoweringError::ConflictingUrls); + } + let source = path_source( + PathBuf::from(path), + Origin::Project, + dir, + dir, + editable.unwrap_or(false), + )?; + (source, marker) + } + Source::Registry { index, marker } => { + let source = registry_source(&requirement, index)?; + (source, marker) + } + Source::Workspace { .. } => { + return Err(LoweringError::WorkspaceMember); + } + Source::CatchAll { .. } => { + // Emit a dedicated error message, which is an improvement over Serde's default + // error. + return Err(LoweringError::InvalidEntry); + } + }; + + marker.and(requirement.marker.clone()); + + Ok(Self(Requirement { + name: requirement.name.clone(), + extras: requirement.extras.clone(), + marker, + source, + origin: requirement.origin.clone(), + })) + }) + .chain(std::iter::once(Ok(remaining))) + .filter(|requirement| match requirement { + Ok(requirement) => !requirement.0.marker.is_false(), + Err(_) => true, + }), + ) } /// Convert back into a [`Requirement`]. diff --git a/crates/uv-distribution/src/metadata/mod.rs b/crates/uv-distribution/src/metadata/mod.rs index 5294c3fe1fa4..ceefd4738a43 100644 --- a/crates/uv-distribution/src/metadata/mod.rs +++ b/crates/uv-distribution/src/metadata/mod.rs @@ -22,6 +22,8 @@ pub enum MetadataError { Workspace(#[from] WorkspaceError), #[error("Failed to parse entry for: `{0}`")] LoweringError(PackageName, #[source] LoweringError), + #[error(transparent)] + Lower(#[from] LoweringError), } #[derive(Debug, Clone)] diff --git a/crates/uv-distribution/src/metadata/requires_dist.rs b/crates/uv-distribution/src/metadata/requires_dist.rs index c2b89103e74d..928e99ab8e8e 100644 --- a/crates/uv-distribution/src/metadata/requires_dist.rs +++ b/crates/uv-distribution/src/metadata/requires_dist.rs @@ -1,6 +1,6 @@ use crate::metadata::{LoweredRequirement, MetadataError}; use crate::Metadata; -use pypi_types::Requirement; + use std::collections::BTreeMap; use std::path::Path; use uv_configuration::SourceStrategy; @@ -84,7 +84,7 @@ impl RequiresDist { .cloned(); let dev_dependencies = match source_strategy { SourceStrategy::Enabled => dev_dependencies - .map(|requirement| { + .flat_map(|requirement| { let requirement_name = requirement.name.clone(); LoweredRequirement::from_requirement( requirement, @@ -93,13 +93,17 @@ impl RequiresDist { sources, project_workspace.workspace(), ) - .map(LoweredRequirement::into_inner) - .map_err(|err| MetadataError::LoweringError(requirement_name.clone(), err)) + .map(move |requirement| match requirement { + Ok(requirement) => Ok(requirement.into_inner()), + Err(err) => { + Err(MetadataError::LoweringError(requirement_name.clone(), err)) + } + }) }) .collect::, _>>()?, SourceStrategy::Disabled => dev_dependencies .into_iter() - .map(Requirement::from) + .map(pypi_types::Requirement::from) .collect(), }; if dev_dependencies.is_empty() { @@ -112,7 +116,7 @@ impl RequiresDist { let requires_dist = metadata.requires_dist.into_iter(); let requires_dist = match source_strategy { SourceStrategy::Enabled => requires_dist - .map(|requirement| { + .flat_map(|requirement| { let requirement_name = requirement.name.clone(); LoweredRequirement::from_requirement( requirement, @@ -121,11 +125,18 @@ impl RequiresDist { sources, project_workspace.workspace(), ) - .map(LoweredRequirement::into_inner) - .map_err(|err| MetadataError::LoweringError(requirement_name.clone(), err)) + .map(move |requirement| match requirement { + Ok(requirement) => Ok(requirement.into_inner()), + Err(err) => { + Err(MetadataError::LoweringError(requirement_name.clone(), err)) + } + }) }) - .collect::>()?, - SourceStrategy::Disabled => requires_dist.into_iter().map(Requirement::from).collect(), + .collect::, _>>()?, + SourceStrategy::Disabled => requires_dist + .into_iter() + .map(pypi_types::Requirement::from) + .collect(), }; Ok(Self { @@ -253,7 +264,7 @@ mod test { | 8 | tqdm = { git = "https://github.com/tqdm/tqdm", ref = "baaaaaab" } | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - data did not match any variant of untagged enum Source + data did not match any variant of untagged enum SourcesWire "###); } @@ -277,7 +288,7 @@ mod test { | 8 | tqdm = { path = "tqdm", index = "torch" } | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - data did not match any variant of untagged enum Source + data did not match any variant of untagged enum SourcesWire "###); } @@ -337,7 +348,7 @@ mod test { | 8 | tqdm = { url = "§invalid#+#*Ä" } | ^^^^^^^^^^^^^^^^^^^^^^^^^^^ - data did not match any variant of untagged enum Source + data did not match any variant of untagged enum SourcesWire "###); } diff --git a/crates/uv-scripts/src/lib.rs b/crates/uv-scripts/src/lib.rs index f257eb3f71c8..9baf174acf83 100644 --- a/crates/uv-scripts/src/lib.rs +++ b/crates/uv-scripts/src/lib.rs @@ -12,7 +12,7 @@ use pep440_rs::VersionSpecifiers; use pep508_rs::PackageName; use pypi_types::VerbatimParsedUrl; use uv_settings::{GlobalOptions, ResolverInstallerOptions}; -use uv_workspace::pyproject::Source; +use uv_workspace::pyproject::Sources; static FINDER: LazyLock = LazyLock::new(|| Finder::new(b"# /// script")); @@ -193,7 +193,7 @@ pub struct ToolUv { pub globals: GlobalOptions, #[serde(flatten)] pub top_level: ResolverInstallerOptions, - pub sources: Option>, + pub sources: Option>, } #[derive(Debug, Error)] diff --git a/crates/uv-workspace/Cargo.toml b/crates/uv-workspace/Cargo.toml index d77df0f98e61..b04eb185cd9a 100644 --- a/crates/uv-workspace/Cargo.toml +++ b/crates/uv-workspace/Cargo.toml @@ -26,6 +26,8 @@ uv-options-metadata = { workspace = true } either = { workspace = true } fs-err = { workspace = true } glob = { workspace = true } +itertools = { workspace = true } +owo-colors = { workspace = true } rustc-hash = { workspace = true } same-file = { workspace = true } schemars = { workspace = true, optional = true } @@ -36,7 +38,6 @@ toml = { workspace = true } toml_edit = { workspace = true } tracing = { workspace = true } url = { workspace = true } -itertools = { workspace = true } [dev-dependencies] anyhow = { workspace = true } diff --git a/crates/uv-workspace/src/pyproject.rs b/crates/uv-workspace/src/pyproject.rs index 4a212e21b095..f873b6451345 100644 --- a/crates/uv-workspace/src/pyproject.rs +++ b/crates/uv-workspace/src/pyproject.rs @@ -7,6 +7,7 @@ //! Then lowers them into a dependency specification. use glob::Pattern; +use owo_colors::OwoColorize; use serde::{de::IntoDeserializer, Deserialize, Serialize}; use std::ops::Deref; use std::path::{Path, PathBuf}; @@ -16,6 +17,7 @@ use thiserror::Error; use url::Url; use pep440_rs::{Version, VersionSpecifiers}; +use pep508_rs::MarkerTree; use pypi_types::{RequirementSource, SupportedEnvironments, VerbatimParsedUrl}; use uv_fs::{relative_to, PortablePathBuf}; use uv_git::GitReference; @@ -294,17 +296,17 @@ pub struct ToolUv { #[derive(Default, Debug, Clone, PartialEq, Eq)] #[cfg_attr(test, derive(Serialize))] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] -pub struct ToolUvSources(BTreeMap); +pub struct ToolUvSources(BTreeMap); impl ToolUvSources { /// Returns the underlying `BTreeMap` of package names to sources. - pub fn inner(&self) -> &BTreeMap { + pub fn inner(&self) -> &BTreeMap { &self.0 } /// Convert the [`ToolUvSources`] into its inner `BTreeMap`. #[must_use] - pub fn into_inner(self) -> BTreeMap { + pub fn into_inner(self) -> BTreeMap { self.0 } } @@ -329,7 +331,7 @@ impl<'de> serde::de::Deserialize<'de> for ToolUvSources { M: serde::de::MapAccess<'de>, { let mut sources = BTreeMap::new(); - while let Some((key, value)) = access.next_entry::()? { + while let Some((key, value)) = access.next_entry::()? { match sources.entry(key) { std::collections::btree_map::Entry::Occupied(entry) => { return Err(serde::de::Error::custom(format!( @@ -407,6 +409,105 @@ impl Deref for SerdePattern { } } +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[serde(rename_all = "kebab-case", try_from = "SourcesWire")] +pub struct Sources(#[cfg_attr(feature = "schemars", schemars(with = "SourcesWire"))] Vec); + +impl Sources { + /// Return an [`Iterator`] over the sources. + /// + /// If the iterator contains multiple entries, they will always use disjoint markers. + /// + /// The iterator will contain at most one registry source. + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + /// Returns `true` if the sources list is empty. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Returns the number of sources in the list. + pub fn len(&self) -> usize { + self.0.len() + } +} + +impl IntoIterator for Sources { + type Item = Source; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "kebab-case", untagged)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +#[allow(clippy::large_enum_variant)] +enum SourcesWire { + One(Source), + Many(Vec), +} + +impl TryFrom for Sources { + type Error = SourceError; + + fn try_from(wire: SourcesWire) -> Result { + match wire { + SourcesWire::One(source) => Ok(Self(vec![source])), + SourcesWire::Many(sources) => { + // Ensure that the markers are disjoint. + for (lhs, rhs) in sources + .iter() + .map(Source::marker) + .zip(sources.iter().skip(1).map(Source::marker)) + { + if !lhs.is_disjoint(&rhs) { + let mut hint = lhs.negate(); + hint.and(rhs.clone()); + + let lhs = lhs + .contents() + .map(|contents| contents.to_string()) + .unwrap_or_else(|| "true".to_string()); + let rhs = rhs + .contents() + .map(|contents| contents.to_string()) + .unwrap_or_else(|| "true".to_string()); + let hint = hint + .contents() + .map(|contents| contents.to_string()) + .unwrap_or_else(|| "true".to_string()); + + return Err(SourceError::OverlappingMarkers(lhs, rhs, hint)); + } + } + + // Ensure that there is at least one source. + if sources.is_empty() { + return Err(SourceError::EmptySources); + } + + // Ensure that there is at most one registry source. + if sources + .iter() + .filter(|source| matches!(source, Source::Registry { .. })) + .nth(1) + .is_some() + { + return Err(SourceError::MultipleIndexes); + } + + Ok(Self(sources)) + } + } + } +} + /// A `tool.uv.sources` value. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] @@ -427,6 +528,12 @@ pub enum Source { rev: Option, tag: Option, branch: Option, + #[serde( + skip_serializing_if = "pep508_rs::marker::ser::is_empty", + serialize_with = "pep508_rs::marker::ser::serialize", + default + )] + marker: MarkerTree, }, /// A remote `http://` or `https://` URL, either a wheel (`.whl`) or a source distribution /// (`.zip`, `.tar.gz`). @@ -440,6 +547,12 @@ pub enum Source { /// For source distributions, the path to the directory with the `pyproject.toml`, if it's /// not in the archive root. subdirectory: Option, + #[serde( + skip_serializing_if = "pep508_rs::marker::ser::is_empty", + serialize_with = "pep508_rs::marker::ser::serialize", + default + )] + marker: MarkerTree, }, /// The path to a dependency, either a wheel (a `.whl` file), source distribution (a `.zip` or /// `.tar.gz` file), or source tree (i.e., a directory containing a `pyproject.toml` or @@ -448,17 +561,35 @@ pub enum Source { path: PortablePathBuf, /// `false` by default. editable: Option, + #[serde( + skip_serializing_if = "pep508_rs::marker::ser::is_empty", + serialize_with = "pep508_rs::marker::ser::serialize", + default + )] + marker: MarkerTree, }, /// A dependency pinned to a specific index, e.g., `torch` after setting `torch` to `https://download.pytorch.org/whl/cu118`. Registry { // TODO(konstin): The string is more-or-less a placeholder index: String, + #[serde( + skip_serializing_if = "pep508_rs::marker::ser::is_empty", + serialize_with = "pep508_rs::marker::ser::serialize", + default + )] + marker: MarkerTree, }, /// A dependency on another package in the workspace. Workspace { /// When set to `false`, the package will be fetched from the remote index, rather than /// included as a workspace package. workspace: bool, + #[serde( + skip_serializing_if = "pep508_rs::marker::ser::is_empty", + serialize_with = "pep508_rs::marker::ser::serialize", + default + )] + marker: MarkerTree, }, /// A catch-all variant used to emit precise error messages when deserializing. CatchAll { @@ -471,6 +602,12 @@ pub enum Source { path: PortablePathBuf, index: String, workspace: bool, + #[serde( + skip_serializing_if = "pep508_rs::marker::ser::is_empty", + serialize_with = "pep508_rs::marker::ser::serialize", + default + )] + marker: MarkerTree, }, } @@ -494,6 +631,12 @@ pub enum SourceError { Absolute(#[from] std::io::Error), #[error("Path contains invalid characters: `{}`", _0.display())] NonUtf8Path(PathBuf), + #[error("Source markers must be disjoint, but the following markers overlap: `{0}` and `{1}`.\n\n{hint}{colon} replace `{1}` with `{2}`.", hint = "hint".bold().cyan(), colon = ":".bold())] + OverlappingMarkers(String, String, String), + #[error("Must provide at least one source")] + EmptySources, + #[error("Sources can only include a single index source")] + MultipleIndexes, } impl Source { @@ -524,7 +667,10 @@ impl Source { if workspace { return match source { RequirementSource::Registry { .. } | RequirementSource::Directory { .. } => { - Ok(Some(Source::Workspace { workspace: true })) + Ok(Some(Source::Workspace { + workspace: true, + marker: MarkerTree::TRUE, + })) } RequirementSource::Url { .. } => { Err(SourceError::WorkspacePackageUrl(name.to_string())) @@ -548,12 +694,14 @@ impl Source { .or_else(|_| std::path::absolute(&install_path)) .map_err(SourceError::Absolute)?, ), + marker: MarkerTree::TRUE, }, RequirementSource::Url { subdirectory, url, .. } => Source::Url { url: url.to_url(), subdirectory: subdirectory.map(PortablePathBuf::from), + marker: MarkerTree::TRUE, }, RequirementSource::Git { repository, @@ -578,6 +726,7 @@ impl Source { branch, git: repository, subdirectory: subdirectory.map(PortablePathBuf::from), + marker: MarkerTree::TRUE, } } else { Source::Git { @@ -586,6 +735,7 @@ impl Source { branch, git: repository, subdirectory: subdirectory.map(PortablePathBuf::from), + marker: MarkerTree::TRUE, } } } @@ -593,6 +743,18 @@ impl Source { Ok(Some(source)) } + + /// Return the [`MarkerTree`] for the source. + pub fn marker(&self) -> MarkerTree { + match self { + Source::Git { marker, .. } => marker.clone(), + Source::Url { marker, .. } => marker.clone(), + Source::Path { marker, .. } => marker.clone(), + Source::Registry { marker, .. } => marker.clone(), + Source::Workspace { marker, .. } => marker.clone(), + Source::CatchAll { marker, .. } => marker.clone(), + } + } } /// The type of a dependency in a `pyproject.toml`. diff --git a/crates/uv-workspace/src/workspace.rs b/crates/uv-workspace/src/workspace.rs index 19fbc9baca90..b135cbdd0e10 100644 --- a/crates/uv-workspace/src/workspace.rs +++ b/crates/uv-workspace/src/workspace.rs @@ -14,7 +14,7 @@ use uv_normalize::{GroupName, PackageName, DEV_DEPENDENCIES}; use uv_warnings::{warn_user, warn_user_once}; use crate::pyproject::{ - Project, PyProjectToml, PyprojectTomlError, Source, ToolUvSources, ToolUvWorkspace, + Project, PyProjectToml, PyprojectTomlError, Source, Sources, ToolUvSources, ToolUvWorkspace, }; #[derive(thiserror::Error, Debug)] @@ -78,7 +78,7 @@ pub struct Workspace { /// The sources table from the workspace `pyproject.toml`. /// /// This table is overridden by the project sources. - sources: BTreeMap, + sources: BTreeMap, /// The `pyproject.toml` of the workspace root. pyproject_toml: PyProjectToml, } @@ -517,7 +517,7 @@ impl Workspace { } /// The sources table from the workspace `pyproject.toml`. - pub fn sources(&self) -> &BTreeMap { + pub fn sources(&self) -> &BTreeMap { &self.sources } @@ -531,7 +531,7 @@ impl Workspace { .as_ref() .and_then(|uv| uv.sources.as_ref()) .map(ToolUvSources::inner) - .map(|sources| sources.values()) + .map(|sources| sources.values().flat_map(Sources::iter)) }) }) .flatten() @@ -1755,9 +1755,11 @@ mod tests { } }, "sources": { - "bird-feeder": { - "workspace": true - } + "bird-feeder": [ + { + "workspace": true + } + ] }, "pyproject_toml": { "project": { @@ -1773,9 +1775,11 @@ mod tests { "tool": { "uv": { "sources": { - "bird-feeder": { - "workspace": true - } + "bird-feeder": [ + { + "workspace": true + } + ] }, "workspace": { "members": [ diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index c451f99c466f..d2e225d07aab 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -453,6 +453,7 @@ pub(crate) async fn add( rev, tag, branch, + marker, }) => { let credentials = Credentials::from_url(&git); if let Some(credentials) = credentials { @@ -468,6 +469,7 @@ pub(crate) async fn add( rev, tag, branch, + marker, }) } _ => source, diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index 40819ac4b3a0..7ae9b50b3bea 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -293,15 +293,15 @@ async fn do_lock( let lhs = lhs .contents() .map(|contents| contents.to_string()) - .unwrap_or("true".to_string()); + .unwrap_or_else(|| "true".to_string()); let rhs = rhs .contents() .map(|contents| contents.to_string()) - .unwrap_or("true".to_string()); + .unwrap_or_else(|| "true".to_string()); let hint = hint .contents() .map(|contents| contents.to_string()) - .unwrap_or("true".to_string()); + .unwrap_or_else(|| "true".to_string()); return Err(ProjectError::OverlappingMarkers(lhs, rhs, hint)); } diff --git a/crates/uv/src/commands/project/run.rs b/crates/uv/src/commands/project/run.rs index b854f6c40faa..48f3c2fb72f2 100644 --- a/crates/uv/src/commands/project/run.rs +++ b/crates/uv/src/commands/project/run.rs @@ -196,13 +196,13 @@ pub(crate) async fn run( let requirements = dependencies .into_iter() - .map(|requirement| { + .flat_map(|requirement| { LoweredRequirement::from_non_workspace_requirement( requirement, script_dir, script_sources, ) - .map(LoweredRequirement::into_inner) + .map_ok(uv_distribution::LoweredRequirement::into_inner) }) .collect::>()?; let spec = RequirementsSpecification::from_requirements(requirements); diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index b891b56a1d83..a9ee9d8f119f 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -28,7 +28,7 @@ use uv_python::{PythonDownloads, PythonEnvironment, PythonPreference, PythonRequ use uv_resolver::{FlatIndex, Lock}; use uv_types::{BuildIsolation, HashStrategy}; use uv_warnings::warn_user; -use uv_workspace::pyproject::{Source, ToolUvSources}; +use uv_workspace::pyproject::{Source, Sources, ToolUvSources}; use uv_workspace::{DiscoveryOptions, InstallTarget, MemberDiscovery, VirtualProject, Workspace}; /// Sync the project environment. @@ -425,7 +425,7 @@ fn store_credentials_from_workspace(workspace: &Workspace) { .and_then(|uv| uv.sources.as_ref()) .map(ToolUvSources::inner) .iter() - .flat_map(|sources| sources.values()) + .flat_map(|sources| sources.values().flat_map(Sources::iter)) { match source { Source::Git { git, .. } => { diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index afdb635e0337..bbc35396e7b0 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -13323,3 +13323,468 @@ fn lock_change_requires_python() -> Result<()> { Ok(()) } + +#[test] +fn lock_multiple_sources() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig"] + + [tool.uv.sources] + iniconfig = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", marker = "sys_platform != 'win32'" }, + { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", marker = "sys_platform == 'win32'" }, + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + resolution-markers = [ + "sys_platform != 'win32'", + "sys_platform == 'win32'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz" } + resolution-markers = [ + "sys_platform == 'win32'", + ] + sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3" } + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" } + resolution-markers = [ + "sys_platform != 'win32'", + ] + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "iniconfig", version = "2.0.0", source = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz" }, marker = "sys_platform == 'win32'" }, + { name = "iniconfig", version = "2.0.0", source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, marker = "sys_platform != 'win32'" }, + ] + + [package.metadata] + requires-dist = [ + { name = "iniconfig", marker = "sys_platform != 'win32'", url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, + { name = "iniconfig", marker = "sys_platform == 'win32'", url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz" }, + ] + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + Ok(()) +} + +#[test] +fn lock_multiple_sources_conflict() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig"] + + [tool.uv.sources] + iniconfig = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", marker = "sys_platform == 'win32' and python_version == '3.12'" }, + { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", marker = "sys_platform == 'win32'" }, + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Failed to parse: `pyproject.toml` + Caused by: TOML parse error at line 9, column 21 + | + 9 | iniconfig = [ + | ^ + Source markers must be disjoint, but the following markers overlap: `python_full_version == '3.12.*' and sys_platform == 'win32'` and `sys_platform == 'win32'`. + + hint: replace `sys_platform == 'win32'` with `python_full_version != '3.12.*' and sys_platform == 'win32'`. + + "###); + + Ok(()) +} + +/// Multiple `index` entries is not yet supported. +#[test] +fn lock_multiple_sources_index() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig"] + + [tool.uv.sources] + iniconfig = [ + { index = "pytorch", marker = "sys_platform != 'win32'" }, + { index = "internal", marker = "sys_platform == 'win32'" }, + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: Failed to parse: `pyproject.toml` + Caused by: TOML parse error at line 9, column 21 + | + 9 | iniconfig = [ + | ^ + Sources can only include a single index source + + "###); + + Ok(()) +} + +#[test] +fn lock_multiple_sources_non_total() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig"] + + [tool.uv.sources] + iniconfig = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", marker = "sys_platform == 'darwin'" }, + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + resolution-markers = [ + "sys_platform == 'darwin'", + "sys_platform != 'darwin'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "sys_platform != 'darwin'", + ] + sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, + ] + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" } + resolution-markers = [ + "sys_platform == 'darwin'", + ] + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "iniconfig", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'darwin'" }, + { name = "iniconfig", version = "2.0.0", source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, marker = "sys_platform == 'darwin'" }, + ] + + [package.metadata] + requires-dist = [ + { name = "iniconfig", marker = "sys_platform != 'darwin'" }, + { name = "iniconfig", marker = "sys_platform == 'darwin'", url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, + ] + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + Ok(()) +} + +#[test] +fn lock_multiple_sources_respect_marker() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig ; platform_system == 'Windows'"] + + [tool.uv.sources] + iniconfig = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", marker = "sys_platform == 'darwin'" }, + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + resolution-markers = [ + "platform_system == 'Windows' and sys_platform == 'darwin'", + "platform_system == 'Windows' and sys_platform != 'darwin'", + "platform_system != 'Windows'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "platform_system == 'Windows' and sys_platform != 'darwin'", + ] + sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, + ] + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" } + resolution-markers = [ + "platform_system == 'Windows' and sys_platform == 'darwin'", + ] + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "iniconfig", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_system == 'Windows' and sys_platform != 'darwin'" }, + { name = "iniconfig", version = "2.0.0", source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, marker = "platform_system == 'Windows' and sys_platform == 'darwin'" }, + ] + + [package.metadata] + requires-dist = [ + { name = "iniconfig", marker = "platform_system == 'Windows' and sys_platform != 'darwin'" }, + { name = "iniconfig", marker = "platform_system == 'Windows' and sys_platform == 'darwin'", url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, + ] + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 3 packages in [TIME] + "###); + + Ok(()) +} + +#[test] +fn lock_multiple_sources_extra() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = ["iniconfig"] + + [project.optional-dependencies] + cpu = [] + + [tool.uv.sources] + iniconfig = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", marker = "extra == 'cpu'" }, + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "iniconfig" + version = "2.0.0" + source = { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" } + wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + + [package.optional-dependencies] + cpu = [ + { name = "iniconfig" }, + ] + + [package.metadata] + requires-dist = [ + { name = "iniconfig", marker = "extra == 'cpu'", url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl" }, + { name = "iniconfig", marker = "extra != 'cpu'" }, + ] + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 2 packages in [TIME] + "###); + + Ok(()) +} diff --git a/docs/concepts/dependencies.md b/docs/concepts/dependencies.md index e45ea9dcdbeb..20da9f23d926 100644 --- a/docs/concepts/dependencies.md +++ b/docs/concepts/dependencies.md @@ -223,6 +223,46 @@ members = [ ] ``` +### Platform-specific sources + +You can limit a source to a given platform or Python version by providing +[PEP 508](https://peps.python.org/pep-0508/#environment-markers)-compatible environment markers for +the source. + +For example, to pull `httpx` from GitHub, but only on macOS, use the following: + +```toml title="pyproject.toml" +[project] +dependencies = [ + "httpx", +] + +[tool.uv.sources] +httpx = { git = "https://github.com/encode/httpx", tag = "0.27.2", marker = "sys_platform == 'darwin'" } +``` + +By specifying the marker on the source, uv will still include `httpx` on all platforms, but will +download the source from GitHub on macOS, and fall back to PyPI on all other platforms. + +### Multiple sources + +You can specify multiple sources for a single dependency by providing a list of sources, +disambiguated by [PEP 508](https://peps.python.org/pep-0508/#environment-markers)-compatible +environment markers. For example, to pull in different `httpx` commits on macOS vs. Linux: + +```toml title="pyproject.toml" +[project] +dependencies = [ + "httpx", +] + +[tool.uv.sources] +httpx = [ + { git = "https://github.com/encode/httpx", tag = "0.27.2", marker = "sys_platform == 'darwin'" }, + { git = "https://github.com/encode/httpx", tag = "0.24.1", marker = "sys_platform == 'linux'" }, +] +``` + ## Optional dependencies It is common for projects that are published as libraries to make some features optional to reduce diff --git a/uv.schema.json b/uv.schema.json index 645ffc4582fb..fa14d871eefe 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -625,6 +625,10 @@ } ] }, + "MarkerTree": { + "description": "A PEP 508-compliant marker expression, e.g., `sys_platform == 'Darwin'`", + "type": "string" + }, "PackageName": { "description": "The normalized name of a package.\n\nConverts the name to lowercase and collapses runs of `-`, `_`, and `.` down to a single `-`. For example, `---`, `.`, and `__` are all converted to a single `-`.\n\nSee: ", "type": "string" @@ -1240,6 +1244,9 @@ "type": "string", "format": "uri" }, + "marker": { + "$ref": "#/definitions/MarkerTree" + }, "rev": { "type": [ "string", @@ -1273,6 +1280,9 @@ "url" ], "properties": { + "marker": { + "$ref": "#/definitions/MarkerTree" + }, "subdirectory": { "description": "For source distributions, the path to the directory with the `pyproject.toml`, if it's not in the archive root.", "anyOf": [ @@ -1305,6 +1315,9 @@ "null" ] }, + "marker": { + "$ref": "#/definitions/MarkerTree" + }, "path": { "$ref": "#/definitions/String" } @@ -1320,6 +1333,9 @@ "properties": { "index": { "type": "string" + }, + "marker": { + "$ref": "#/definitions/MarkerTree" } }, "additionalProperties": false @@ -1331,6 +1347,9 @@ "workspace" ], "properties": { + "marker": { + "$ref": "#/definitions/MarkerTree" + }, "workspace": { "description": "When set to `false`, the package will be fetched from the remote index, rather than included as a workspace package.", "type": "boolean" @@ -1361,6 +1380,9 @@ "index": { "type": "string" }, + "marker": { + "$ref": "#/definitions/MarkerTree" + }, "path": { "$ref": "#/definitions/String" }, @@ -1397,6 +1419,22 @@ } ] }, + "Sources": { + "$ref": "#/definitions/SourcesWire" + }, + "SourcesWire": { + "anyOf": [ + { + "$ref": "#/definitions/Source" + }, + { + "type": "array", + "items": { + "$ref": "#/definitions/Source" + } + } + ] + }, "StaticMetadata": { "description": "A subset of the Python Package Metadata 2.3 standard as specified in .", "type": "object", @@ -1565,7 +1603,7 @@ "ToolUvSources": { "type": "object", "additionalProperties": { - "$ref": "#/definitions/Source" + "$ref": "#/definitions/Sources" } }, "ToolUvWorkspace": {