Fork version selection based on requires-python requirements (#9827)

## Summary

This PR addresses a significant limitation in the resolver whereby we
avoid choosing the latest versions of packages when the user supports a
wider range.

For example, with NumPy, the latest versions only support Python 3.10
and later. If you lock a project with `requires-python = ">=3.8"`, we
pick the last NumPy version that supported Python 3.8, and use that for
_all_ Python versions. So you get `1.24.4` for all versions, rather than
`2.2.0`. And we'll never upgrade you unless you bump your
`requires-python`. (Even worse, those versions don't have wheels for
Python 3.12, etc., so you end up building from source.)

(As-is, this is intentional. We optimize for minimizing the number of
selected versions, and the current logic does that well!)

Instead, we know recognize when a version has an elevated
`requires-python` specifier and fork. This is a new fork point, since we
need to fork once we have the package metadata, as opposed to when we
see the dependencies.

In this iteration, I've made this behavior the default. I'm sort of
undecided on whether I want to push on that... Previously, I'd suggested
making it opt-in via a setting
(#8686).

Closes #8492.

crates/uv-distribution-types/src/prioritized_distribution.rs

@@ -60,6 +60,18 @@ pub enum CompatibleDist<'a> {
     },
 }
 
+impl CompatibleDist<'_> {
+    /// Return the `requires-python` specifier for the distribution, if any.
+    pub fn requires_python(&self) -> Option<&VersionSpecifiers> {
+        match self {
+            CompatibleDist::InstalledDist(_) => None,
+            CompatibleDist::SourceDist { sdist, .. } => sdist.file.requires_python.as_ref(),
+            CompatibleDist::CompatibleWheel { wheel, .. } => wheel.file.requires_python.as_ref(),
+            CompatibleDist::IncompatibleWheel { sdist, .. } => sdist.file.requires_python.as_ref(),
+        }
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Clone)]
 pub enum IncompatibleDist {
     /// An incompatible wheel is available.

crates/uv-resolver/src/python_requirement.rs

@@ -1,3 +1,5 @@
+use std::collections::Bound;
+
 use uv_pep440::Version;
 use uv_pep508::{MarkerEnvironment, MarkerTree};
 use uv_python::{Interpreter, PythonVersion};
@@ -84,6 +86,28 @@ impl PythonRequirement {
         })
     }
 
+    /// Split the [`PythonRequirement`] at the given version.
+    ///
+    /// For example, if the current requirement is `>=3.10`, and the split point is `3.11`, then
+    /// the result will be `>=3.10 and <3.11` and `>=3.11`.
+    pub fn split(&self, at: Bound<Version>) -> Option<(Self, Self)> {
+        let (lower, upper) = self.target.split(at)?;
+        Some((
+            Self {
+                exact: self.exact.clone(),
+                installed: self.installed.clone(),
+                target: lower,
+                source: self.source,
+            },
+            Self {
+                exact: self.exact.clone(),
+                installed: self.installed.clone(),
+                target: upper,
+                source: self.source,
+            },
+        ))
+    }
+
     /// Returns `true` if the minimum version of Python required by the target is greater than the
     /// installed version.
     pub fn raises(&self, target: &RequiresPythonRange) -> bool {

crates/uv-resolver/src/requires_python.rs

@@ -1,8 +1,9 @@
-use pubgrub::Range;
 use std::cmp::Ordering;
 use std::collections::Bound;
 use std::ops::Deref;
 
+use pubgrub::Range;
+
 use uv_distribution_filename::WheelFilename;
 use uv_pep440::{release_specifiers_to_ranges, Version, VersionSpecifier, VersionSpecifiers};
 use uv_pep508::{MarkerExpression, MarkerTree, MarkerValueVersion};
@@ -73,24 +74,43 @@ impl RequiresPython {
                 }
             })?;
 
-        // Extract the bounds.
-        let (lower_bound, upper_bound) = range
-            .bounding_range()
-            .map(|(lower_bound, upper_bound)| {
-                (
-                    LowerBound(lower_bound.cloned()),
-                    UpperBound(upper_bound.cloned()),
-                )
-            })
-            .unwrap_or((LowerBound::default(), UpperBound::default()));
-
         // Convert back to PEP 440 specifiers.
         let specifiers = VersionSpecifiers::from_release_only_bounds(range.iter());
 
-        Some(Self {
-            specifiers,
-            range: RequiresPythonRange(lower_bound, upper_bound),
-        })
+        // Extract the bounds.
+        let range = RequiresPythonRange::from_range(&range);
+
+        Some(Self { specifiers, range })
+    }
+
+    /// Split the [`RequiresPython`] at the given version.
+    ///
+    /// For example, if the current requirement is `>=3.10`, and the split point is `3.11`, then
+    /// the result will be `>=3.10 and <3.11` and `>=3.11`.
+    pub fn split(&self, bound: Bound<Version>) -> Option<(Self, Self)> {
+        let RequiresPythonRange(.., upper) = &self.range;
+
+        let upper = Range::from_range_bounds((bound, upper.clone().into()));
+        let lower = upper.complement();
+
+        // Intersect left and right with the existing range.
+        let lower = lower.intersection(&Range::from(self.range.clone()));
+        let upper = upper.intersection(&Range::from(self.range.clone()));
+
+        if lower.is_empty() || upper.is_empty() {
+            None
+        } else {
+            Some((
+                Self {
+                    specifiers: VersionSpecifiers::from_release_only_bounds(lower.iter()),
+                    range: RequiresPythonRange::from_range(&lower),
+                },
+                Self {
+                    specifiers: VersionSpecifiers::from_release_only_bounds(upper.iter()),
+                    range: RequiresPythonRange::from_range(&upper),
+                },
+            ))
+        }
     }
 
     /// Narrow the [`RequiresPython`] by computing the intersection with the given range.
@@ -489,21 +509,25 @@ impl serde::Serialize for RequiresPython {
 impl<'de> serde::Deserialize<'de> for RequiresPython {
     fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
         let specifiers = VersionSpecifiers::deserialize(deserializer)?;
-        let (lower_bound, upper_bound) = release_specifiers_to_ranges(specifiers.clone())
-            .bounding_range()
-            .map(|(lower_bound, upper_bound)| (lower_bound.cloned(), upper_bound.cloned()))
-            .unwrap_or((Bound::Unbounded, Bound::Unbounded));
-        Ok(Self {
-            specifiers,
-            range: RequiresPythonRange(LowerBound(lower_bound), UpperBound(upper_bound)),
-        })
+        let range = release_specifiers_to_ranges(specifiers.clone());
+        let range = RequiresPythonRange::from_range(&range);
+        Ok(Self { specifiers, range })
     }
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Hash)]
 pub struct RequiresPythonRange(LowerBound, UpperBound);
 
 impl RequiresPythonRange {
+    /// Initialize a [`RequiresPythonRange`] from a [`Range`].
+    pub fn from_range(range: &Range<Version>) -> Self {
+        let (lower, upper) = range
+            .bounding_range()
+            .map(|(lower_bound, upper_bound)| (lower_bound.cloned(), upper_bound.cloned()))
+            .unwrap_or((Bound::Unbounded, Bound::Unbounded));
+        Self(LowerBound(lower), UpperBound(upper))
+    }
+
     /// Initialize a [`RequiresPythonRange`] with the given bounds.
     pub fn new(lower: LowerBound, upper: UpperBound) -> Self {
         Self(lower, upper)
@@ -967,4 +991,68 @@ mod tests {
             assert_eq!(requires_python.is_exact_without_patch(), expected);
         }
     }
+
+    #[test]
+    fn split_version() {
+        // Splitting `>=3.10` on `>3.12` should result in `>=3.10, <=3.12` and `>3.12`.
+        let version_specifiers = VersionSpecifiers::from_str(">=3.10").unwrap();
+        let requires_python = RequiresPython::from_specifiers(&version_specifiers);
+        let (lower, upper) = requires_python
+            .split(Bound::Excluded(Version::new([3, 12])))
+            .unwrap();
+        assert_eq!(
+            lower,
+            RequiresPython::from_specifiers(
+                &VersionSpecifiers::from_str(">=3.10, <=3.12").unwrap()
+            )
+        );
+        assert_eq!(
+            upper,
+            RequiresPython::from_specifiers(&VersionSpecifiers::from_str(">3.12").unwrap())
+        );
+
+        // Splitting `>=3.10` on `>=3.12` should result in `>=3.10, <3.12` and `>=3.12`.
+        let version_specifiers = VersionSpecifiers::from_str(">=3.10").unwrap();
+        let requires_python = RequiresPython::from_specifiers(&version_specifiers);
+        let (lower, upper) = requires_python
+            .split(Bound::Included(Version::new([3, 12])))
+            .unwrap();
+        assert_eq!(
+            lower,
+            RequiresPython::from_specifiers(&VersionSpecifiers::from_str(">=3.10, <3.12").unwrap())
+        );
+        assert_eq!(
+            upper,
+            RequiresPython::from_specifiers(&VersionSpecifiers::from_str(">=3.12").unwrap())
+        );
+
+        // Splitting `>=3.10` on `>=3.9` should return `None`.
+        let version_specifiers = VersionSpecifiers::from_str(">=3.10").unwrap();
+        let requires_python = RequiresPython::from_specifiers(&version_specifiers);
+        assert!(requires_python
+            .split(Bound::Included(Version::new([3, 9])))
+            .is_none());
+
+        // Splitting `>=3.10` on `>=3.10` should return `None`.
+        let version_specifiers = VersionSpecifiers::from_str(">=3.10").unwrap();
+        let requires_python = RequiresPython::from_specifiers(&version_specifiers);
+        assert!(requires_python
+            .split(Bound::Included(Version::new([3, 10])))
+            .is_none());
+
+        // Splitting `>=3.9, <3.13` on `>=3.11` should result in `>=3.9, <3.11` and `>=3.11, <3.13`.
+        let version_specifiers = VersionSpecifiers::from_str(">=3.9, <3.13").unwrap();
+        let requires_python = RequiresPython::from_specifiers(&version_specifiers);
+        let (lower, upper) = requires_python
+            .split(Bound::Included(Version::new([3, 11])))
+            .unwrap();
+        assert_eq!(
+            lower,
+            RequiresPython::from_specifiers(&VersionSpecifiers::from_str(">=3.9, <3.11").unwrap())
+        );
+        assert_eq!(
+            upper,
+            RequiresPython::from_specifiers(&VersionSpecifiers::from_str(">=3.11, <3.13").unwrap())
+        );
+    }
 }

crates/uv-resolver/src/resolver/availability.rs

@@ -1,9 +1,11 @@
 use std::fmt::{Display, Formatter};
 
-use crate::resolver::MetadataUnavailable;
 use uv_distribution_types::IncompatibleDist;
 use uv_pep440::{Version, VersionSpecifiers};
 
+use crate::resolver::MetadataUnavailable;
+use crate::ResolverEnvironment;
+
 /// The reason why a package or a version cannot be used.
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub(crate) enum UnavailableReason {
@@ -164,8 +166,10 @@ impl From<&MetadataUnavailable> for UnavailablePackage {
 
 #[derive(Debug, Clone)]
 pub(crate) enum ResolverVersion {
-    /// A usable version
-    Available(Version),
     /// A version that is not usable for some reason
     Unavailable(Version, UnavailableVersion),
+    /// A usable version
+    Unforked(Version),
+    /// A set of forks.
+    Forked(Vec<ResolverEnvironment>),
 }

crates/uv-resolver/src/resolver/environment.rs

@@ -1,13 +1,14 @@
 use std::sync::Arc;
-
+use tracing::trace;
+use uv_pep440::VersionSpecifiers;
 use uv_pep508::{MarkerEnvironment, MarkerTree};
 use uv_pypi_types::{ConflictItem, ConflictItemRef, ResolverMarkerEnvironment};
 
 use crate::pubgrub::{PubGrubDependency, PubGrubPackage};
 use crate::requires_python::RequiresPythonRange;
 use crate::resolver::ForkState;
 use crate::universal_marker::{ConflictMarker, UniversalMarker};
-use crate::PythonRequirement;
+use crate::{PythonRequirement, RequiresPython};
 
 /// Represents one or more marker environments for a resolution.
 ///
@@ -510,6 +511,50 @@ impl<'d> Forker<'d> {
     }
 }
 
+/// Fork the resolver based on a `Requires-Python` specifier.
+pub(crate) fn fork_python_requirement(
+    requires_python: &VersionSpecifiers,
+    python_requirement: &PythonRequirement,
+    env: &ResolverEnvironment,
+) -> Vec<ResolverEnvironment> {
+    let requires_python = RequiresPython::from_specifiers(requires_python);
+    let lower = requires_python.range().lower().clone();
+
+    // Attempt to split the current Python requirement based on the `requires-python` specifier.
+    //
+    // For example, if the current requirement is `>=3.10`, and the split point is `>=3.11`, then
+    // the result will be `>=3.10 and <3.11` and `>=3.11`.
+    //
+    // However, if the current requirement is `>=3.10`, and the split point is `>=3.9`, then the
+    // lower segment will be empty, so we should return an empty list.
+    let Some((lower, upper)) = python_requirement.split(lower.into()) else {
+        trace!(
+            "Unable to split Python requirement `{}` via `Requires-Python` specifier `{}`",
+            python_requirement.target(),
+            requires_python,
+        );
+        return vec![];
+    };
+
+    let Kind::Universal {
+        markers: ref env_marker,
+        ..
+    } = env.kind
+    else {
+        panic!("resolver must be in universal mode for forking")
+    };
+
+    let mut envs = vec![];
+    if !env_marker.is_disjoint(lower.to_marker_tree()) {
+        envs.push(env.narrow_environment(lower.to_marker_tree()));
+    }
+    if !env_marker.is_disjoint(upper.to_marker_tree()) {
+        envs.push(env.narrow_environment(upper.to_marker_tree()));
+    }
+    debug_assert!(!envs.is_empty(), "at least one fork should be produced");
+    envs
+}
+
 #[cfg(test)]
 mod tests {
     use std::ops::Bound;