Add support for reading and resolving include-group in dependency g…

…roups (#8266)

Part of #8090 

Adds the ability to read group inclusions (`include-group = <name>`) in
the `pyproject.toml`. Resolves groups into concrete dependencies for
resolution.

See #8110 for a bit more commentary
on deferred work.

---------

Co-authored-by: Charlie Marsh <charlie.r.marsh@gmail.com>

crates/uv-distribution/src/metadata/mod.rs

@@ -24,14 +24,37 @@ pub enum MetadataError {
     Workspace(#[from] WorkspaceError),
     #[error("Failed to parse entry: `{0}`")]
     LoweringError(PackageName, #[source] Box<LoweringError>),
-    #[error("Failed to parse entry in `{0}`: `{1}`")]
+    #[error("Failed to parse entry in group `{0}`: `{1}`")]
     GroupLoweringError(GroupName, PackageName, #[source] Box<LoweringError>),
-    #[error("Failed to parse entry in `{0}`: `{1}`")]
+    #[error("Failed to parse entry in group `{0}`: `{1}`")]
     GroupParseError(
         GroupName,
         String,
         #[source] Box<Pep508Error<VerbatimParsedUrl>>,
     ),
+    #[error("Failed to find group `{0}` included by `{1}`")]
+    GroupNotFound(GroupName, GroupName),
+    #[error("Detected a cycle in `dependency-groups`: {0}")]
+    DependencyGroupCycle(Cycle),
+}
+
+/// A cycle in the `dependency-groups` table.
+#[derive(Debug)]
+pub struct Cycle(Vec<GroupName>);
+
+/// Display a cycle, e.g., `a -> b -> c -> a`.
+impl std::fmt::Display for Cycle {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let [first, rest @ ..] = self.0.as_slice() else {
+            return Ok(());
+        };
+        write!(f, "`{first}`")?;
+        for group in rest {
+            write!(f, " -> `{group}`")?;
+        }
+        write!(f, " -> `{first}`")?;
+        Ok(())
+    }
 }
 
 #[derive(Debug, Clone)]

crates/uv-distribution/src/metadata/requires_dist.rs

@@ -1,17 +1,19 @@
-use crate::metadata::{LoweredRequirement, MetadataError};
-use crate::Metadata;
-
 use std::collections::BTreeMap;
 use std::path::Path;
 use std::str::FromStr;
 
+use tracing::warn;
+
 use uv_configuration::{LowerBound, SourceStrategy};
 use uv_distribution_types::IndexLocations;
 use uv_normalize::{ExtraName, GroupName, PackageName, DEV_DEPENDENCIES};
 use uv_pypi_types::VerbatimParsedUrl;
-use uv_workspace::pyproject::{Sources, ToolUvSources};
+use uv_workspace::pyproject::{DependencyGroupSpecifier, ToolUvSources};
 use uv_workspace::{DiscoveryOptions, ProjectWorkspace};
 
+use crate::metadata::{Cycle, LoweredRequirement, MetadataError};
+use crate::Metadata;
+
 #[derive(Debug, Clone)]
 pub struct RequiresDist {
     pub name: PackageName,
@@ -116,55 +118,71 @@ impl RequiresDist {
                 .dependency_groups
                 .iter()
                 .flatten()
-                .map(|(name, requirements)| {
-                    (
-                        name.clone(),
-                        requirements
-                            .iter()
-                            .map(|requirement| {
-                                match uv_pep508::Requirement::<VerbatimParsedUrl>::from_str(
-                                    requirement,
-                                ) {
-                                    Ok(requirement) => Ok(requirement),
-                                    Err(err) => Err(MetadataError::GroupParseError(
-                                        name.clone(),
-                                        requirement.clone(),
-                                        Box::new(err),
-                                    )),
-                                }
-                            })
-                            .collect::<Result<Vec<_>, _>>(),
-                    )
-                })
+                .collect::<BTreeMap<_, _>>();
+
+            // Resolve any `include-group` entries in `dependency-groups`.
+            let dependency_groups = resolve_dependency_groups(&dependency_groups)?
+                .into_iter()
                 .chain(
-                    // Only add the `dev` group if `dev-dependencies` is defined
+                    // Only add the `dev` group if `dev-dependencies` is defined.
                     dev_dependencies
                         .into_iter()
-                        .map(|requirements| (DEV_DEPENDENCIES.clone(), Ok(requirements.clone()))),
+                        .map(|requirements| (DEV_DEPENDENCIES.clone(), requirements.clone())),
                 )
                 .map(|(name, requirements)| {
-                    // Apply sources to the requirements
-                    match requirements {
-                        Ok(requirements) => match apply_source_strategy(
-                            source_strategy,
-                            requirements,
-                            &metadata,
-                            project_sources,
-                            project_indexes,
-                            locations,
-                            project_workspace,
-                            lower_bound,
-                            &name,
-                        ) {
-                            Ok(requirements) => Ok((name, requirements)),
-                            Err(err) => Err(err),
-                        },
-                        Err(err) => Err(err),
-                    }
+                    let requirements = match source_strategy {
+                        SourceStrategy::Enabled => requirements
+                            .into_iter()
+                            .flat_map(|requirement| {
+                                let group_name = name.clone();
+                                let requirement_name = requirement.name.clone();
+                                LoweredRequirement::from_requirement(
+                                    requirement,
+                                    &metadata.name,
+                                    project_workspace.project_root(),
+                                    project_sources,
+                                    project_indexes,
+                                    locations,
+                                    project_workspace.workspace(),
+                                    lower_bound,
+                                )
+                                .map(move |requirement| {
+                                    match requirement {
+                                        Ok(requirement) => Ok(requirement.into_inner()),
+                                        Err(err) => Err(MetadataError::GroupLoweringError(
+                                            group_name.clone(),
+                                            requirement_name.clone(),
+                                            Box::new(err),
+                                        )),
+                                    }
+                                })
+                            })
+                            .collect::<Result<Vec<_>, _>>(),
+                        SourceStrategy::Disabled => Ok(requirements
+                            .into_iter()
+                            .map(uv_pypi_types::Requirement::from)
+                            .collect()),
+                    }?;
+                    Ok::<(GroupName, Vec<uv_pypi_types::Requirement>), MetadataError>((
+                        name,
+                        requirements,
+                    ))
                 })
                 .collect::<Result<Vec<_>, _>>()?;
 
-            dependency_groups.into_iter().collect::<BTreeMap<_, _>>()
+            // Merge any overlapping groups.
+            let mut map = BTreeMap::new();
+            for (name, dependencies) in dependency_groups {
+                match map.entry(name) {
+                    std::collections::btree_map::Entry::Vacant(entry) => {
+                        entry.insert(dependencies);
+                    }
+                    std::collections::btree_map::Entry::Occupied(mut entry) => {
+                        entry.get_mut().extend(dependencies);
+                    }
+                }
+            }
+            map
         };
 
         let requires_dist = metadata.requires_dist.into_iter();
@@ -217,47 +235,79 @@ impl From<Metadata> for RequiresDist {
     }
 }
 
-fn apply_source_strategy(
-    source_strategy: SourceStrategy,
-    requirements: Vec<uv_pep508::Requirement<VerbatimParsedUrl>>,
-    metadata: &uv_pypi_types::RequiresDist,
-    project_sources: &BTreeMap<PackageName, Sources>,
-    project_indexes: &[uv_distribution_types::Index],
-    locations: &IndexLocations,
-    project_workspace: &ProjectWorkspace,
-    lower_bound: LowerBound,
-    group_name: &GroupName,
-) -> Result<Vec<uv_pypi_types::Requirement>, MetadataError> {
-    match source_strategy {
-        SourceStrategy::Enabled => requirements
-            .into_iter()
-            .flat_map(|requirement| {
-                let requirement_name = requirement.name.clone();
-                LoweredRequirement::from_requirement(
-                    requirement,
-                    &metadata.name,
-                    project_workspace.project_root(),
-                    project_sources,
-                    project_indexes,
-                    locations,
-                    project_workspace.workspace(),
-                    lower_bound,
-                )
-                .map(move |requirement| match requirement {
-                    Ok(requirement) => Ok(requirement.into_inner()),
-                    Err(err) => Err(MetadataError::GroupLoweringError(
-                        group_name.clone(),
-                        requirement_name.clone(),
-                        Box::new(err),
-                    )),
-                })
-            })
-            .collect::<Result<Vec<_>, _>>(),
-        SourceStrategy::Disabled => Ok(requirements
-            .into_iter()
-            .map(uv_pypi_types::Requirement::from)
-            .collect()),
+/// Resolve the dependency groups (which may contain references to other groups) into concrete
+/// lists of requirements.
+fn resolve_dependency_groups(
+    groups: &BTreeMap<&GroupName, &Vec<DependencyGroupSpecifier>>,
+) -> Result<BTreeMap<GroupName, Vec<uv_pep508::Requirement<VerbatimParsedUrl>>>, MetadataError> {
+    fn resolve_group<'data>(
+        resolved: &mut BTreeMap<GroupName, Vec<uv_pep508::Requirement<VerbatimParsedUrl>>>,
+        groups: &'data BTreeMap<&GroupName, &Vec<DependencyGroupSpecifier>>,
+        name: &'data GroupName,
+        parents: &mut Vec<&'data GroupName>,
+    ) -> Result<(), MetadataError> {
+        let Some(specifiers) = groups.get(name) else {
+            // Missing group
+            let parent_name = parents
+                .iter()
+                .last()
+                .copied()
+                .expect("parent when group is missing");
+            return Err(MetadataError::GroupNotFound(
+                name.clone(),
+                parent_name.clone(),
+            ));
+        };
+
+        // "Dependency Group Includes MUST NOT include cycles, and tools SHOULD report an error if they detect a cycle."
+        if parents.contains(&name) {
+            return Err(MetadataError::DependencyGroupCycle(Cycle(
+                parents.iter().copied().cloned().collect(),
+            )));
+        }
+
+        // If we already resolved this group, short-circuit.
+        if resolved.contains_key(name) {
+            return Ok(());
+        }
+
+        parents.push(name);
+        let mut requirements = Vec::with_capacity(specifiers.len());
+        for specifier in *specifiers {
+            match specifier {
+                DependencyGroupSpecifier::Requirement(requirement) => {
+                    match uv_pep508::Requirement::<VerbatimParsedUrl>::from_str(requirement) {
+                        Ok(requirement) => requirements.push(requirement),
+                        Err(err) => {
+                            return Err(MetadataError::GroupParseError(
+                                name.clone(),
+                                requirement.clone(),
+                                Box::new(err),
+                            ));
+                        }
+                    }
+                }
+                DependencyGroupSpecifier::IncludeGroup { include_group } => {
+                    resolve_group(resolved, groups, include_group, parents)?;
+                    requirements.extend(resolved.get(include_group).into_iter().flatten().cloned());
+                }
+                DependencyGroupSpecifier::Object(map) => {
+                    warn!("Ignoring Dependency Object Specifier referenced by `{name}`: {map:?}");
+                }
+            }
+        }
+        parents.pop();
+
+        resolved.insert(name.clone(), requirements);
+        Ok(())
+    }
+
+    let mut resolved = BTreeMap::new();
+    for name in groups.keys() {
+        let mut parents = Vec::new();
+        resolve_group(&mut resolved, groups, name, &mut parents)?;
     }
+    Ok(resolved)
 }
 
 #[cfg(test)]