Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for named and explicit indexes #7481

Merged
merged 2 commits into from
Oct 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

74 changes: 63 additions & 11 deletions crates/uv-cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ use anyhow::{anyhow, Result};
use clap::builder::styling::{AnsiColor, Effects, Style};
use clap::builder::Styles;
use clap::{Args, Parser, Subcommand};

use url::Url;
use uv_cache::CacheArgs;
use uv_configuration::{
ConfigSettingEntry, ExportFormat, IndexStrategy, KeyringProviderType, PackageNameSpecifier,
TargetTriple, TrustedHost, TrustedPublishing, VersionControlSystem,
};
use uv_distribution_types::{FlatIndexLocation, IndexUrl};
use uv_distribution_types::{FlatIndexLocation, Index, IndexUrl};
use uv_normalize::{ExtraName, PackageName};
use uv_pep508::Requirement;
use uv_pypi_types::VerbatimParsedUrl;
Expand Down Expand Up @@ -793,6 +794,36 @@ fn parse_flat_index(input: &str) -> Result<Maybe<FlatIndexLocation>, String> {
}
}

/// Parse a string into an [`Index`], mapping the empty string to `None`.
fn parse_index_source(input: &str) -> Result<Maybe<Index>, String> {
if input.is_empty() {
Ok(Maybe::None)
} else {
match Index::from_str(input) {
Ok(index) => Ok(Maybe::Some(Index {
default: false,
..index
})),
Err(err) => Err(err.to_string()),
}
}
}

/// Parse a string into an [`Index`], mapping the empty string to `None`.
fn parse_default_index_source(input: &str) -> Result<Maybe<Index>, String> {
if input.is_empty() {
Ok(Maybe::None)
} else {
match Index::from_str(input) {
Ok(index) => Ok(Maybe::Some(Index {
default: true,
..index
})),
Err(err) => Err(err.to_string()),
}
}
}

/// Parse a string into an [`Url`], mapping the empty string to `None`.
fn parse_insecure_host(input: &str) -> Result<Maybe<TrustedHost>, String> {
if input.is_empty() {
Expand Down Expand Up @@ -2282,8 +2313,8 @@ pub struct VenvArgs {
///
/// By default, uv will stop at the first index on which a given package is available, and
/// limit resolutions to those present on that first index (`first-match`). This prevents
/// "dependency confusion" attacks, whereby an attack can upload a malicious package under the
/// same name to a secondary.
/// "dependency confusion" attacks, whereby an attacker can upload a malicious package under the
/// same name to an alternate index.
#[arg(long, value_enum, env = EnvVars::UV_INDEX_STRATEGY)]
pub index_strategy: Option<IndexStrategy>,

Expand Down Expand Up @@ -3808,7 +3839,28 @@ pub struct GenerateShellCompletionArgs {
#[derive(Args)]
#[allow(clippy::struct_excessive_bools)]
pub struct IndexArgs {
/// The URL of the Python package index (by default: <https://pypi.org/simple>).
/// The URLs to use when resolving dependencies, in addition to the default index.
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
///
/// All indexes provided via this flag take priority over the index specified by
/// `--default-index` (which defaults to PyPI). When multiple `--index` flags are
/// provided, earlier values take priority.
#[arg(long, env = "UV_INDEX", value_delimiter = ' ', value_parser = parse_index_source, help_heading = "Index options")]
pub index: Option<Vec<Maybe<Index>>>,

/// The URL of the default package index (by default: <https://pypi.org/simple>).
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
///
/// The index given by this flag is given lower priority than all other indexes specified via
/// the `--index` flag.
#[arg(long, env = "UV_DEFAULT_INDEX", value_parser = parse_default_index_source, help_heading = "Index options")]
pub default_index: Option<Maybe<Index>>,

/// (Deprecated: use `--default-index` instead) The URL of the Python package index (by default: <https://pypi.org/simple>).
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
Expand All @@ -3818,7 +3870,7 @@ pub struct IndexArgs {
#[arg(long, short, env = EnvVars::UV_INDEX_URL, value_parser = parse_index_url, help_heading = "Index options")]
pub index_url: Option<Maybe<IndexUrl>>,
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@zanieb -- Should we... hide these?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In a future release, I think.


/// Extra URLs of package indexes to use, in addition to `--index-url`.
/// (Deprecated: use `--index` instead) Extra URLs of package indexes to use, in addition to `--index-url`.
///
/// Accepts either a repository compliant with PEP 503 (the simple repository API), or a local
/// directory laid out in the same format.
Expand Down Expand Up @@ -3955,8 +4007,8 @@ pub struct InstallerArgs {
///
/// By default, uv will stop at the first index on which a given package is available, and
/// limit resolutions to those present on that first index (`first-match`). This prevents
/// "dependency confusion" attacks, whereby an attack can upload a malicious package under the
/// same name to a secondary.
/// "dependency confusion" attacks, whereby an attacker can upload a malicious package under the
/// same name to an alternate index.
#[arg(
long,
value_enum,
Expand Down Expand Up @@ -4117,8 +4169,8 @@ pub struct ResolverArgs {
///
/// By default, uv will stop at the first index on which a given package is available, and
/// limit resolutions to those present on that first index (`first-match`). This prevents
/// "dependency confusion" attacks, whereby an attack can upload a malicious package under the
/// same name to a secondary.
/// "dependency confusion" attacks, whereby an attacker can upload a malicious package under the
/// same name to an alternate index.
#[arg(
long,
value_enum,
Expand Down Expand Up @@ -4309,8 +4361,8 @@ pub struct ResolverInstallerArgs {
///
/// By default, uv will stop at the first index on which a given package is available, and
/// limit resolutions to those present on that first index (`first-match`). This prevents
/// "dependency confusion" attacks, whereby an attack can upload a malicious package under the
/// same name to a secondary.
/// "dependency confusion" attacks, whereby an attacker can upload a malicious package under the
/// same name to an alternate index.
#[arg(
long,
value_enum,
Expand Down
28 changes: 27 additions & 1 deletion crates/uv-cli/src/options.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
use uv_cache::Refresh;
use uv_configuration::ConfigSettings;
use uv_resolver::PrereleaseMode;
use uv_settings::{PipOptions, ResolverInstallerOptions, ResolverOptions};
use uv_settings::{Combine, PipOptions, ResolverInstallerOptions, ResolverOptions};

use crate::{
BuildOptionsArgs, IndexArgs, InstallerArgs, Maybe, RefreshArgs, ResolverArgs,
Expand Down Expand Up @@ -186,13 +186,21 @@ impl From<ResolverInstallerArgs> for PipOptions {
impl From<IndexArgs> for PipOptions {
fn from(args: IndexArgs) -> Self {
let IndexArgs {
default_index,
index,
index_url,
extra_index_url,
no_index,
find_links,
} = args;

Self {
index: default_index
.and_then(Maybe::into_option)
.map(|default_index| vec![default_index])
.combine(
index.map(|index| index.into_iter().filter_map(Maybe::into_option).collect()),
),
index_url: index_url.and_then(Maybe::into_option),
extra_index_url: extra_index_url.map(|extra_index_url| {
extra_index_url
Expand Down Expand Up @@ -247,6 +255,15 @@ pub fn resolver_options(
} = build_args;

ResolverOptions {
index: index_args
.default_index
.and_then(Maybe::into_option)
.map(|default_index| vec![default_index])
.combine(
index_args
.index
.map(|index| index.into_iter().filter_map(Maybe::into_option).collect()),
),
index_url: index_args.index_url.and_then(Maybe::into_option),
extra_index_url: index_args.extra_index_url.map(|extra_index_url| {
extra_index_url
Expand Down Expand Up @@ -335,7 +352,16 @@ pub fn resolver_installer_options(
no_binary_package,
} = build_args;

let default_index = index_args
.default_index
.and_then(Maybe::into_option)
.map(|default_index| vec![default_index]);
let index = index_args
.index
.map(|index| index.into_iter().filter_map(Maybe::into_option).collect());

ResolverInstallerOptions {
index: default_index.combine(index),
index_url: index_args.index_url.and_then(Maybe::into_option),
extra_index_url: index_args.extra_index_url.map(|extra_index_url| {
extra_index_url
Expand Down
12 changes: 10 additions & 2 deletions crates/uv-client/src/registry_client.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use async_http_range_reader::AsyncHttpRangeReader;
use futures::{FutureExt, TryStreamExt};
use http::HeaderMap;
use itertools::Either;
use reqwest::{Client, Response, StatusCode};
use reqwest_middleware::ClientWithMiddleware;
use std::collections::BTreeMap;
Expand All @@ -16,7 +17,7 @@ use uv_configuration::KeyringProviderType;
use uv_configuration::{IndexStrategy, TrustedHost};
use uv_distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
use uv_distribution_types::{
BuiltDist, File, FileLocation, IndexCapabilities, IndexUrl, IndexUrls, Name,
BuiltDist, File, FileLocation, Index, IndexCapabilities, IndexUrl, IndexUrls, Name,
};
use uv_metadata::{read_metadata_async_seek, read_metadata_async_stream};
use uv_normalize::PackageName;
Expand Down Expand Up @@ -204,8 +205,15 @@ impl RegistryClient {
pub async fn simple(
&self,
package_name: &PackageName,
index: Option<&IndexUrl>,
) -> Result<Vec<(IndexUrl, OwnedArchive<SimpleMetadata>)>, Error> {
let mut it = self.index_urls.indexes().peekable();
let indexes = if let Some(index) = index {
Either::Left(std::iter::once(index))
} else {
Either::Right(self.index_urls.indexes().map(Index::url))
};

let mut it = indexes.peekable();
if it.peek().is_none() {
return Err(ErrorKind::NoIndex(package_name.to_string()).into());
}
Expand Down
146 changes: 146 additions & 0 deletions crates/uv-distribution-types/src/index.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
use crate::{IndexUrl, IndexUrlError};
use std::str::FromStr;
use thiserror::Error;
use url::Url;

#[derive(Debug, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
pub struct Index {
/// The name of the index.
///
/// Index names can be used to reference indexes elsewhere in the configuration. For example,
/// you can pin a package to a specific index by name:
///
/// ```toml
/// [[tool.uv.index]]
/// name = "pytorch"
/// url = "https://download.pytorch.org/whl/cu121"
///
/// [tool.uv.sources]
/// torch = { index = "pytorch" }
/// ```
pub name: Option<String>,
/// The URL of the index.
///
/// Expects to receive a URL (e.g., `https://pypi.org/simple`) or a local path.
pub url: IndexUrl,
/// Mark the index as explicit.
///
/// Explicit indexes will _only_ be used when explicitly requested via a `[tool.uv.sources]`
/// definition, as in:
///
/// ```toml
/// [[tool.uv.index]]
/// name = "pytorch"
/// url = "https://download.pytorch.org/whl/cu121"
/// explicit = true
///
/// [tool.uv.sources]
/// torch = { index = "pytorch" }
/// ```
#[serde(default)]
pub explicit: bool,
/// Mark the index as the default index.
///
/// By default, uv uses PyPI as the default index, such that even if additional indexes are
/// defined via `[[tool.uv.index]]`, PyPI will still be used as a fallback for packages that
/// aren't found elsewhere. To disable the PyPI default, set `default = true` on at least one
/// other index.
///
/// Marking an index as default will move it to the front of the list of indexes, such that it
/// is given the highest priority when resolving packages.
#[serde(default)]
pub default: bool,
// /// The type of the index.
// ///
// /// Indexes can either be PEP 503-compliant (i.e., a registry implementing the Simple API) or
// /// structured as a flat list of distributions (e.g., `--find-links`). In both cases, indexes
// /// can point to either local or remote resources.
// #[serde(default)]
// pub r#type: IndexKind,
}

// #[derive(
// Default, Debug, Copy, Clone, Hash, Eq, PartialEq, serde::Serialize, serde::Deserialize,
// )]
// #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
// pub enum IndexKind {
// /// A PEP 503 and/or PEP 691-compliant index.
// #[default]
// Simple,
// /// An index containing a list of links to distributions (e.g., `--find-links`).
// Flat,
// }

impl Index {
/// Initialize an [`Index`] from a pip-style `--index-url`.
pub fn from_index_url(url: IndexUrl) -> Self {
Self {
url,
name: None,
explicit: false,
default: true,
}
}

/// Initialize an [`Index`] from a pip-style `--extra-index-url`.
pub fn from_extra_index_url(url: IndexUrl) -> Self {
Self {
url,
name: None,
explicit: false,
default: false,
}
}

/// Return the [`IndexUrl`] of the index.
pub fn url(&self) -> &IndexUrl {
&self.url
}

/// Return the raw [`URL`] of the index.
pub fn raw_url(&self) -> &Url {
self.url.url()
}
}

impl FromStr for Index {
type Err = IndexSourceError;

fn from_str(s: &str) -> Result<Self, Self::Err> {
// Determine whether the source is prefixed with a name, as in `name=https://pypi.org/simple`.
if let Some((name, url)) = s.split_once('=') {
if name.is_empty() {
return Err(IndexSourceError::EmptyName);
}

if name.chars().all(char::is_alphanumeric) {
let url = IndexUrl::from_str(url)?;
return Ok(Self {
name: Some(name.to_string()),
url,
explicit: false,
default: false,
});
}
}

// Otherwise, assume the source is a URL.
let url = IndexUrl::from_str(s)?;
Ok(Self {
name: None,
url,
explicit: false,
default: false,
})
}
}

/// An error that can occur when parsing an [`Index`].
#[derive(Error, Debug)]
pub enum IndexSourceError {
#[error(transparent)]
Url(#[from] IndexUrlError),
#[error("Index included a name, but the name was empty")]
EmptyName,
}
Loading
Loading