diff --git a/crates/distribution-types/src/index_url.rs b/crates/distribution-types/src/index_url.rs index 4b206a576e409..5c4af77d8003f 100644 --- a/crates/distribution-types/src/index_url.rs +++ b/crates/distribution-types/src/index_url.rs @@ -9,7 +9,7 @@ use once_cell::sync::Lazy; use serde::{Deserialize, Serialize}; use url::Url; -use pep508_rs::{split_scheme, Scheme, VerbatimUrl}; +use pep508_rs::{split_scheme, Scheme, VerbatimUrl, expand_env_vars}; use uv_fs::normalize_url_path; use crate::Verbatim; @@ -108,7 +108,11 @@ impl FromStr for FlatIndexLocation { /// - `../ferris/` /// - `https://download.pytorch.org/whl/torch_stable.html` fn from_str(s: &str) -> Result { - if let Some((scheme, path)) = split_scheme(s) { + // Expand environment variables. + let expanded = expand_env_vars(s); + + // Parse the expanded path. + if let Some((scheme, path)) = split_scheme(&expanded) { match Scheme::parse(scheme) { // Ex) `file:///home/ferris/project/scripts/...` or `file:../ferris/` Some(Scheme::File) => { @@ -123,19 +127,19 @@ impl FromStr for FlatIndexLocation { // Ex) `https://download.pytorch.org/whl/torch_stable.html` Some(_) => { - let url = Url::parse(s)?; + let url = Url::parse(expanded.as_ref())?; Ok(Self::Url(url)) } // Ex) `C:\Users\ferris\wheel-0.42.0.tar.gz` None => { - let path = PathBuf::from(s); + let path = PathBuf::from(expanded.as_ref()); Ok(Self::Path(path)) } } } else { // Ex) `../ferris/` - let path = PathBuf::from(s); + let path = PathBuf::from(expanded.as_ref()); Ok(Self::Path(path)) } } diff --git a/crates/pep508-rs/src/lib.rs b/crates/pep508-rs/src/lib.rs index ebde80ab9abff..030698ffca0f5 100644 --- a/crates/pep508-rs/src/lib.rs +++ b/crates/pep508-rs/src/lib.rs @@ -45,7 +45,8 @@ use pep440_rs::{Version, VersionSpecifier, VersionSpecifiers}; use uv_fs::normalize_url_path; // Parity with the crates.io version of pep508_rs pub use uv_normalize::{ExtraName, InvalidNameError, PackageName}; -pub use verbatim_url::{expand_path_vars, split_scheme, Scheme, VerbatimUrl}; +pub use verbatim_url::{expand_env_vars, split_scheme, Scheme, VerbatimUrl}; +use crate::verbatim_url::VerbatimUrlError; mod marker; mod verbatim_url; @@ -803,7 +804,10 @@ fn preprocess_url( start: usize, len: usize, ) -> Result { - if let Some((scheme, path)) = split_scheme(url) { + // Expand environment variables in the URL. + let expanded = expand_env_vars(url); + + if let Some((scheme, path)) = split_scheme(&expanded) { match Scheme::parse(scheme) { // Ex) `file:///home/ferris/project/scripts/...` or `file:../editable/`. Some(Scheme::File) => { @@ -815,11 +819,11 @@ fn preprocess_url( #[cfg(feature = "non-pep508-extensions")] if let Some(working_dir) = working_dir { return Ok( - VerbatimUrl::parse_path(path, working_dir).with_given(url.to_string()) + VerbatimUrl::parse_path(path.as_ref(), working_dir).with_given(url.to_string()) ); } - Ok(VerbatimUrl::parse_absolute_path(path) + Ok(VerbatimUrl::parse_absolute_path(path.as_ref()) .map_err(|err| Pep508Error { message: Pep508ErrorSource::UrlError(err), start, @@ -831,12 +835,12 @@ fn preprocess_url( // Ex) `https://download.pytorch.org/whl/torch_stable.html` Some(_) => { // Ex) `https://download.pytorch.org/whl/torch_stable.html` - Ok(VerbatimUrl::from_str(url).map_err(|err| Pep508Error { - message: Pep508ErrorSource::UrlError(err), + Ok(VerbatimUrl::parse_url(expanded.as_ref()).map_err(|err| Pep508Error { + message: Pep508ErrorSource::UrlError(VerbatimUrlError::Url(expanded.to_string(), err)), start, len, input: cursor.to_string(), - })?) + })?.with_given(url.to_string())) } // Ex) `C:\Users\ferris\wheel-0.42.0.tar.gz` @@ -844,11 +848,11 @@ fn preprocess_url( #[cfg(feature = "non-pep508-extensions")] if let Some(working_dir) = working_dir { return Ok( - VerbatimUrl::parse_path(url, working_dir).with_given(url.to_string()) + VerbatimUrl::parse_path(expanded.as_ref(), working_dir).with_given(url.to_string()) ); } - Ok(VerbatimUrl::parse_absolute_path(url) + Ok(VerbatimUrl::parse_absolute_path(expanded.as_ref()) .map_err(|err| Pep508Error { message: Pep508ErrorSource::UrlError(err), start, @@ -862,10 +866,10 @@ fn preprocess_url( // Ex) `../editable/` #[cfg(feature = "non-pep508-extensions")] if let Some(working_dir) = working_dir { - return Ok(VerbatimUrl::parse_path(url, working_dir).with_given(url.to_string())); + return Ok(VerbatimUrl::parse_path(expanded.as_ref(), working_dir).with_given(url.to_string())); } - Ok(VerbatimUrl::parse_absolute_path(url) + Ok(VerbatimUrl::parse_absolute_path(expanded.as_ref()) .map_err(|err| Pep508Error { message: Pep508ErrorSource::UrlError(err), start, diff --git a/crates/pep508-rs/src/verbatim_url.rs b/crates/pep508-rs/src/verbatim_url.rs index 32123639d732c..2c27b97ec147c 100644 --- a/crates/pep508-rs/src/verbatim_url.rs +++ b/crates/pep508-rs/src/verbatim_url.rs @@ -30,12 +30,6 @@ pub struct VerbatimUrl { } impl VerbatimUrl { - /// Parse a URL from a string, expanding any environment variables. - pub fn parse(given: impl AsRef) -> Result { - let url = Url::parse(&expand_env_vars(given.as_ref(), Escape::Url))?; - Ok(Self { url, given: None }) - } - /// Create a [`VerbatimUrl`] from a [`Url`]. pub fn from_url(url: Url) -> Self { Self { url, given: None } @@ -48,15 +42,18 @@ impl VerbatimUrl { Self { url, given: None } } + /// Parse a URL from a string, expanding any environment variables. + pub fn parse_url(given: impl AsRef) -> Result { + let url = Url::parse(given.as_ref())?; + Ok(Self { url, given: None }) + } + /// Parse a URL from an absolute or relative path. #[cfg(feature = "non-pep508-extensions")] // PEP 508 arguably only allows absolute file URLs. - pub fn parse_path(path: impl AsRef, working_dir: impl AsRef) -> Self { - // Expand any environment variables. - let path = PathBuf::from(expand_env_vars(path.as_ref(), Escape::Path).as_ref()); - + pub fn parse_path(path: impl AsRef, working_dir: impl AsRef) -> Self { // Convert the path to an absolute path, if necessary. - let path = if path.is_absolute() { - path + let path = if path.as_ref().is_absolute() { + path.as_ref().to_path_buf() } else { working_dir.as_ref().join(path) }; @@ -71,15 +68,12 @@ impl VerbatimUrl { } /// Parse a URL from an absolute path. - pub fn parse_absolute_path(path: impl AsRef) -> Result { - // Expand any environment variables. - let path = PathBuf::from(expand_env_vars(path.as_ref(), Escape::Path).as_ref()); - + pub fn parse_absolute_path(path: impl AsRef) -> Result { // Convert the path to an absolute path, if necessary. - let path = if path.is_absolute() { - path + let path = if path.as_ref().is_absolute() { + path.as_ref().to_path_buf() } else { - return Err(VerbatimUrlError::RelativePath(path)); + return Err(VerbatimUrlError::RelativePath(path.as_ref().to_path_buf())); }; // Normalize the path. @@ -128,7 +122,7 @@ impl std::str::FromStr for VerbatimUrl { type Err = VerbatimUrlError; fn from_str(s: &str) -> Result { - Self::parse(s) + Self::parse_url(s) .map(|url| url.with_given(s.to_owned())) .map_err(|e| VerbatimUrlError::Url(s.to_owned(), e)) } @@ -160,15 +154,6 @@ pub enum VerbatimUrlError { RelativePath(PathBuf), } -/// Whether to apply percent-encoding when expanding environment variables. -#[derive(Debug, Clone, PartialEq, Eq)] -enum Escape { - /// Apply percent-encoding. - Url, - /// Do not apply percent-encoding. - Path, -} - /// Expand all available environment variables. /// /// This is modeled off of pip's environment variable expansion, which states: @@ -184,7 +169,7 @@ enum Escape { /// Valid characters in variable names follow the `POSIX standard /// `_ and are limited /// to uppercase letter, digits and the `_` (underscore). -fn expand_env_vars(s: &str, escape: Escape) -> Cow<'_, str> { +pub fn expand_env_vars(s: &str) -> Cow<'_, str> { // Generate the project root, to be used via the `${PROJECT_ROOT}` // environment variable. static PROJECT_ROOT_FRAGMENT: Lazy = Lazy::new(|| { @@ -198,21 +183,12 @@ fn expand_env_vars(s: &str, escape: Escape) -> Cow<'_, str> { RE.replace_all(s, |caps: ®ex::Captures<'_>| { let name = caps.name("name").unwrap().as_str(); std::env::var(name).unwrap_or_else(|_| match name { - // Ensure that the variable is URL-escaped, if necessary. - "PROJECT_ROOT" => match escape { - Escape::Url => PROJECT_ROOT_FRAGMENT.replace(' ', "%20"), - Escape::Path => PROJECT_ROOT_FRAGMENT.to_string(), - }, + "PROJECT_ROOT" => PROJECT_ROOT_FRAGMENT.to_string(), _ => caps["var"].to_owned(), }) }) } -/// Expand all available environment variables in a path-like string. -pub fn expand_path_vars(path: &str) -> Cow<'_, str> { - expand_env_vars(path, Escape::Path) -} - /// Like [`Url::parse`], but only splits the scheme. Derived from the `url` crate. pub fn split_scheme(s: &str) -> Option<(&str, &str)> { /// diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs index 9506af352e0da..96bdab2ce1f0e 100644 --- a/crates/requirements-txt/src/lib.rs +++ b/crates/requirements-txt/src/lib.rs @@ -45,10 +45,7 @@ use tracing::instrument; use unscanny::{Pattern, Scanner}; use url::Url; -use pep508_rs::{ - expand_path_vars, split_scheme, Extras, Pep508Error, Pep508ErrorSource, Requirement, Scheme, - VerbatimUrl, -}; +use pep508_rs::{split_scheme, Extras, Pep508Error, Pep508ErrorSource, Requirement, Scheme, VerbatimUrl, expand_env_vars}; use uv_client::Connectivity; use uv_fs::{normalize_url_path, Simplified}; use uv_normalize::ExtraName; @@ -97,7 +94,10 @@ impl FindLink { /// - `../ferris/` /// - `https://download.pytorch.org/whl/torch_stable.html` pub fn parse(given: &str, working_dir: impl AsRef) -> Result { - if let Some((scheme, path)) = split_scheme(given) { + // Expand environment variables. + let expanded = expand_env_vars(given); + + if let Some((scheme, path)) = split_scheme(&expanded) { match Scheme::parse(scheme) { // Ex) `file:///home/ferris/project/scripts/...` or `file:../ferris/` Some(Scheme::File) => { @@ -117,13 +117,13 @@ impl FindLink { // Ex) `https://download.pytorch.org/whl/torch_stable.html` Some(_) => { - let url = Url::parse(given)?; + let url = Url::parse(&expanded)?; Ok(Self::Url(url)) } // Ex) `C:/Users/ferris/wheel-0.42.0.tar.gz` _ => { - let path = PathBuf::from(given); + let path = PathBuf::from(expanded.as_ref()); let path = if path.is_absolute() { path } else { @@ -134,7 +134,7 @@ impl FindLink { } } else { // Ex) `../ferris/` - let path = PathBuf::from(given); + let path = PathBuf::from(expanded.as_ref()); let path = if path.is_absolute() { path } else { @@ -208,8 +208,11 @@ impl EditableRequirement { (given, vec![]) }; + // Expand environment variables. + let expanded = expand_env_vars(requirement); + // Create a `VerbatimUrl` to represent the editable requirement. - let url = if let Some((scheme, path)) = split_scheme(requirement) { + let url = if let Some((scheme, path)) = split_scheme(&expanded) { match Scheme::parse(scheme) { // Ex) `file:///home/ferris/project/scripts/...` or `file:../editable/` Some(Scheme::File) => { @@ -218,27 +221,27 @@ impl EditableRequirement { // Transform, e.g., `/C:/Users/ferris/wheel-0.42.0.tar.gz` to `C:\Users\ferris\wheel-0.42.0.tar.gz`. let path = normalize_url_path(path); - VerbatimUrl::parse_path(path, working_dir.as_ref()) + VerbatimUrl::parse_path(path.as_ref(), working_dir.as_ref()) } // Ex) `https://download.pytorch.org/whl/torch_stable.html` Some(_) => { return Err(RequirementsTxtParserError::UnsupportedUrl( - requirement.to_string(), + expanded.to_string(), )); } // Ex) `C:/Users/ferris/wheel-0.42.0.tar.gz` - _ => VerbatimUrl::parse_path(requirement, working_dir.as_ref()), + _ => VerbatimUrl::parse_path(expanded.as_ref(), working_dir.as_ref()), } } else { // Ex) `../editable/` - VerbatimUrl::parse_path(requirement, working_dir.as_ref()) + VerbatimUrl::parse_path(expanded.as_ref(), working_dir.as_ref()) }; // Create a `PathBuf`. let path = url.to_file_path().map_err(|()| { - RequirementsTxtParserError::InvalidEditablePath(requirement.to_string()) + RequirementsTxtParserError::InvalidEditablePath(expanded.to_string()) })?; // Add the verbatim representation of the URL to the `VerbatimUrl`. @@ -409,7 +412,7 @@ impl RequirementsTxt { start, end, } => { - let filename = expand_path_vars(&filename); + let filename = expand_env_vars(&filename); let sub_file = if filename.starts_with("http://") || filename.starts_with("https://") { PathBuf::from(filename.as_ref()) @@ -447,7 +450,7 @@ impl RequirementsTxt { start, end, } => { - let filename = expand_path_vars(&filename); + let filename = expand_env_vars(&filename); let sub_file = if filename.starts_with("http://") || filename.starts_with("https://") { PathBuf::from(filename.as_ref()) @@ -569,7 +572,7 @@ fn parse_entry( RequirementsTxtStatement::EditableRequirement(editable_requirement) } else if s.eat_if("-i") || s.eat_if("--index-url") { let given = parse_value(content, s, |c: char| !['\n', '\r'].contains(&c))?; - let url = VerbatimUrl::parse(given) + let url = VerbatimUrl::parse_url(given) .map(|url| url.with_given(given.to_owned())) .map_err(|err| RequirementsTxtParserError::Url { source: err, @@ -580,7 +583,7 @@ fn parse_entry( RequirementsTxtStatement::IndexUrl(url) } else if s.eat_if("--extra-index-url") { let given = parse_value(content, s, |c: char| !['\n', '\r'].contains(&c))?; - let url = VerbatimUrl::parse(given) + let url = VerbatimUrl::parse_url(given) .map(|url| url.with_given(given.to_owned())) .map_err(|err| RequirementsTxtParserError::Url { source: err, diff --git a/crates/uv-resolver/src/redirect.rs b/crates/uv-resolver/src/redirect.rs index d6e8699e1f323..f5acd9108fcb3 100644 --- a/crates/uv-resolver/src/redirect.rs +++ b/crates/uv-resolver/src/redirect.rs @@ -44,12 +44,12 @@ mod tests { fn test_apply_redirect() -> Result<(), url::ParseError> { // If there's no `@` in the original representation, we can just append the precise suffix // to the given representation. - let verbatim = VerbatimUrl::parse("https://github.com/flask.git")? + let verbatim = VerbatimUrl::parse_url("https://github.com/flask.git")? .with_given("git+https://github.com/flask.git"); let redirect = Url::parse("https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe")?; - let expected = VerbatimUrl::parse( + let expected = VerbatimUrl::parse_url( "https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe", )? .with_given("https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe"); @@ -58,24 +58,24 @@ mod tests { // If there's an `@` in the original representation, and it's stable between the parsed and // given representations, we preserve everything that precedes the `@` in the precise // representation. - let verbatim = VerbatimUrl::parse("https://github.com/flask.git@main")? + let verbatim = VerbatimUrl::parse_url("https://github.com/flask.git@main")? .with_given("git+https://${DOMAIN}.com/flask.git@main"); let redirect = Url::parse("https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe")?; - let expected = VerbatimUrl::parse( + let expected = VerbatimUrl::parse_url( "https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe", )? .with_given("https://${DOMAIN}.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe"); assert_eq!(apply_redirect(&verbatim, &redirect), expected); // If there's a conflict after the `@`, discard the original representation. - let verbatim = VerbatimUrl::parse("https://github.com/flask.git@main")? + let verbatim = VerbatimUrl::parse_url("https://github.com/flask.git@main")? .with_given("git+https://github.com/flask.git@${TAG}".to_string()); let redirect = Url::parse("https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe")?; - let expected = VerbatimUrl::parse( + let expected = VerbatimUrl::parse_url( "https://github.com/flask.git@b90a4f1f4a370e92054b9cc9db0efcb864f87ebe", )?; assert_eq!(apply_redirect(&verbatim, &redirect), expected); diff --git a/crates/uv-resolver/src/resolver/urls.rs b/crates/uv-resolver/src/resolver/urls.rs index df0354c02fc26..9230f5627eeb8 100644 --- a/crates/uv-resolver/src/resolver/urls.rs +++ b/crates/uv-resolver/src/resolver/urls.rs @@ -200,28 +200,28 @@ mod tests { #[test] fn url_compatibility() -> Result<(), url::ParseError> { // Same repository, same tag. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; - let url = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; + let url = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; assert!(is_equal(&previous, &url)); // Same repository, different tags. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; - let url = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.1")?; + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; + let url = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.1")?; assert!(!is_equal(&previous, &url)); // Same repository (with and without `.git`), same tag. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject@v1.0")?; - let url = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject@v1.0")?; + let url = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; assert!(is_equal(&previous, &url)); // Same repository, no tag on the previous URL. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git")?; - let url = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git")?; + let url = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; assert!(!is_equal(&previous, &url)); // Same repository, tag on the previous URL, no tag on the overriding URL. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; - let url = VerbatimUrl::parse("git+https://example.com/MyProject.git")?; + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; + let url = VerbatimUrl::parse_url("git+https://example.com/MyProject.git")?; assert!(!is_equal(&previous, &url)); Ok(()) @@ -230,29 +230,29 @@ mod tests { #[test] fn url_precision() -> Result<(), url::ParseError> { // Same repository, no tag on the previous URL, non-SHA on the overriding URL. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git")?; - let url = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git")?; + let url = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; assert!(!is_precise(&previous, &url)); // Same repository, no tag on the previous URL, SHA on the overriding URL. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git")?; - let url = VerbatimUrl::parse( + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git")?; + let url = VerbatimUrl::parse_url( "git+https://example.com/MyProject.git@c3cd550a7a7c41b2c286ca52fbb6dec5fea195ef", )?; assert!(is_precise(&previous, &url)); // Same repository, tag on the previous URL, SHA on the overriding URL. - let previous = VerbatimUrl::parse("git+https://example.com/MyProject.git@v1.0")?; - let url = VerbatimUrl::parse( + let previous = VerbatimUrl::parse_url("git+https://example.com/MyProject.git@v1.0")?; + let url = VerbatimUrl::parse_url( "git+https://example.com/MyProject.git@c3cd550a7a7c41b2c286ca52fbb6dec5fea195ef", )?; assert!(is_precise(&previous, &url)); // Same repository, SHA on the previous URL, different SHA on the overriding URL. - let previous = VerbatimUrl::parse( + let previous = VerbatimUrl::parse_url( "git+https://example.com/MyProject.git@5ae5980c885e350a34ca019a84ba14a2a228d262", )?; - let url = VerbatimUrl::parse( + let url = VerbatimUrl::parse_url( "git+https://example.com/MyProject.git@c3cd550a7a7c41b2c286ca52fbb6dec5fea195ef", )?; assert!(!is_precise(&previous, &url)); diff --git a/foo bar/requirements.txt b/foo bar/requirements.txt new file mode 100644 index 0000000000000..dde585498da74 --- /dev/null +++ b/foo bar/requirements.txt @@ -0,0 +1 @@ +black @ file://${PROJECT_ROOT}/../scripts/editable-installs/black_editable diff --git a/pyproject.toml b/pyproject.toml index 0a32a0625b644..fdbb9825fce5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ classifiers = [ "Topic :: Software Development :: Libraries", ] readme = "README.md" +dependencies = ["-e ./scripts/editable-installs/black_editable"] [project.urls] Repository = "https://github.com/astral-sh/uv"