From 9754199f4f47cdd7469eddcfd1515b04f0df9c79 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 09:32:45 +0200 Subject: [PATCH 01/15] fix: url parsing for namelessmatchspec and cleanup functions --- .../src/match_spec/parse.rs | 227 +++++++++++------- ...ts__test_nameless_from_string_Lenient.snap | 3 +- ...sts__test_nameless_from_string_Strict.snap | 3 +- 3 files changed, 147 insertions(+), 86 deletions(-) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 24131cefb..2e6ca3b4c 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -79,7 +79,8 @@ pub enum ParseMatchSpecError { InvalidVersionAndBuild(String), /// Invalid build string - #[error("The build string '{0}' is not valid, it can only contain alphanumeric characters and underscores")] + #[error("The build string '{0}' is not valid, it can only contain alphanumeric characters and underscores" + )] InvalidBuildString(String), /// Invalid version spec @@ -107,7 +108,7 @@ impl FromStr for MatchSpec { type Err = ParseMatchSpecError; fn from_str(s: &str) -> Result { - Self::from_str(s, ParseStrictness::Lenient) + Self::from_str(s, Lenient) } } @@ -187,7 +188,7 @@ fn parse_bracket_list(input: &str) -> Result, ParseMatchSpecError separated_pair(parse_key, char('='), parse_value)(input) } - /// Parses a list of `key=value` pairs seperate by commas + /// Parses a list of `key=value` pairs separate by commas fn parse_key_value_list(input: &str) -> IResult<&str, Vec<(&str, &str)>> { separated_list0(whitespace_enclosed(char(',')), parse_key_value)(input) } @@ -249,6 +250,24 @@ fn parse_bracket_vec_into_components( ); } "fn" => match_spec.file_name = Some(value.to_string()), + "url" => { + // Is the spec an url, parse it as an url + let url = if parse_scheme(value).is_some() { + Url::parse(value)? + } + // 2 Is the spec a path, parse it as an url + else if is_path(value) { + let path = Utf8TypedPath::from(value); + file_url::file_path_to_url(path) + .map_err(|_error| ParseMatchSpecError::InvalidPackagePathOrUrl)? + } else { + return Err(ParseMatchSpecError::InvalidPackagePathOrUrl); + }; + + match_spec.url = Some(url); + } + // TODO: Still need to add `track_features`, `features`, `license` and `license_family` + // to the match spec. _ => Err(ParseMatchSpecError::InvalidBracketKey(key.to_owned()))?, } } @@ -256,6 +275,24 @@ fn parse_bracket_vec_into_components( Ok(match_spec) } +/// Parses an url or path like string into an url. +fn parse_url_like(input: &str) -> Result, ParseMatchSpecError> { + // Is the spec an url, parse it as an url + if parse_scheme(input).is_some() { + return Url::parse(input) + .map(Some) + .map_err(ParseMatchSpecError::from); + } + // Is the spec a path, parse it as an url + if is_path(input) { + let path = Utf8TypedPath::from(input); + return file_url::file_path_to_url(path) + .map(Some) + .map_err(|_err| ParseMatchSpecError::InvalidPackagePathOrUrl); + } + Ok(None) +} + /// Strip the package name from the input. fn strip_package_name(input: &str) -> Result<(PackageName, &str), ParseMatchSpecError> { let (rest, package_name) = @@ -353,12 +390,47 @@ fn split_version_and_build( )), } } +/// Parse version and build string. +fn parse_version_and_build( + input: &str, + strictness: ParseStrictness, +) -> Result<(Option, Option), ParseMatchSpecError> { + if input.find('[').is_some() { + return Err(ParseMatchSpecError::MultipleBracketSectionsNotAllowed); + } + + let (version_str, build_str) = split_version_and_build(input, strictness)?; + + let version_str = if version_str.find(char::is_whitespace).is_some() { + Cow::Owned(version_str.replace(char::is_whitespace, "")) + } else { + Cow::Borrowed(version_str) + }; + + // Under certain circumstances we strip the `=` or `==` parts of the version + // string. See the function for more info. + let version_str = optionally_strip_equals(&version_str, build_str, strictness); + + // Parse the version spec + let version = Some( + VersionSpec::from_str(version_str.as_ref(), strictness) + .map_err(ParseMatchSpecError::InvalidVersionSpec)?, + ); + + // Parse the build string + let mut build = None; + if let Some(build_str) = build_str { + build = Some(StringMatcher::from_str(build_str)?); + } + + Ok((version, build)) +} impl FromStr for NamelessMatchSpec { type Err = ParseMatchSpecError; fn from_str(input: &str) -> Result { - Self::from_str(input, ParseStrictness::Lenient) + Self::from_str(input, Lenient) } } @@ -367,37 +439,24 @@ impl NamelessMatchSpec { pub fn from_str(input: &str, strictness: ParseStrictness) -> Result { // Strip off brackets portion let (input, brackets) = strip_brackets(input.trim())?; + let input = input.trim(); + + // Parse url or path spec + if let Some(url) = parse_url_like(input)? { + return Ok(NamelessMatchSpec { + url: Some(url), + ..NamelessMatchSpec::default() + }); + } + let mut match_spec = parse_bracket_vec_into_components(brackets, NamelessMatchSpec::default(), strictness)?; // Get the version and optional build string - let input = input.trim(); if !input.is_empty() { - if input.find('[').is_some() { - return Err(ParseMatchSpecError::MultipleBracketSectionsNotAllowed); - } - - let (version_str, build_str) = split_version_and_build(input, strictness)?; - - let version_str = if version_str.find(char::is_whitespace).is_some() { - Cow::Owned(version_str.replace(char::is_whitespace, "")) - } else { - Cow::Borrowed(version_str) - }; - - // Under certum circumstances we strip the `=` part of the version string. See - // the function documentation for more info. - let version_str = optionally_strip_equals(&version_str, build_str, strictness); - - // Parse the version spec - match_spec.version = Some( - VersionSpec::from_str(version_str.as_ref(), strictness) - .map_err(ParseMatchSpecError::InvalidVersionSpec)?, - ); - - if let Some(build) = build_str { - match_spec.build = Some(StringMatcher::from_str(build)?); - } + let (version, build) = parse_version_and_build(input, strictness)?; + match_spec.version = version; + match_spec.build = build; } Ok(match_spec) @@ -414,30 +473,8 @@ fn matchspec_parser( let (input, _comment) = strip_comment(input); let (input, _if_clause) = strip_if(input); - // 2.a Is the spec an url, parse it as an url - if parse_scheme(input).is_some() { - let url = Url::parse(input)?; - - let archive = ArchiveIdentifier::try_from_url(&url); - let name = archive.and_then(|a| a.try_into().ok()); - - // TODO: This should also work without a proper name from the url filename - if name.is_none() { - return Err(ParseMatchSpecError::MissingPackageName); - } - - return Ok(MatchSpec { - url: Some(url), - name, - ..MatchSpec::default() - }); - } - // 2.b Is the spec a path, parse it as an url - if is_path(input) { - let path = Utf8TypedPath::from(input); - let url = file_url::file_path_to_url(path) - .map_err(|_error| ParseMatchSpecError::InvalidPackagePathOrUrl)?; - + // 2. parse as url + if let Some(url) = parse_url_like(input)? { let archive = ArchiveIdentifier::try_from_url(&url); let name = archive.and_then(|a| a.try_into().ok()); @@ -446,6 +483,9 @@ fn matchspec_parser( return Err(ParseMatchSpecError::MissingPackageName); } + // Only return the 'url' and 'name' to avoid miss parsing the rest of the + // information. e.g. when a version is provided in the url is not the + // actual version this might be a problem when solving. return Ok(MatchSpec { url: Some(url), name, @@ -459,7 +499,7 @@ fn matchspec_parser( parse_bracket_vec_into_components(brackets, NamelessMatchSpec::default(), strictness)?; // 4. Strip off parens portion - // TODO: What is this? I've never seen in + // TODO: What is this? I've never seen it // 5. Strip of '::' channel and namespace let mut input_split = input.split(':').fuse(); @@ -500,34 +540,12 @@ fn matchspec_parser( let (name, input) = strip_package_name(input)?; let mut match_spec = MatchSpec::from_nameless(nameless_match_spec, Some(name)); - // Step 7. Otherwise sort our version + build + // Step 7. Otherwise, sort our version + build let input = input.trim(); if !input.is_empty() { - if input.find('[').is_some() { - return Err(ParseMatchSpecError::MultipleBracketSectionsNotAllowed); - } - - let (version_str, build_str) = split_version_and_build(input, strictness)?; - - let version_str = if version_str.find(char::is_whitespace).is_some() { - Cow::Owned(version_str.replace(char::is_whitespace, "")) - } else { - Cow::Borrowed(version_str) - }; - - // Under certain circumstantances we strip the `=` or `==` parts of the version - // string. See the function for more info. - let version_str = optionally_strip_equals(&version_str, build_str, strictness); - - // Parse the version spec - match_spec.version = Some( - VersionSpec::from_str(version_str.as_ref(), strictness) - .map_err(ParseMatchSpecError::InvalidVersionSpec)?, - ); - - if let Some(build) = build_str { - match_spec.build = Some(StringMatcher::from_str(build)?); - } + let (version, build) = parse_version_and_build(input, strictness)?; + match_spec.version = version; + match_spec.build = build; } Ok(match_spec) @@ -535,7 +553,7 @@ fn matchspec_parser( /// HERE BE DRAGONS! /// -/// In some circumstainces we strip the `=` or `==` parts of the version string. +/// In some circumstances we strip the `=` or `==` parts of the version string. /// This is for conda legacy reasons. This function implements that behavior and /// returns the stripped/updated version. /// @@ -785,6 +803,44 @@ mod tests { ); } + #[test] + fn test_nameless_url() { + let url_str = + "https://conda.anaconda.org/conda-forge/linux-64/py-rattler-0.6.1-py39h8169da8_0.conda"; + let url = Url::parse(url_str).unwrap(); + let spec1 = NamelessMatchSpec::from_str(url_str, Strict).unwrap(); + assert_eq!(spec1.url, Some(url.clone())); + + let spec_with_brackets = + NamelessMatchSpec::from_str(format!("[url={}]", url_str).as_str(), Strict).unwrap(); + assert_eq!(spec_with_brackets.url, Some(url)); + } + + #[test] + fn test_nameless_url_path() { + // Windows + let win_path_str = "C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2"; + let spec = NamelessMatchSpec::from_str(win_path_str, Strict).unwrap(); + let win_path = file_url::file_path_to_url(win_path_str).unwrap(); + assert_eq!(spec.url, Some(win_path.clone())); + + let spec_with_brackets = + NamelessMatchSpec::from_str(format!("[url={}]", win_path_str).as_str(), Strict) + .unwrap(); + assert_eq!(spec_with_brackets.url, Some(win_path)); + + // Unix + let unix_path_str = "/users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2"; + let spec = NamelessMatchSpec::from_str(unix_path_str, Strict).unwrap(); + let unix_path = file_url::file_path_to_url(unix_path_str).unwrap(); + assert_eq!(spec.url, Some(unix_path.clone())); + + let spec_with_brackets = + NamelessMatchSpec::from_str(format!("[url={}]", unix_path_str).as_str(), Strict) + .unwrap(); + assert_eq!(spec_with_brackets.url, Some(unix_path)); + } + #[test] fn test_hash_spec() { let spec = MatchSpec::from_str("conda-forge::foo[md5=1234567890]", Strict); @@ -928,7 +984,10 @@ mod tests { // A list of matchspecs to parse. // Please keep this list sorted. - let specs = ["2.7|>=3.6"]; + let specs = [ + "2.7|>=3.6", + "https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2", + ]; let evaluated: BTreeMap<_, _> = specs .iter() @@ -1036,7 +1095,7 @@ mod tests { let spec = MatchSpec::from_str("/home/user/Downloads/package", Strict).unwrap_err(); assert_matches!(spec, ParseMatchSpecError::MissingPackageName); - let err = MatchSpec::from_str("http://username@", Strict).expect_err("Invalid url"); + let err = MatchSpec::from_str("https://username@", Strict).expect_err("Invalid url"); assert_eq!(err.to_string(), "invalid package spec url"); let err = MatchSpec::from_str("bla/bla", Strict) diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Lenient.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Lenient.snap index e5faabbb8..81c734197 100644 --- a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Lenient.snap +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Lenient.snap @@ -1,7 +1,8 @@ --- source: crates/rattler_conda_types/src/match_spec/parse.rs -assertion_line: 930 expression: evaluated --- 2.7|>=3.6: version: "==2.7|>=3.6" +"https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2": + url: "https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2" diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Strict.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Strict.snap index e5faabbb8..81c734197 100644 --- a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Strict.snap +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_nameless_from_string_Strict.snap @@ -1,7 +1,8 @@ --- source: crates/rattler_conda_types/src/match_spec/parse.rs -assertion_line: 930 expression: evaluated --- 2.7|>=3.6: version: "==2.7|>=3.6" +"https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2": + url: "https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2" From 3746209337dcf3187f43fdb3f99a48a0ebcd089a Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 11:29:39 +0200 Subject: [PATCH 02/15] fix: parse url as conda matchspec url --- .../rattler_conda_types/src/match_spec/mod.rs | 55 +++++++++++++++++++ .../src/match_spec/parse.rs | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/crates/rattler_conda_types/src/match_spec/mod.rs b/crates/rattler_conda_types/src/match_spec/mod.rs index 95f4c2d0b..bfc626883 100644 --- a/crates/rattler_conda_types/src/match_spec/mod.rs +++ b/crates/rattler_conda_types/src/match_spec/mod.rs @@ -144,6 +144,7 @@ pub struct MatchSpec { #[serde_as(as = "Option>")] pub sha256: Option, /// The url of the package + #[serde(deserialize_with = "deserialize_url")] pub url: Option, } @@ -256,6 +257,7 @@ pub struct NamelessMatchSpec { #[serde_as(as = "Option>")] pub sha256: Option, /// The url of the package + #[serde(deserialize_with = "deserialize_url")] pub url: Option, } @@ -347,6 +349,19 @@ where } } +/// Deserialize matchspec url from string +fn deserialize_url<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s: Option = Option::deserialize(deserializer)?; + + match s { + None => Ok(None), + Some(s) => parse::parse_url_like(s.as_ref()).map_err(serde::de::Error::custom), + } +} + /// A trait that defines the behavior of matching a spec against a record. pub trait Matches { /// Match a [`MatchSpec`] against a record. @@ -470,6 +485,7 @@ impl Matches for NamelessMatchSpec { #[cfg(test)] mod tests { + use serde_json::json; use std::str::FromStr; use rattler_digest::{parse_digest_from_hex, Md5, Sha256}; @@ -624,4 +640,43 @@ mod tests { .collect::>() .join("\n")); } + + #[test] + fn test_deserialize_url_in_nameless_match_spec() { + // Test with a valid URL + let json = json!({ + "url": "https://test.com/conda/file-0.1-bla_1.conda" + }); + let result: Result = serde_json::from_value(json); + assert_eq!( + result.unwrap().url.unwrap().as_str(), + "https://test.com/conda/file-0.1-bla_1.conda" + ); + + // Test with a valid file path + let json = json!({ + "url": "/home/user/file.conda" + }); + let result: Result = serde_json::from_value(json); + assert_eq!( + result.unwrap().url.unwrap().as_str(), + "file:///home/user/file.conda" + ); + + // Test with a valid windows path + let json = json!({ + "url": "C:\\Users\\user\\file.conda" + }); + let result: Result = serde_json::from_value(json); + assert_eq!( + result.unwrap().url.unwrap().as_str(), + "file:///C:/Users/user/file.conda" + ); + // Test with None + let json = json!({ + "url": null + }); + let result: Result = serde_json::from_value(json); + assert!(result.unwrap().url.is_none()); + } } diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 2e6ca3b4c..211dc2044 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -276,7 +276,7 @@ fn parse_bracket_vec_into_components( } /// Parses an url or path like string into an url. -fn parse_url_like(input: &str) -> Result, ParseMatchSpecError> { +pub fn parse_url_like(input: &str) -> Result, ParseMatchSpecError> { // Is the spec an url, parse it as an url if parse_scheme(input).is_some() { return Url::parse(input) From 34b16071f3b3ba13c1d0c40bf0f1cf4cabfc7fb7 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 15:17:54 +0200 Subject: [PATCH 03/15] fix: namespace parsing, and channel as url parsing --- crates/rattler_conda_types/src/channel/mod.rs | 8 ++ .../src/match_spec/parse.rs | 132 +++++++++++------- ...arse__tests__test_from_string_Lenient.snap | 65 +++++---- ...parse__tests__test_from_string_Strict.snap | 61 ++++---- 4 files changed, 154 insertions(+), 112 deletions(-) diff --git a/crates/rattler_conda_types/src/channel/mod.rs b/crates/rattler_conda_types/src/channel/mod.rs index f0d0fb79d..5d43f4f66 100644 --- a/crates/rattler_conda_types/src/channel/mod.rs +++ b/crates/rattler_conda_types/src/channel/mod.rs @@ -198,6 +198,10 @@ impl Channel { } } } else { + // Validate that the channel is a valid name + if channel.contains(|c| c == ':' || c == '\\') { + return Err(ParseChannelError::InvalidName(channel.to_owned())); + } Channel { platforms, ..Channel::from_name(channel, config) @@ -367,6 +371,10 @@ pub enum ParseChannelError { #[error("invalid path '{0}'")] InvalidPath(String), + /// Error when the channel name is invalid. + #[error("invalid channel name: '{0}'")] + InvalidName(String), + /// The root directory is not an absolute path #[error("root directory from channel config is not an absolute path")] NonAbsoluteRootDir(PathBuf), diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index e7536f0d8..6caa328d4 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -54,10 +54,6 @@ pub enum ParseMatchSpecError { #[error("invalid bracket")] InvalidBracket, - /// Invalid number of colons in match spec - #[error("invalid number of colons")] - InvalidNumberOfColons, - /// Invalid channel provided in match spec #[error("invalid channel")] ParseChannelError(#[from] ParseChannelError), @@ -277,6 +273,11 @@ fn parse_bracket_vec_into_components( /// Parses an url or path like string into an url. pub fn parse_url_like(input: &str) -> Result, ParseMatchSpecError> { + // Skip if channel is provided, this avoids parsing namespaces as urls + if input.contains("::") { + return Ok(None); + } + // Is the spec an url, parse it as an url if parse_scheme(input).is_some() { return Url::parse(input) @@ -473,49 +474,49 @@ fn matchspec_parser( let (input, _comment) = strip_comment(input); let (input, _if_clause) = strip_if(input); - // 2. parse as url - if let Some(url) = parse_url_like(input)? { - let archive = ArchiveIdentifier::try_from_url(&url); - let name = archive.and_then(|a| a.try_into().ok()); - - // TODO: This should also work without a proper name from the url filename - if name.is_none() { - return Err(ParseMatchSpecError::MissingPackageName); - } - - // Only return the 'url' and 'name' to avoid miss parsing the rest of the - // information. e.g. when a version is provided in the url is not the - // actual version this might be a problem when solving. - return Ok(MatchSpec { - url: Some(url), - name, - ..MatchSpec::default() - }); - } - - // 3. Strip off brackets portion + // 2. Strip off brackets portion let (input, brackets) = strip_brackets(input.trim())?; let mut nameless_match_spec = parse_bracket_vec_into_components(brackets, NamelessMatchSpec::default(), strictness)?; - // 4. Strip off parens portion + // 3. Strip off parens portion // TODO: What is this? I've never seen it - // 5. Strip of '::' channel and namespace - let mut input_split = input.split(':').fuse(); - let (input, namespace, channel_str) = match ( - input_split.next(), - input_split.next(), - input_split.next(), - input_split.next(), - ) { - (Some(input), None, _, _) => (input, None, None), - (Some(namespace), Some(input), None, _) => (input, Some(namespace), None), - (Some(channel_str), Some(namespace), Some(input), None) => { - (input, Some(namespace), Some(channel_str)) + // 4. Parse as url + if nameless_match_spec.url.is_none() { + if let Some(url) = parse_url_like(&input)? { + let archive = ArchiveIdentifier::try_from_url(&url); + let name = archive.and_then(|a| a.try_into().ok()); + + // TODO: This should also work without a proper name from the url filename + if name.is_none() { + return Err(ParseMatchSpecError::MissingPackageName); + } + + // Only return the 'url' and 'name' to avoid miss parsing the rest of the + // information. e.g. when a version is provided in the url is not the + // actual version this might be a problem when solving. + return Ok(MatchSpec { + url: Some(url), + name, + ..MatchSpec::default() + }); } - _ => return Err(ParseMatchSpecError::InvalidNumberOfColons), - }; + } + + // 5. Strip of ':' to find channel and namespace + // This assumes the [*] portions is stripped off, and then strip reverse to + // ignore the first colon As that might be in the channel url. + let mut input_split = input.rsplitn(3, ':').fuse(); + let (input, namespace, channel_str) = + match (input_split.next(), input_split.next(), input_split.next()) { + (Some(input), None, _) => (input, None, None), + (Some(input), Some(namespace), None) => (input, Some(namespace), None), + (Some(input), Some(namespace), Some(channel_str)) => { + (input, Some(namespace), Some(channel_str)) + } + (None, _, _) => ("", None, None), + }; nameless_match_spec.namespace = namespace .map(str::trim) @@ -527,13 +528,9 @@ fn matchspec_parser( let channel_config = ChannelConfig::default_with_root_dir( std::env::current_dir().expect("Could not get current directory"), ); - if let Some((channel, subdir)) = channel_str.rsplit_once('/') { - nameless_match_spec.channel = Some(Channel::from_str(channel, &channel_config)?.into()); - nameless_match_spec.subdir = Some(subdir.to_string()); - } else { - nameless_match_spec.channel = - Some(Channel::from_str(channel_str, &channel_config)?.into()); - } + // No subdir split, assume subdir is not part of the channel url. + let channel = Channel::from_str(channel_str, &channel_config)?; + nameless_match_spec.channel = Some(channel.into()); } // Step 6. Strip off the package name from the input @@ -609,8 +606,8 @@ fn optionally_strip_equals<'a>( #[cfg(test)] mod tests { - use std::{collections::BTreeMap, str::FromStr, sync::Arc}; - + use std::{str::FromStr, sync::Arc}; + use indexmap::IndexMap; use assert_matches::assert_matches; use rattler_digest::{parse_digest_from_hex, Md5, Sha256}; use rstest::rstest; @@ -624,7 +621,7 @@ mod tests { }; use crate::{ match_spec::parse::parse_bracket_list, BuildNumberSpec, Channel, ChannelConfig, - NamelessMatchSpec, ParseStrictness, ParseStrictness::*, VersionSpec, + NamelessMatchSpec, ParseChannelError, ParseStrictness, ParseStrictness::*, VersionSpec, }; fn channel_config() -> ChannelConfig { @@ -952,9 +949,10 @@ mod tests { "python ==2.7.*.*|>=3.6", "python=3.9", "python=*", + "https://software.repos.intel.com/python/conda::python[version=3.9]", ]; - let evaluated: BTreeMap<_, _> = specs + let evaluated: IndexMap<_, _> = specs .iter() .map(|spec| { ( @@ -989,7 +987,7 @@ mod tests { "https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2", ]; - let evaluated: BTreeMap<_, _> = specs + let evaluated: IndexMap<_, _> = specs .iter() .map(|spec| { ( @@ -1032,9 +1030,22 @@ mod tests { } #[test] - fn test_invalid_number_of_colons() { + fn test_invalid_channel_name() { let spec = MatchSpec::from_str("conda-forge::::foo[version=\"1.0.*\"]", Strict); - assert_matches!(spec, Err(ParseMatchSpecError::InvalidNumberOfColons)); + assert_matches!( + spec, + Err(ParseMatchSpecError::ParseChannelError( + ParseChannelError::InvalidName(_) + )) + ); + + let spec = MatchSpec::from_str("conda-forge\\::foo[version=\"1.0.*\"]", Strict); + assert_matches!( + spec, + Err(ParseMatchSpecError::ParseChannelError( + ParseChannelError::InvalidName(_) + )) + ); } #[test] @@ -1049,6 +1060,19 @@ mod tests { assert!(spec.namespace.is_none()); } + #[test] + fn test_namespace() { + // Test with url channel and url in brackets + let spec = MatchSpec::from_str("https://a.b.c/conda-forge:namespace:foo[url=https://a.b/c/d/p-1-b_0.conda]", Strict).unwrap(); + assert_eq!(spec.namespace, Some("namespace".to_owned())); + assert_eq!(spec.name, Some("foo".parse().unwrap())); + assert_eq!(spec.channel.unwrap().name(), "conda-forge"); + assert_eq!( + spec.url, + Some(Url::parse("https://a.b/c/d/p-1-b_0.conda").unwrap()) + ); + } + #[test] fn test_parsing_url() { let spec = MatchSpec::from_str( diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap index d8727e80f..dc8403e36 100644 --- a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap @@ -1,33 +1,14 @@ --- source: crates/rattler_conda_types/src/match_spec/parse.rs -assertion_line: 917 expression: evaluated --- -/home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2: - name: foo - url: "file:///home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" -"C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2": - name: foo - url: "file:///C:/Users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" blas *.* mkl: name: blas version: "*" build: mkl -"conda-forge::foo[version=1.0.*, build_number=\">6\"]": - name: foo - version: 1.0.* - build_number: - op: Gt - rhs: 6 - channel: - base_url: "https://conda.anaconda.org/conda-forge/" - name: conda-forge -"conda-forge::foo[version=1.0.*]": +"C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2": name: foo - version: 1.0.* - channel: - base_url: "https://conda.anaconda.org/conda-forge/" - name: conda-forge + url: "file:///C:/Users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" foo=1.0=py27_0: name: foo version: "==1.0" @@ -46,15 +27,6 @@ python 3.8.* *_cpython: name: python version: 3.8.* build: "*_cpython" -python ==2.7.*.*|>=3.6: - name: python - version: 2.7.*|>=3.6 -python=*: - name: python - version: "*" -python=3.9: - name: python - version: 3.9.* pytorch=*=cuda*: name: pytorch version: "*" @@ -62,3 +34,36 @@ pytorch=*=cuda*: "x264 >=1!164.3095,<1!165": name: x264 version: ">=1!164.3095,<1!165" +/home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2: + name: foo + url: "file:///home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" +"conda-forge::foo[version=1.0.*]": + name: foo + version: 1.0.* + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge +"conda-forge::foo[version=1.0.*, build_number=\">6\"]": + name: foo + version: 1.0.* + build_number: + op: Gt + rhs: 6 + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge +python ==2.7.*.*|>=3.6: + name: python + version: 2.7.*|>=3.6 +python=3.9: + name: python + version: 3.9.* +python=*: + name: python + version: "*" +"https://software.repos.intel.com/python/conda::python[version=3.9]": + name: python + version: "==3.9" + channel: + base_url: "https://software.repos.intel.com/python/conda/" + name: python/conda diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap index 233dee823..c4909deb9 100644 --- a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap @@ -1,33 +1,14 @@ --- source: crates/rattler_conda_types/src/match_spec/parse.rs -assertion_line: 915 expression: evaluated --- -/home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2: - name: foo - url: "file:///home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" -"C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2": - name: foo - url: "file:///C:/Users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" blas *.* mkl: name: blas version: "*" build: mkl -"conda-forge::foo[version=1.0.*, build_number=\">6\"]": - name: foo - version: 1.0.* - build_number: - op: Gt - rhs: 6 - channel: - base_url: "https://conda.anaconda.org/conda-forge/" - name: conda-forge -"conda-forge::foo[version=1.0.*]": +"C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2": name: foo - version: 1.0.* - channel: - base_url: "https://conda.anaconda.org/conda-forge/" - name: conda-forge + url: "file:///C:/Users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" foo=1.0=py27_0: error: "The build string '=py27_0' is not valid, it can only contain alphanumeric characters and underscores" foo==1.0=py27_0: @@ -42,15 +23,39 @@ python 3.8.* *_cpython: name: python version: 3.8.* build: "*_cpython" -python ==2.7.*.*|>=3.6: - error: "invalid version constraint: regex constraints are not supported" -python=*: - error: "invalid version constraint: '*' is incompatible with '=' operator'" -python=3.9: - name: python - version: 3.9.* pytorch=*=cuda*: error: "The build string '=cuda*' is not valid, it can only contain alphanumeric characters and underscores" "x264 >=1!164.3095,<1!165": name: x264 version: ">=1!164.3095,<1!165" +/home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2: + name: foo + url: "file:///home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" +"conda-forge::foo[version=1.0.*]": + name: foo + version: 1.0.* + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge +"conda-forge::foo[version=1.0.*, build_number=\">6\"]": + name: foo + version: 1.0.* + build_number: + op: Gt + rhs: 6 + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge +python ==2.7.*.*|>=3.6: + error: "invalid version constraint: regex constraints are not supported" +python=3.9: + name: python + version: 3.9.* +python=*: + error: "invalid version constraint: '*' is incompatible with '=' operator'" +"https://software.repos.intel.com/python/conda::python[version=3.9]": + name: python + version: "==3.9" + channel: + base_url: "https://software.repos.intel.com/python/conda/" + name: python/conda From d33d726bf986cc6d5c785905d8e72008ec1627d3 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 15:18:13 +0200 Subject: [PATCH 04/15] test: add snapshot --- ..._spec__parse__tests__test_from_string.snap | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string.snap diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string.snap new file mode 100644 index 000000000..e620acf26 --- /dev/null +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string.snap @@ -0,0 +1,37 @@ +--- +source: crates/rattler_conda_types/src/match_spec/parse.rs +expression: evaluated +--- +/home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2: + name: foo + url: "file:///home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" +"C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2": + name: foo + url: "file:///C:/Users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2" +"conda-forge::foo[version=1.0.*, build_number=\">6\"]": + name: foo + version: 1.0.* + build_number: + op: Gt + rhs: 6 + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge +"conda-forge::foo[version=1.0.*]": + name: foo + version: 1.0.* + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge +"https://conda.anaconda.org/conda-forge/linux-64/py-rattler-0.6.1-py39h8169da8_0.conda": + name: py-rattler + url: "https://conda.anaconda.org/conda-forge/linux-64/py-rattler-0.6.1-py39h8169da8_0.conda" +"https://repo.prefix.dev/ruben-arts/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2": + name: boost-cpp + url: "https://repo.prefix.dev/ruben-arts/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2" +"https://software.repos.intel.com/python/conda::python[version=3.9]": + name: python + version: "==3.9" + channel: + base_url: "https://software.repos.intel.com/python/conda/" + name: python/conda From 5e3ed433aeb51dd4a0a0272d752dc1ab407d01e8 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 15:22:56 +0200 Subject: [PATCH 05/15] fmt --- crates/rattler_conda_types/src/match_spec/parse.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 6caa328d4..310f7611e 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -606,13 +606,13 @@ fn optionally_strip_equals<'a>( #[cfg(test)] mod tests { - use std::{str::FromStr, sync::Arc}; - use indexmap::IndexMap; use assert_matches::assert_matches; + use indexmap::IndexMap; use rattler_digest::{parse_digest_from_hex, Md5, Sha256}; use rstest::rstest; use serde::Serialize; use smallvec::smallvec; + use std::{str::FromStr, sync::Arc}; use url::Url; use super::{ @@ -1063,7 +1063,11 @@ mod tests { #[test] fn test_namespace() { // Test with url channel and url in brackets - let spec = MatchSpec::from_str("https://a.b.c/conda-forge:namespace:foo[url=https://a.b/c/d/p-1-b_0.conda]", Strict).unwrap(); + let spec = MatchSpec::from_str( + "https://a.b.c/conda-forge:namespace:foo[url=https://a.b/c/d/p-1-b_0.conda]", + Strict, + ) + .unwrap(); assert_eq!(spec.namespace, Some("namespace".to_owned())); assert_eq!(spec.name, Some("foo".parse().unwrap())); assert_eq!(spec.channel.unwrap().name(), "conda-forge"); From ecc6a85a0dc28af5533a93f022a68e2646874135 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 15:32:38 +0200 Subject: [PATCH 06/15] fix: remove url deserialization from path --- .../rattler_conda_types/src/match_spec/mod.rs | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/crates/rattler_conda_types/src/match_spec/mod.rs b/crates/rattler_conda_types/src/match_spec/mod.rs index bfc626883..566b6c3bf 100644 --- a/crates/rattler_conda_types/src/match_spec/mod.rs +++ b/crates/rattler_conda_types/src/match_spec/mod.rs @@ -144,7 +144,6 @@ pub struct MatchSpec { #[serde_as(as = "Option>")] pub sha256: Option, /// The url of the package - #[serde(deserialize_with = "deserialize_url")] pub url: Option, } @@ -257,7 +256,6 @@ pub struct NamelessMatchSpec { #[serde_as(as = "Option>")] pub sha256: Option, /// The url of the package - #[serde(deserialize_with = "deserialize_url")] pub url: Option, } @@ -349,19 +347,6 @@ where } } -/// Deserialize matchspec url from string -fn deserialize_url<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let s: Option = Option::deserialize(deserializer)?; - - match s { - None => Ok(None), - Some(s) => parse::parse_url_like(s.as_ref()).map_err(serde::de::Error::custom), - } -} - /// A trait that defines the behavior of matching a spec against a record. pub trait Matches { /// Match a [`MatchSpec`] against a record. @@ -640,43 +625,4 @@ mod tests { .collect::>() .join("\n")); } - - #[test] - fn test_deserialize_url_in_nameless_match_spec() { - // Test with a valid URL - let json = json!({ - "url": "https://test.com/conda/file-0.1-bla_1.conda" - }); - let result: Result = serde_json::from_value(json); - assert_eq!( - result.unwrap().url.unwrap().as_str(), - "https://test.com/conda/file-0.1-bla_1.conda" - ); - - // Test with a valid file path - let json = json!({ - "url": "/home/user/file.conda" - }); - let result: Result = serde_json::from_value(json); - assert_eq!( - result.unwrap().url.unwrap().as_str(), - "file:///home/user/file.conda" - ); - - // Test with a valid windows path - let json = json!({ - "url": "C:\\Users\\user\\file.conda" - }); - let result: Result = serde_json::from_value(json); - assert_eq!( - result.unwrap().url.unwrap().as_str(), - "file:///C:/Users/user/file.conda" - ); - // Test with None - let json = json!({ - "url": null - }); - let result: Result = serde_json::from_value(json); - assert!(result.unwrap().url.is_none()); - } } From 918a91d72bd0cb50e33b6f138d114e9c33df1014 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 16:27:10 +0200 Subject: [PATCH 07/15] feat: add subdir to the bracket parser --- crates/rattler_conda_types/src/channel/mod.rs | 6 ++++++ crates/rattler_conda_types/src/match_spec/mod.rs | 5 ++--- crates/rattler_conda_types/src/match_spec/parse.rs | 3 ++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/rattler_conda_types/src/channel/mod.rs b/crates/rattler_conda_types/src/channel/mod.rs index 5d43f4f66..be527c532 100644 --- a/crates/rattler_conda_types/src/channel/mod.rs +++ b/crates/rattler_conda_types/src/channel/mod.rs @@ -549,6 +549,12 @@ mod tests { channel.base_url().to_string(), "https://conda.anaconda.org/conda-forge/" ); + + let channel = Channel::from_str( + "https://conda.anaconda.org/conda-forge/label/rust_dev", + &config, + ); + assert_eq!(channel.unwrap().name(), "conda-forge/label/rust_dev",); } #[test] diff --git a/crates/rattler_conda_types/src/match_spec/mod.rs b/crates/rattler_conda_types/src/match_spec/mod.rs index 566b6c3bf..630fae4fb 100644 --- a/crates/rattler_conda_types/src/match_spec/mod.rs +++ b/crates/rattler_conda_types/src/match_spec/mod.rs @@ -89,13 +89,13 @@ use matcher::StringMatcher; /// assert_eq!(spec.version, Some(VersionSpec::from_str("1.0.*", Strict).unwrap())); /// assert_eq!(spec.channel, Some(Channel::from_str("conda-forge", &channel_config).map(|channel| Arc::new(channel)).unwrap())); /// -/// let spec = MatchSpec::from_str("conda-forge/linux-64::foo >=1.0", Strict).unwrap(); +/// let spec = MatchSpec::from_str(r#"conda-forge::foo >=1.0[subdir="linux-64"]"#, Strict).unwrap(); /// assert_eq!(spec.name, Some(PackageName::new_unchecked("foo"))); /// assert_eq!(spec.version, Some(VersionSpec::from_str(">=1.0", Strict).unwrap())); /// assert_eq!(spec.channel, Some(Channel::from_str("conda-forge", &channel_config).map(|channel| Arc::new(channel)).unwrap())); /// assert_eq!(spec.subdir, Some("linux-64".to_string())); /// -/// let spec = MatchSpec::from_str("*/linux-64::foo >=1.0", Strict).unwrap(); +/// let spec = MatchSpec::from_str(r#"*::foo >=1.0[subdir="linux-64"]"#, Strict).unwrap(); /// assert_eq!(spec.name, Some(PackageName::new_unchecked("foo"))); /// assert_eq!(spec.version, Some(VersionSpec::from_str(">=1.0", Strict).unwrap())); /// assert_eq!(spec.channel, Some(Channel::from_str("*", &channel_config).map(|channel| Arc::new(channel)).unwrap())); @@ -470,7 +470,6 @@ impl Matches for NamelessMatchSpec { #[cfg(test)] mod tests { - use serde_json::json; use std::str::FromStr; use rattler_digest::{parse_digest_from_hex, Md5, Sha256}; diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 310f7611e..1f323f9e5 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -261,7 +261,8 @@ fn parse_bracket_vec_into_components( }; match_spec.url = Some(url); - } + }, + "subdir" => match_spec.subdir = Some(value.to_string()), // TODO: Still need to add `track_features`, `features`, `license` and `license_family` // to the match spec. _ => Err(ParseMatchSpecError::InvalidBracketKey(key.to_owned()))?, From 5862a9af95f0a3a4fb16b738ca8086385e1f42b6 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Thu, 25 Jul 2024 16:34:41 +0200 Subject: [PATCH 08/15] fmt --- crates/rattler_conda_types/src/match_spec/parse.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 1f323f9e5..acc34c69b 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -261,7 +261,7 @@ fn parse_bracket_vec_into_components( }; match_spec.url = Some(url); - }, + } "subdir" => match_spec.subdir = Some(value.to_string()), // TODO: Still need to add `track_features`, `features`, `license` and `license_family` // to the match spec. From 7f7b96ced5b0d7ecee28bf12de1f56f15653f54a Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Fri, 26 Jul 2024 11:13:10 +0200 Subject: [PATCH 09/15] fix: reintroduce subdir from channel str --- .../rattler_conda_types/src/match_spec/mod.rs | 2 +- .../src/match_spec/parse.rs | 79 ++++++++++++++++--- 2 files changed, 69 insertions(+), 12 deletions(-) diff --git a/crates/rattler_conda_types/src/match_spec/mod.rs b/crates/rattler_conda_types/src/match_spec/mod.rs index 630fae4fb..15a3e869c 100644 --- a/crates/rattler_conda_types/src/match_spec/mod.rs +++ b/crates/rattler_conda_types/src/match_spec/mod.rs @@ -95,7 +95,7 @@ use matcher::StringMatcher; /// assert_eq!(spec.channel, Some(Channel::from_str("conda-forge", &channel_config).map(|channel| Arc::new(channel)).unwrap())); /// assert_eq!(spec.subdir, Some("linux-64".to_string())); /// -/// let spec = MatchSpec::from_str(r#"*::foo >=1.0[subdir="linux-64"]"#, Strict).unwrap(); +/// let spec = MatchSpec::from_str("*/linux-64::foo >=1.0", Strict).unwrap(); /// assert_eq!(spec.name, Some(PackageName::new_unchecked("foo"))); /// assert_eq!(spec.version, Some(VersionSpec::from_str(">=1.0", Strict).unwrap())); /// assert_eq!(spec.channel, Some(Channel::from_str("*", &channel_config).map(|channel| Arc::new(channel)).unwrap())); diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index acc34c69b..13117c3dd 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, ops::Not, str::FromStr}; +use std::{borrow::Cow, ops::Not, str::FromStr, sync::Arc}; use nom::{ branch::alt, @@ -32,7 +32,7 @@ use crate::{ Channel, ChannelConfig, InvalidPackageNameError, NamelessMatchSpec, PackageName, ParseChannelError, ParseStrictness, ParseStrictness::{Lenient, Strict}, - ParseVersionError, VersionSpec, + ParseVersionError, Platform, VersionSpec, }; /// The type of parse error that occurred when parsing match spec. @@ -465,6 +465,26 @@ impl NamelessMatchSpec { } } +/// Parse channel and subdir from a string. +fn parse_channel_and_subdir( + input: &str, +) -> Result<(Option, Option), ParseMatchSpecError> { + let channel_config = ChannelConfig::default_with_root_dir( + std::env::current_dir().expect("Could not get current directory"), + ); + + if let Some((channel, subdir)) = input.rsplit_once('/') { + // If the subdir is a platform, we assume the channel has a subdir + if Platform::from_str(subdir).is_ok() { + return Ok(( + Some(Channel::from_str(channel, &channel_config)?), + Some(subdir.to_string()), + )); + } + } + Ok((Some(Channel::from_str(input, &channel_config)?), None)) +} + /// Parses a conda match spec. /// This is based on: fn matchspec_parser( @@ -526,12 +546,9 @@ fn matchspec_parser( .or(nameless_match_spec.namespace); if let Some(channel_str) = channel_str { - let channel_config = ChannelConfig::default_with_root_dir( - std::env::current_dir().expect("Could not get current directory"), - ); - // No subdir split, assume subdir is not part of the channel url. - let channel = Channel::from_str(channel_str, &channel_config)?; - nameless_match_spec.channel = Some(channel.into()); + let (channel, subdir) = parse_channel_and_subdir(channel_str)?; + nameless_match_spec.channel = nameless_match_spec.channel.or(channel.map(Arc::new)); + nameless_match_spec.subdir = nameless_match_spec.subdir.or(subdir); } // Step 6. Strip off the package name from the input @@ -607,18 +624,19 @@ fn optionally_strip_equals<'a>( #[cfg(test)] mod tests { + use std::{str::FromStr, sync::Arc}; + use assert_matches::assert_matches; use indexmap::IndexMap; use rattler_digest::{parse_digest_from_hex, Md5, Sha256}; use rstest::rstest; use serde::Serialize; use smallvec::smallvec; - use std::{str::FromStr, sync::Arc}; use url::Url; use super::{ - split_version_and_build, strip_brackets, strip_package_name, BracketVec, MatchSpec, - ParseMatchSpecError, + parse_channel_and_subdir, split_version_and_build, strip_brackets, strip_package_name, + BracketVec, MatchSpec, ParseMatchSpecError, }; use crate::{ match_spec::parse::parse_bracket_list, BuildNumberSpec, Channel, ChannelConfig, @@ -1154,4 +1172,43 @@ mod tests { MatchSpec::from_str("python ==2.7.*.*|>=3.6", Strict).expect_err("nameful"); NamelessMatchSpec::from_str("==2.7.*.*|>=3.6", Strict).expect_err("nameless"); } + + #[test] + fn test_parse_channel_subdir() { + let (channel, subdir) = parse_channel_and_subdir("conda-forge").unwrap(); + assert_eq!( + channel.unwrap(), + Channel::from_str("conda-forge", &channel_config()).unwrap() + ); + assert_eq!(subdir, None); + + let (channel, subdir) = parse_channel_and_subdir("conda-forge/linux-64").unwrap(); + assert_eq!( + channel.unwrap(), + Channel::from_str("conda-forge", &channel_config()).unwrap() + ); + assert_eq!(subdir, Some("linux-64".to_string())); + + let (channel, subdir) = parse_channel_and_subdir("conda-forge/label/test").unwrap(); + assert_eq!( + channel.unwrap(), + Channel::from_str("conda-forge/label/test", &channel_config()).unwrap() + ); + assert_eq!(subdir, None); + + let (channel, subdir) = + parse_channel_and_subdir("conda-forge/linux-64/label/test").unwrap(); + assert_eq!( + channel.unwrap(), + Channel::from_str("conda-forge/linux-64/label/test", &channel_config()).unwrap() + ); + assert_eq!(subdir, None); + + let (channel, subdir) = parse_channel_and_subdir("*/linux-64").unwrap(); + assert_eq!( + channel.unwrap(), + Channel::from_str("*", &channel_config()).unwrap() + ); + assert_eq!(subdir, Some("linux-64".to_string())); + } } From fc8ded33bf929a096250bf381d9d552f7147608e Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 31 Jul 2024 09:31:45 +0200 Subject: [PATCH 10/15] Update crates/rattler_conda_types/src/match_spec/parse.rs Co-authored-by: Bas Zalmstra --- crates/rattler_conda_types/src/match_spec/parse.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 13117c3dd..fa3e12e4c 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -969,6 +969,9 @@ mod tests { "python=3.9", "python=*", "https://software.repos.intel.com/python/conda::python[version=3.9]", + "https://software.repos.intel.com/python/conda/linux-64::python[version=3.9]", + "https://software.repos.intel.com/python/conda::python[version=3.9, subdir=linux-64]" + "https://software.repos.intel.com/python/conda/linux-64::python[version=3.9, subdir=linux-64]", ]; let evaluated: IndexMap<_, _> = specs From 3c38655224aa6c7d112876a4aa766c0357bac3ff Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 31 Jul 2024 09:32:01 +0200 Subject: [PATCH 11/15] Update crates/rattler_conda_types/src/match_spec/parse.rs Co-authored-by: Bas Zalmstra --- crates/rattler_conda_types/src/match_spec/parse.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index fa3e12e4c..01388e88a 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -529,15 +529,9 @@ fn matchspec_parser( // This assumes the [*] portions is stripped off, and then strip reverse to // ignore the first colon As that might be in the channel url. let mut input_split = input.rsplitn(3, ':').fuse(); - let (input, namespace, channel_str) = - match (input_split.next(), input_split.next(), input_split.next()) { - (Some(input), None, _) => (input, None, None), - (Some(input), Some(namespace), None) => (input, Some(namespace), None), - (Some(input), Some(namespace), Some(channel_str)) => { - (input, Some(namespace), Some(channel_str)) - } - (None, _, _) => ("", None, None), - }; + let input = input_split.next().unwrap_or(""); + let namespace = input_split.next(); + let channel_str = input_split.next(); nameless_match_spec.namespace = namespace .map(str::trim) From 1a2d5987afea2e3ee64fdba910cd1a16565670d2 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 31 Jul 2024 09:42:36 +0200 Subject: [PATCH 12/15] test: extend channel/subdir parse tests --- .../src/match_spec/parse.rs | 8 +++-- ...arse__tests__test_from_string_Lenient.snap | 31 +++++++++++++++++++ ...parse__tests__test_from_string_Strict.snap | 31 +++++++++++++++++++ 3 files changed, 67 insertions(+), 3 deletions(-) diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 01388e88a..dbd53a23c 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -963,9 +963,11 @@ mod tests { "python=3.9", "python=*", "https://software.repos.intel.com/python/conda::python[version=3.9]", - "https://software.repos.intel.com/python/conda/linux-64::python[version=3.9]", - "https://software.repos.intel.com/python/conda::python[version=3.9, subdir=linux-64]" - "https://software.repos.intel.com/python/conda/linux-64::python[version=3.9, subdir=linux-64]", + "https://c.com/p/conda/linux-64::python[version=3.9]", + "https://c.com/p/conda::python[version=3.9, subdir=linux-64]", + // subdir in brackets take precedence + "conda-forge/linux-32::python[version=3.9, subdir=linux-64]", + "conda-forge/linux-32::python ==3.9[subdir=linux-64, build_number=\"0\"]", ]; let evaluated: IndexMap<_, _> = specs diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap index dc8403e36..9d6bfc745 100644 --- a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Lenient.snap @@ -67,3 +67,34 @@ python=*: channel: base_url: "https://software.repos.intel.com/python/conda/" name: python/conda +"https://c.com/p/conda/linux-64::python[version=3.9]": + name: python + version: "==3.9" + channel: + base_url: "https://c.com/p/conda/" + name: p/conda + subdir: linux-64 +"https://c.com/p/conda::python[version=3.9, subdir=linux-64]": + name: python + version: "==3.9" + channel: + base_url: "https://c.com/p/conda/" + name: p/conda + subdir: linux-64 +"conda-forge/linux-32::python[version=3.9, subdir=linux-64]": + name: python + version: "==3.9" + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge + subdir: linux-64 +"conda-forge/linux-32::python ==3.9[subdir=linux-64, build_number=\"0\"]": + name: python + version: "==3.9" + build_number: + op: Eq + rhs: 0 + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge + subdir: linux-64 diff --git a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap index c4909deb9..8a77decf3 100644 --- a/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap +++ b/crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__test_from_string_Strict.snap @@ -59,3 +59,34 @@ python=*: channel: base_url: "https://software.repos.intel.com/python/conda/" name: python/conda +"https://c.com/p/conda/linux-64::python[version=3.9]": + name: python + version: "==3.9" + channel: + base_url: "https://c.com/p/conda/" + name: p/conda + subdir: linux-64 +"https://c.com/p/conda::python[version=3.9, subdir=linux-64]": + name: python + version: "==3.9" + channel: + base_url: "https://c.com/p/conda/" + name: p/conda + subdir: linux-64 +"conda-forge/linux-32::python[version=3.9, subdir=linux-64]": + name: python + version: "==3.9" + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge + subdir: linux-64 +"conda-forge/linux-32::python ==3.9[subdir=linux-64, build_number=\"0\"]": + name: python + version: "==3.9" + build_number: + op: Eq + rhs: 0 + channel: + base_url: "https://conda.anaconda.org/conda-forge/" + name: conda-forge + subdir: linux-64 From 36a03b3500cde36aa292160a2b8277ea8d5b9d82 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 31 Jul 2024 09:48:21 +0200 Subject: [PATCH 13/15] test: compare different subdir parse options as similar --- crates/rattler_conda_types/src/match_spec/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/rattler_conda_types/src/match_spec/mod.rs b/crates/rattler_conda_types/src/match_spec/mod.rs index 15a3e869c..18d4a00b7 100644 --- a/crates/rattler_conda_types/src/match_spec/mod.rs +++ b/crates/rattler_conda_types/src/match_spec/mod.rs @@ -94,6 +94,7 @@ use matcher::StringMatcher; /// assert_eq!(spec.version, Some(VersionSpec::from_str(">=1.0", Strict).unwrap())); /// assert_eq!(spec.channel, Some(Channel::from_str("conda-forge", &channel_config).map(|channel| Arc::new(channel)).unwrap())); /// assert_eq!(spec.subdir, Some("linux-64".to_string())); +/// assert_eq!(spec, MatchSpec::from_str("conda-forge/linux-64::foo >=1.0", Strict).unwrap()); /// /// let spec = MatchSpec::from_str("*/linux-64::foo >=1.0", Strict).unwrap(); /// assert_eq!(spec.name, Some(PackageName::new_unchecked("foo"))); From 7f653af74bbabcfb40da0fa1977dc323e264dabf Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 31 Jul 2024 09:58:52 +0200 Subject: [PATCH 14/15] fix: improve contains function with slice. --- crates/rattler_conda_types/src/channel/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rattler_conda_types/src/channel/mod.rs b/crates/rattler_conda_types/src/channel/mod.rs index be527c532..e836f5566 100644 --- a/crates/rattler_conda_types/src/channel/mod.rs +++ b/crates/rattler_conda_types/src/channel/mod.rs @@ -199,7 +199,7 @@ impl Channel { } } else { // Validate that the channel is a valid name - if channel.contains(|c| c == ':' || c == '\\') { + if channel.contains(&[':', '\\']) { return Err(ParseChannelError::InvalidName(channel.to_owned())); } Channel { From 08f0183376a221bbf57ad5ac875d7994f3c4ef81 Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Wed, 31 Jul 2024 10:02:38 +0200 Subject: [PATCH 15/15] clippy fix --- crates/rattler_conda_types/src/channel/mod.rs | 2 +- crates/rattler_conda_types/src/match_spec/parse.rs | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/rattler_conda_types/src/channel/mod.rs b/crates/rattler_conda_types/src/channel/mod.rs index e836f5566..23f758122 100644 --- a/crates/rattler_conda_types/src/channel/mod.rs +++ b/crates/rattler_conda_types/src/channel/mod.rs @@ -199,7 +199,7 @@ impl Channel { } } else { // Validate that the channel is a valid name - if channel.contains(&[':', '\\']) { + if channel.contains([':', '\\']) { return Err(ParseChannelError::InvalidName(channel.to_owned())); } Channel { diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index dbd53a23c..11e7030ce 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -822,7 +822,7 @@ mod tests { assert_eq!(spec1.url, Some(url.clone())); let spec_with_brackets = - NamelessMatchSpec::from_str(format!("[url={}]", url_str).as_str(), Strict).unwrap(); + NamelessMatchSpec::from_str(format!("[url={url_str}]").as_str(), Strict).unwrap(); assert_eq!(spec_with_brackets.url, Some(url)); } @@ -835,8 +835,7 @@ mod tests { assert_eq!(spec.url, Some(win_path.clone())); let spec_with_brackets = - NamelessMatchSpec::from_str(format!("[url={}]", win_path_str).as_str(), Strict) - .unwrap(); + NamelessMatchSpec::from_str(format!("[url={win_path_str}]").as_str(), Strict).unwrap(); assert_eq!(spec_with_brackets.url, Some(win_path)); // Unix @@ -846,8 +845,7 @@ mod tests { assert_eq!(spec.url, Some(unix_path.clone())); let spec_with_brackets = - NamelessMatchSpec::from_str(format!("[url={}]", unix_path_str).as_str(), Strict) - .unwrap(); + NamelessMatchSpec::from_str(format!("[url={unix_path_str}]").as_str(), Strict).unwrap(); assert_eq!(spec_with_brackets.url, Some(unix_path)); }