From 0c0cd87c61889868d891e9b0f8a0176654fedd21 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Sat, 1 Apr 2023 14:08:24 +0200 Subject: [PATCH 1/6] feat: now determines subdir Uses platform and architecture and values that were found in the production database --- .../rattler_conda_types/src/repo_data/mod.rs | 87 +++++++++++++++++-- 1 file changed, 82 insertions(+), 5 deletions(-) diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 74d4d61e2..9bd3839a8 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -11,11 +11,12 @@ use fxhash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, skip_serializing_none, DisplayFromStr, OneOrMany}; +use thiserror::Error; use rattler_macros::sorted; use crate::package::IndexJson; -use crate::{Channel, NoArchType, RepoDataRecord, Version}; +use crate::{Channel, NoArchType, Platform, RepoDataRecord, Version}; /// [`RepoData`] is an index of package binaries available on in a subdirectory of a Conda channel. // Note: we cannot use the sorted macro here, because the `packages` and `conda_packages` fields are @@ -176,6 +177,66 @@ impl RepoData { } } +/// An error that can occur when parsing a platform from a string. +#[derive(Debug, Error, Clone, Eq, PartialEq)] +pub enum ConvertSubdirError { + #[error("platform: {platform}, arch: {arch} is not a known combination")] + NoKnownCombination { + /// The platform string that could not be parsed. + platform: String, + /// The architecture. + arch: String, + }, + #[error("platform key is empty in index.json")] + PlatformEmpty, + #[error("arch key is empty in index.json")] + ArchEmpty, +} + +/// Determine the subdir based on result taken from the prefix.dev +/// database +/// These were the combinations that have been found in the database. +/// and have been represented in the function. +/// +/// # Why can we not use Platform::FromStr? +/// +/// We cannot use the Platform FromStr directly because x86 and x86_64 +/// are different architecture strings. Also some combinations have been removed, +/// because they have not been found. +fn determine_subdir( + platform: Option, + arch: Option, +) -> Result { + + let platform = platform.ok_or(ConvertSubdirError::PlatformEmpty)?; + let arch = arch.ok_or(ConvertSubdirError::ArchEmpty)?; + let canonical = format!("{platform}-{arch}"); + // Convert to Platform first + let plat = match canonical.as_ref() { + "linux-x86" => Platform::Linux32, + "linux-x86_64" => Platform::Linux64, + "linux-aarch64" => Platform::LinuxAarch64, + "linux-armv6l" => Platform::LinuxArmV6l, + "linux-armv7l" => Platform::LinuxArmV7l, + "linux-ppc64le" => Platform::LinuxPpc64le, + "linux-ppc64" => Platform::LinuxPpc64, + "linux-s390x" => Platform::LinuxS390X, + "osx-x86_64" => Platform::Osx64, + "osx-arm64" => Platform::OsxArm64, + "win-32" => Platform::Win32, + "win-64" => Platform::Win64, + "win-arm64" => Platform::WinArm64, + _ => { + return Err(ConvertSubdirError::NoKnownCombination { + platform, + arch, + }) + } + }; + // Convert back to Platform string which should correspond to known subdirs + Ok(plat.to_string()) +} + impl PackageRecord { /// Builds a [`PackageRecord`] from a [`IndexJson`] and optionally a size, sha256 and md5 hash. pub fn from_index_json( @@ -183,8 +244,14 @@ impl PackageRecord { size: Option, sha256: Option, md5: Option, - ) -> Self { - PackageRecord { + ) -> Result { + // Determine the subdir if it can't be found + let subdir = match index.subdir { + None => determine_subdir(index.platform.clone(), index.arch.clone())?, + Some(s) => s, + }; + + Ok(PackageRecord { arch: index.arch, build: index.build, build_number: index.build_number, @@ -201,11 +268,11 @@ impl PackageRecord { platform: index.platform, sha256, size, - subdir: index.subdir.unwrap(), + subdir, timestamp: index.timestamp, track_features: index.track_features, version: index.version, - } + }) } } @@ -226,9 +293,19 @@ fn sort_set_alphabetically( #[cfg(test)] mod test { use fxhash::FxHashSet; + use crate::repo_data::determine_subdir; use crate::RepoData; + // isl-0.12.2-1.tar.bz2 + // gmp-5.1.2-6.tar.bz2 + // Are both package variants in the osx-64 subdir + // Will just test for this case + #[test] + fn test_determine_subdir() { + assert_eq!(determine_subdir(Some("osx".to_string()), Some("x86_64".to_string())).unwrap(), "osx-64"); + } + #[test] fn test_serialize() { let repodata = RepoData { From 20585fb974836f312d396b27daf1a26c7cbd5f66 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Sat, 1 Apr 2023 14:11:35 +0200 Subject: [PATCH 2/6] fix: fmt --- crates/rattler_conda_types/src/repo_data/mod.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 9bd3839a8..7051a4571 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -207,7 +207,6 @@ fn determine_subdir( platform: Option, arch: Option, ) -> Result { - let platform = platform.ok_or(ConvertSubdirError::PlatformEmpty)?; let arch = arch.ok_or(ConvertSubdirError::ArchEmpty)?; let canonical = format!("{platform}-{arch}"); @@ -226,12 +225,7 @@ fn determine_subdir( "win-32" => Platform::Win32, "win-64" => Platform::Win64, "win-arm64" => Platform::WinArm64, - _ => { - return Err(ConvertSubdirError::NoKnownCombination { - platform, - arch, - }) - } + _ => return Err(ConvertSubdirError::NoKnownCombination { platform, arch }), }; // Convert back to Platform string which should correspond to known subdirs Ok(plat.to_string()) @@ -292,8 +286,8 @@ fn sort_set_alphabetically( #[cfg(test)] mod test { - use fxhash::FxHashSet; use crate::repo_data::determine_subdir; + use fxhash::FxHashSet; use crate::RepoData; @@ -303,7 +297,10 @@ mod test { // Will just test for this case #[test] fn test_determine_subdir() { - assert_eq!(determine_subdir(Some("osx".to_string()), Some("x86_64".to_string())).unwrap(), "osx-64"); + assert_eq!( + determine_subdir(Some("osx".to_string()), Some("x86_64".to_string())).unwrap(), + "osx-64" + ); } #[test] From b5355e88e7b1090e537e000bc5318fb5a436ce4d Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Sat, 1 Apr 2023 14:11:46 +0200 Subject: [PATCH 3/6] fix: clippy --- crates/rattler_package_streaming/tests/write.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rattler_package_streaming/tests/write.rs b/crates/rattler_package_streaming/tests/write.rs index 37990b7c8..b36c5fbfe 100644 --- a/crates/rattler_package_streaming/tests/write.rs +++ b/crates/rattler_package_streaming/tests/write.rs @@ -17,7 +17,7 @@ fn find_all_archives() -> impl Iterator { } fn find_all_package_files(path: &Path) -> Vec { - WalkDir::new(&path) + WalkDir::new(path) .into_iter() .filter_map(|e| e.ok()) .map(|e| e.into_path()) @@ -35,11 +35,11 @@ impl Decoder { match self { Decoder::TarBz2 => { let d = bzip2::read::BzDecoder::new(f); - return tar::Archive::new(Box::new(d)); + tar::Archive::new(Box::new(d)) } Decoder::Zst => { let d = zstd::stream::read::Decoder::new(f).unwrap(); - return tar::Archive::new(Box::new(d)); + tar::Archive::new(Box::new(d)) } } } From 8f94e0544560196d889354e3736c4539eecc602a Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Mon, 3 Apr 2023 09:08:20 +0200 Subject: [PATCH 4/6] fix: changed match statement around --- .../rattler_conda_types/src/repo_data/mod.rs | 49 ++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 7051a4571..01811a934 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -209,23 +209,38 @@ fn determine_subdir( ) -> Result { let platform = platform.ok_or(ConvertSubdirError::PlatformEmpty)?; let arch = arch.ok_or(ConvertSubdirError::ArchEmpty)?; - let canonical = format!("{platform}-{arch}"); - // Convert to Platform first - let plat = match canonical.as_ref() { - "linux-x86" => Platform::Linux32, - "linux-x86_64" => Platform::Linux64, - "linux-aarch64" => Platform::LinuxAarch64, - "linux-armv6l" => Platform::LinuxArmV6l, - "linux-armv7l" => Platform::LinuxArmV7l, - "linux-ppc64le" => Platform::LinuxPpc64le, - "linux-ppc64" => Platform::LinuxPpc64, - "linux-s390x" => Platform::LinuxS390X, - "osx-x86_64" => Platform::Osx64, - "osx-arm64" => Platform::OsxArm64, - "win-32" => Platform::Win32, - "win-64" => Platform::Win64, - "win-arm64" => Platform::WinArm64, - _ => return Err(ConvertSubdirError::NoKnownCombination { platform, arch }), + + let plat = match platform.as_ref() { + "linux" => { + match arch.as_ref() { + "x86" => Platform::Linux32, + "x86_64" => Platform::Linux64, + "aarch64" => Platform::LinuxAarch64, + "armv61" => Platform::LinuxArmV6l, + "armv71" => Platform::LinuxArmV7l, + "ppc64le" => Platform::LinuxPpc64le, + "ppc64" => Platform::LinuxPpc64, + "s390x" => Platform::LinuxS390X, + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), + } + + }, + "osx" => { + match arch.as_ref() { + "x86_64" => Platform::Osx64, + "arm64" => Platform::OsxArm64, + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), + } + }, + "windows" => { + match arch.as_ref() { + "x86" => Platform::Win32, + "x86_64" => Platform::Win64, + "arm64" => Platform::WinArm64, + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), + } + } + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), }; // Convert back to Platform string which should correspond to known subdirs Ok(plat.to_string()) From 4b34c9f8e1ddd03f8dc0b5cded8656ae00ac6e0e Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Mon, 3 Apr 2023 09:15:51 +0200 Subject: [PATCH 5/6] fix: compile error --- .../rattler_conda_types/src/repo_data/mod.rs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 01811a934..289482541 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -213,35 +213,35 @@ fn determine_subdir( let plat = match platform.as_ref() { "linux" => { match arch.as_ref() { - "x86" => Platform::Linux32, - "x86_64" => Platform::Linux64, - "aarch64" => Platform::LinuxAarch64, - "armv61" => Platform::LinuxArmV6l, - "armv71" => Platform::LinuxArmV7l, - "ppc64le" => Platform::LinuxPpc64le, - "ppc64" => Platform::LinuxPpc64, - "s390x" => Platform::LinuxS390X, + "x86" => Ok(Platform::Linux32), + "x86_64" => Ok(Platform::Linux64), + "aarch64" => Ok(Platform::LinuxAarch64), + "armv61" => Ok(Platform::LinuxArmV6l), + "armv71" => Ok(Platform::LinuxArmV7l), + "ppc64le" => Ok(Platform::LinuxPpc64le), + "ppc64" => Ok(Platform::LinuxPpc64), + "s390x" => Ok(Platform::LinuxS390X), _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), } }, "osx" => { match arch.as_ref() { - "x86_64" => Platform::Osx64, - "arm64" => Platform::OsxArm64, + "x86_64" => Ok(Platform::Osx64), + "arm64" => Ok(Platform::OsxArm64), _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), } }, "windows" => { match arch.as_ref() { - "x86" => Platform::Win32, - "x86_64" => Platform::Win64, - "arm64" => Platform::WinArm64, + "x86" => Ok(Platform::Win32), + "x86_64" => Ok(Platform::Win64), + "arm64" => Ok(Platform::WinArm64), _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), } } _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), - }; + }?; // Convert back to Platform string which should correspond to known subdirs Ok(plat.to_string()) } From a5186c2ebd429e24b3790ea243a068e89003aad7 Mon Sep 17 00:00:00 2001 From: Tim de Jager Date: Mon, 3 Apr 2023 09:23:20 +0200 Subject: [PATCH 6/6] fix: cargo fmt --- .../rattler_conda_types/src/repo_data/mod.rs | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 289482541..bd1df8302 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -211,35 +211,28 @@ fn determine_subdir( let arch = arch.ok_or(ConvertSubdirError::ArchEmpty)?; let plat = match platform.as_ref() { - "linux" => { - match arch.as_ref() { - "x86" => Ok(Platform::Linux32), - "x86_64" => Ok(Platform::Linux64), - "aarch64" => Ok(Platform::LinuxAarch64), - "armv61" => Ok(Platform::LinuxArmV6l), - "armv71" => Ok(Platform::LinuxArmV7l), - "ppc64le" => Ok(Platform::LinuxPpc64le), - "ppc64" => Ok(Platform::LinuxPpc64), - "s390x" => Ok(Platform::LinuxS390X), - _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), - } - + "linux" => match arch.as_ref() { + "x86" => Ok(Platform::Linux32), + "x86_64" => Ok(Platform::Linux64), + "aarch64" => Ok(Platform::LinuxAarch64), + "armv61" => Ok(Platform::LinuxArmV6l), + "armv71" => Ok(Platform::LinuxArmV7l), + "ppc64le" => Ok(Platform::LinuxPpc64le), + "ppc64" => Ok(Platform::LinuxPpc64), + "s390x" => Ok(Platform::LinuxS390X), + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), }, - "osx" => { - match arch.as_ref() { - "x86_64" => Ok(Platform::Osx64), - "arm64" => Ok(Platform::OsxArm64), - _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), - } + "osx" => match arch.as_ref() { + "x86_64" => Ok(Platform::Osx64), + "arm64" => Ok(Platform::OsxArm64), + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), + }, + "windows" => match arch.as_ref() { + "x86" => Ok(Platform::Win32), + "x86_64" => Ok(Platform::Win64), + "arm64" => Ok(Platform::WinArm64), + _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), }, - "windows" => { - match arch.as_ref() { - "x86" => Ok(Platform::Win32), - "x86_64" => Ok(Platform::Win64), - "arm64" => Ok(Platform::WinArm64), - _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), - } - } _ => Err(ConvertSubdirError::NoKnownCombination { platform, arch }), }?; // Convert back to Platform string which should correspond to known subdirs