From 5ddb6f1ca6858f31f44f151b80703252441ad52c Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 13 Jun 2023 10:34:17 +0200 Subject: [PATCH] feat: more derives for conda lock (#213) --- .../src/conda_lock/builder.rs | 10 ++-- .../rattler_conda_types/src/conda_lock/mod.rs | 56 +++++++++-------- crates/rattler_conda_types/src/platform.rs | 13 ++++ crates/rattler_conda_types/src/utils/serde.rs | 60 +++++++++++++++++++ 4 files changed, 109 insertions(+), 30 deletions(-) diff --git a/crates/rattler_conda_types/src/conda_lock/builder.rs b/crates/rattler_conda_types/src/conda_lock/builder.rs index b3e585c50..4b6c69559 100644 --- a/crates/rattler_conda_types/src/conda_lock/builder.rs +++ b/crates/rattler_conda_types/src/conda_lock/builder.rs @@ -6,7 +6,7 @@ use crate::conda_lock::{ TimeMeta, VersionConstraint, }; use crate::{MatchSpec, Platform}; -use std::collections::{HashMap, HashSet}; +use fxhash::{FxHashMap, FxHashSet}; use url::Url; /// Struct used to build a conda-lock file @@ -15,7 +15,7 @@ pub struct LockFileBuilder { /// Channels used to resolve dependencies pub channels: Vec, /// The platforms this lock file supports - pub platforms: HashSet, + pub platforms: FxHashSet, /// Paths to source files, relative to the parent directory of the lockfile pub sources: Option>, /// Metadata dealing with the time lockfile was created @@ -24,7 +24,7 @@ pub struct LockFileBuilder { pub git_metadata: Option, /// Keep track of locked packages per platform - pub locked_packages: HashMap, + pub locked_packages: FxHashMap, /// MatchSpecs input /// This is only used to calculate the content_hash @@ -74,7 +74,7 @@ impl LockFileBuilder { content_hash::calculate_content_hash(plat, &self.input_specs, &self.channels)?, )) }) - .collect::, CalculateContentHashError>>()?; + .collect::, CalculateContentHashError>>()?; let lock = CondaLock { metadata: LockMeta { @@ -158,7 +158,7 @@ pub struct LockedPackage { /// Collection of package hash fields pub package_hashes: PackageHashes, /// List of dependencies for this package - pub dependency_list: HashMap, + pub dependency_list: FxHashMap, /// Check if package is optional pub optional: Option, } diff --git a/crates/rattler_conda_types/src/conda_lock/mod.rs b/crates/rattler_conda_types/src/conda_lock/mod.rs index 6ac02391b..808684260 100644 --- a/crates/rattler_conda_types/src/conda_lock/mod.rs +++ b/crates/rattler_conda_types/src/conda_lock/mod.rs @@ -2,18 +2,19 @@ //! It is modeled on the definitions found at: [conda-lock models](https://github.com/conda/conda-lock/blob/main/conda_lock/lockfile/models.py) //! Most names were kept the same as in the models file. So you can refer to those exactly. //! However, some types were added to enforce a bit more type safety. -use crate::conda_lock::PackageHashes::{Md5, Md5Sha256, Sha256}; -use crate::{NamelessMatchSpec, ParsePlatformError, Platform}; -use rattler_digest::serde::SerializableHash; -use rattler_digest::{Md5Hash, Sha256Hash}; -use serde::de::Error; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::collections::HashMap; -use std::fmt::{Display, Formatter}; -use std::fs::File; -use std::io::Read; -use std::path::Path; -use std::str::FromStr; +use self::PackageHashes::{Md5, Md5Sha256, Sha256}; +use crate::{utils::serde::Ordered, NamelessMatchSpec, ParsePlatformError, Platform}; +use fxhash::FxHashMap; +use rattler_digest::{serde::SerializableHash, Md5Hash, Sha256Hash}; +use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; +use serde_with::serde_as; +use std::{ + fmt::{Display, Formatter}, + fs::File, + io::Read, + path::Path, + str::FromStr, +}; use url::Url; pub mod builder; @@ -27,7 +28,7 @@ const fn default_version() -> u32 { /// Represents the conda-lock file /// Contains the metadata regarding the lock files /// also the locked packages -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct CondaLock { /// Metadata for the lock file pub metadata: LockMeta, @@ -93,14 +94,16 @@ impl CondaLock { } } -#[derive(Serialize, Deserialize)] +#[serde_as] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] /// Metadata for the [`CondaLock`] file pub struct LockMeta { /// Hash of dependencies for each target platform - pub content_hash: HashMap, + pub content_hash: FxHashMap, /// Channels used to resolve dependencies pub channels: Vec, /// The platforms this lock file supports + #[serde_as(as = "Ordered<_>")] pub platforms: Vec, /// Paths to source files, relative to the parent directory of the lockfile pub sources: Vec, @@ -109,13 +112,13 @@ pub struct LockMeta { /// Metadata dealing with the git repo the lockfile was created in and the user that created it pub git_metadata: Option, /// Metadata dealing with the input files used to create the lockfile - pub inputs_metadata: Option>, + pub inputs_metadata: Option>, /// Custom metadata provided by the user to be added to the lockfile - pub custom_metadata: Option>, + pub custom_metadata: Option>, } /// Stores information about when the lockfile was generated -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] pub struct TimeMeta { /// Time stamp of lock-file creation format // TODO: I think this is UTC time, change this later, conda-lock is not really using this now @@ -124,7 +127,7 @@ pub struct TimeMeta { /// Stores information about the git repo the lockfile is being generated in (if applicable) and /// the git user generating the file. -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] pub struct GitMeta { /// Git user.name field of global config pub git_user_name: String, @@ -135,7 +138,7 @@ pub struct GitMeta { } /// Represents whether this is a dependency managed by pip or conda -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq, Hash)] #[serde(rename_all = "lowercase")] pub enum Manager { /// The "conda" manager @@ -144,7 +147,7 @@ pub enum Manager { Pip, } -#[derive(Serialize, Deserialize, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Clone, Debug)] /// This is basically a MatchSpec but will never contain the package name /// TODO: Should this just wrap [`NamelessMatchSpec`]? pub struct VersionConstraint(String); @@ -168,6 +171,7 @@ impl From for VersionConstraint { /// If only the `md5` field is present, it constructs a `Md5` instance with its value. /// If only the `sha256` field is present, it constructs a `Sha256` instance with its value. /// If neither field is present it returns an error +#[derive(Eq, PartialEq, Hash, Clone, Debug)] pub enum PackageHashes { /// Contains an MD5 hash Md5(Md5Hash), @@ -234,7 +238,7 @@ fn default_category() -> String { } /// A locked single dependency / package -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Eq, PartialEq, Clone, Debug)] pub struct LockedDependency { /// Package name of dependency pub name: String, @@ -245,7 +249,7 @@ pub struct LockedDependency { /// What platform is this package for pub platform: Platform, /// What are its own dependencies mapping name to version constraint - pub dependencies: HashMap, + pub dependencies: FxHashMap, /// URL to find it at pub url: Url, /// Hashes of the package @@ -262,7 +266,7 @@ pub struct LockedDependency { } /// The URL for the dependency (currently only used for pip packages) -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Eq, PartialEq, Clone, Debug, Hash)] pub struct DependencySource { // According to: // https://github.com/conda/conda-lock/blob/854fca9923faae95dc2ddd1633d26fd6b8c2a82d/conda_lock/lockfile/models.py#L27 @@ -273,11 +277,13 @@ pub struct DependencySource { } /// The conda channel that was used for the dependency -#[derive(Serialize, Deserialize, Clone, Debug)] +#[serde_as] +#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)] pub struct Channel { /// Called `url` but can also be the name of the channel e.g. `conda-forge` pub url: String, /// Used env vars for the channel (e.g. hints for passwords or other secrets) + #[serde_as(as = "Ordered<_>")] pub used_env_vars: Vec, } diff --git a/crates/rattler_conda_types/src/platform.rs b/crates/rattler_conda_types/src/platform.rs index 9f34b3f59..671c83cc1 100644 --- a/crates/rattler_conda_types/src/platform.rs +++ b/crates/rattler_conda_types/src/platform.rs @@ -1,4 +1,5 @@ use serde::{Deserializer, Serializer}; +use std::cmp::Ordering; use std::{fmt, fmt::Formatter, str::FromStr}; use strum::{EnumIter, IntoEnumIterator}; use thiserror::Error; @@ -30,6 +31,18 @@ pub enum Platform { Emscripten32, } +impl PartialOrd for Platform { + fn partial_cmp(&self, other: &Self) -> Option { + self.as_str().partial_cmp(other.as_str()) + } +} + +impl Ord for Platform { + fn cmp(&self, other: &Self) -> Ordering { + self.as_str().cmp(other.as_str()) + } +} + /// Known architectures supported by Conda. #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] pub enum Arch { diff --git a/crates/rattler_conda_types/src/utils/serde.rs b/crates/rattler_conda_types/src/utils/serde.rs index a07a31672..03a02c82c 100644 --- a/crates/rattler_conda_types/src/utils/serde.rs +++ b/crates/rattler_conda_types/src/utils/serde.rs @@ -1,7 +1,11 @@ use chrono::{DateTime, Utc}; use serde::de::Error as _; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_with::de::DeserializeAsWrap; +use serde_with::ser::SerializeAsWrap; use serde_with::{DeserializeAs, SerializeAs}; +use std::collections::HashSet; +use std::hash::{BuildHasher, Hash}; use std::marker::PhantomData; use url::Url; @@ -112,3 +116,59 @@ impl SerializeAs> for Timestamp { timestamp.serialize(serializer) } } + +/// Used with serde_with to serialize a collection as a sorted collection. +#[derive(Default)] +pub(crate) struct Ordered(PhantomData); + +impl<'de, T: Eq + Hash, S: BuildHasher + Default, TAs> DeserializeAs<'de, HashSet> + for Ordered +where + TAs: DeserializeAs<'de, T>, +{ + fn deserialize_as(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let content = + DeserializeAsWrap::, Vec>::deserialize(deserializer)?.into_inner(); + Ok(HashSet::from_iter(content.into_iter())) + } +} + +impl> SerializeAs> for Ordered { + fn serialize_as(source: &HashSet, serializer: S) -> Result + where + S: Serializer, + { + let mut elements = Vec::from_iter(source.iter()); + elements.sort(); + SerializeAsWrap::, Vec<&TAs>>::new(&elements).serialize(serializer) + } +} + +impl<'de, T: Ord, TAs> DeserializeAs<'de, Vec> for Ordered +where + TAs: DeserializeAs<'de, T>, +{ + fn deserialize_as(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let mut content = + DeserializeAsWrap::, Vec>::deserialize(deserializer)?.into_inner(); + content.sort(); + Ok(content) + } +} + +impl> SerializeAs> for Ordered { + fn serialize_as(source: &Vec, serializer: S) -> Result + where + S: Serializer, + { + let mut elements = Vec::from_iter(source.iter()); + elements.sort(); + SerializeAsWrap::, Vec<&TAs>>::new(&elements).serialize(serializer) + } +}