From 66235f684f2cd546161165cfcaeefb5b600b103e Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 23 Nov 2023 15:30:12 +0100 Subject: [PATCH 1/2] feat: add purl to PackageRecord and lockfile --- crates/rattler_conda_types/Cargo.toml | 3 ++- crates/rattler_conda_types/src/lib.rs | 3 +++ crates/rattler_conda_types/src/repo_data/mod.rs | 9 ++++++++- crates/rattler_lock/src/builder.rs | 16 +++++++++++++++- crates/rattler_lock/src/conda.rs | 9 +++++++-- crates/rattler_solve/tests/backends.rs | 1 + 6 files changed, 36 insertions(+), 5 deletions(-) diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 400aa3719..32a987497 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -32,11 +32,12 @@ url = { version = "2.4.1", features = ["serde"] } rattler_digest = { version = "0.12.3", path = "../rattler_digest", features = ["serde"] } rattler_macros = { version = "0.12.3", path = "../rattler_macros" } glob = "0.3.1" +purl = { version = "0.1.2", features = ["serde"] } [dev-dependencies] rand = "0.8.5" insta = { version = "1.33.0", features = ["yaml", "redactions", "toml"] } -rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features=["rustls-tls"] } +rattler_package_streaming = { path = "../rattler_package_streaming", default-features = false, features = ["rustls-tls"] } tempfile = "3.8.0" rstest = "0.18.2" assert_matches = "1.5.0" diff --git a/crates/rattler_conda_types/src/lib.rs b/crates/rattler_conda_types/src/lib.rs index a5a290f2b..11d1b8506 100644 --- a/crates/rattler_conda_types/src/lib.rs +++ b/crates/rattler_conda_types/src/lib.rs @@ -50,6 +50,9 @@ pub use version_spec::VersionSpec; #[cfg(test)] use std::path::{Path, PathBuf}; +/// An package identifier that can be used to identify packages across package ecosystems. +pub type PackageUrl = purl::GenericPurl; + #[cfg(test)] pub(crate) fn get_test_data_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-data") diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index b8c7fdaf7..25714fda2 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -21,7 +21,7 @@ use rattler_macros::sorted; use crate::{ build_spec::BuildNumber, package::IndexJson, utils::serde::DeserializeFromStrUnchecked, - Channel, NoArchType, PackageName, Platform, RepoDataRecord, VersionWithSource, + Channel, NoArchType, PackageName, PackageUrl, Platform, RepoDataRecord, VersionWithSource, }; /// [`RepoData`] is an index of package binaries available on in a subdirectory of a Conda channel. @@ -125,6 +125,11 @@ pub struct PackageRecord { /// Optionally the platform the package supports pub platform: Option, // Note that this does not match the [`Platform`] enum.. + /// Package identifiers of packages that are equivalent to this package but from other + /// ecosystems. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub purls: Vec, + /// Optionally a SHA256 hash of the package archive #[serde_as(as = "Option>")] pub sha256: Option, @@ -279,6 +284,7 @@ impl PackageRecord { timestamp: None, track_features: vec![], version: version.into(), + purls: vec![], } } @@ -393,6 +399,7 @@ impl PackageRecord { timestamp: index.timestamp, track_features: index.track_features, version: index.version, + purls: vec![], }) } } diff --git a/crates/rattler_lock/src/builder.rs b/crates/rattler_lock/src/builder.rs index e21d25171..afbc7de22 100644 --- a/crates/rattler_lock/src/builder.rs +++ b/crates/rattler_lock/src/builder.rs @@ -7,7 +7,7 @@ use crate::{ PackageHashes, PackageName, PipLockedDependency, Platform, RepoDataRecord, TimeMeta, }; use fxhash::{FxHashMap, FxHashSet}; -use rattler_conda_types::NamelessMatchSpec; +use rattler_conda_types::{NamelessMatchSpec, PackageUrl}; use std::collections::HashSet; use url::Url; @@ -174,6 +174,7 @@ impl LockedPackagesBuilder { noarch: locked_package.noarch, size: locked_package.size, timestamp: locked_package.timestamp, + purls: locked_package.purls, } .into(), }, @@ -248,6 +249,9 @@ pub struct CondaLockedDependencyBuilder { /// Experimental: The date this entry was created. pub timestamp: Option>, + + /// Experimental: Defines that the package is an alias for a package from another ecosystem. + pub purls: Vec, } impl TryFrom<&RepoDataRecord> for CondaLockedDependencyBuilder { @@ -286,6 +290,7 @@ impl TryFrom for CondaLockedDependencyBuilder { noarch: record.package_record.noarch, size: record.package_record.size, timestamp: record.package_record.timestamp, + purls: record.package_record.purls, }) } } @@ -401,6 +406,12 @@ impl CondaLockedDependencyBuilder { self.timestamp = Some(timestamp); self } + + /// Adds a PackageUrl to the package + pub fn add_purl(mut self, purl: PackageUrl) -> Self { + self.purls.push(purl); + self + } } pub struct PipLockedDependencyBuilder { @@ -472,6 +483,9 @@ mod tests { noarch: NoArchType::python(), size: Some(12000), timestamp: Some(Utc::now()), + purls: vec![ + "pkg:deb/debian/python@3.11.0?arch=x86_64".parse().unwrap(), + ] })) .build().unwrap(); diff --git a/crates/rattler_lock/src/conda.rs b/crates/rattler_lock/src/conda.rs index 9cda2361d..494c9fd8a 100644 --- a/crates/rattler_lock/src/conda.rs +++ b/crates/rattler_lock/src/conda.rs @@ -3,8 +3,8 @@ use crate::{ PackageHashes::{Md5, Md5Sha256, Sha256}, }; use rattler_conda_types::{ - InvalidPackageNameError, NoArchType, PackageName, PackageRecord, ParseMatchSpecError, - ParseVersionError, RepoDataRecord, + InvalidPackageNameError, NoArchType, PackageName, PackageRecord, PackageUrl, + ParseMatchSpecError, ParseVersionError, RepoDataRecord, }; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, skip_serializing_none, OneOrMany}; @@ -67,6 +67,10 @@ pub struct CondaLockedDependency { /// Experimental: The date this entry was created. #[serde_as(as = "Option")] pub timestamp: Option>, + + /// Experimental: Defines that the package is an alias for a package from another ecosystem. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub purls: Vec, } impl TryFrom<&LockedDependency> for RepoDataRecord { @@ -136,6 +140,7 @@ impl TryFrom for RepoDataRecord { timestamp: value.timestamp, track_features: value.track_features, version, + purls: value.purls, }, file_name, url: value.url, diff --git a/crates/rattler_solve/tests/backends.rs b/crates/rattler_solve/tests/backends.rs index dd12b5440..32cefd9a7 100644 --- a/crates/rattler_solve/tests/backends.rs +++ b/crates/rattler_solve/tests/backends.rs @@ -104,6 +104,7 @@ fn installed_package( timestamp: None, legacy_bz2_size: None, legacy_bz2_md5: None, + purls: Vec::new(), }, } } From f41051e78a685b49fe720f62a1562f4fde593bbe Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Fri, 24 Nov 2023 09:31:35 +0100 Subject: [PATCH 2/2] fix: also add to patches --- .../src/repo_data/patches.rs | 22 ++++- ..._repo_data__patches__test__patch_purl.snap | 84 +++++++++++++++++++ .../patch/linux-64/patch_instructions_4.json | 9 ++ 3 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 crates/rattler_conda_types/src/repo_data/snapshots/rattler_conda_types__repo_data__patches__test__patch_purl.snap create mode 100644 test-data/channels/patch/linux-64/patch_instructions_4.json diff --git a/crates/rattler_conda_types/src/repo_data/patches.rs b/crates/rattler_conda_types/src/repo_data/patches.rs index bd61ede3c..687592b2d 100644 --- a/crates/rattler_conda_types/src/repo_data/patches.rs +++ b/crates/rattler_conda_types/src/repo_data/patches.rs @@ -4,7 +4,7 @@ use serde_with::{serde_as, skip_serializing_none, OneOrMany}; use std::io; use std::path::Path; -use crate::{package::ArchiveType, PackageRecord, RepoData}; +use crate::{package::ArchiveType, PackageRecord, PackageUrl, RepoData}; /// Represents a Conda repodata patch. /// @@ -92,6 +92,10 @@ pub struct PackageRecordPatch { with = "::serde_with::rust::double_option" )] pub license_family: Option>, + + /// Package identifiers of packages that are equivalent to this package but from other + /// ecosystems. + pub purls: Option>, } /// Repodata patch instructions for a single subdirectory. See [`RepoDataPatch`] for more @@ -136,6 +140,9 @@ impl PackageRecord { if let Some(license_family) = &patch.license_family { self.license_family = license_family.clone(); } + if let Some(package_urls) = &patch.purls { + self.purls = package_urls.clone(); + } } } @@ -260,4 +267,17 @@ mod test { // check result insta::assert_yaml_snapshot!(repodata); } + + #[test] + fn test_patch_purl() { + // test data + let mut repodata = load_test_repodata(); + let patch_instructions = load_patch_instructions("patch_instructions_4.json"); + + // apply patch + repodata.apply_patches(&patch_instructions); + + // check result + insta::assert_yaml_snapshot!(repodata); + } } diff --git a/crates/rattler_conda_types/src/repo_data/snapshots/rattler_conda_types__repo_data__patches__test__patch_purl.snap b/crates/rattler_conda_types/src/repo_data/snapshots/rattler_conda_types__repo_data__patches__test__patch_purl.snap new file mode 100644 index 000000000..1c24079d7 --- /dev/null +++ b/crates/rattler_conda_types/src/repo_data/snapshots/rattler_conda_types__repo_data__patches__test__patch_purl.snap @@ -0,0 +1,84 @@ +--- +source: crates/rattler_conda_types/src/repo_data/patches.rs +assertion_line: 281 +expression: repodata +--- +info: + subdir: linux-64 +packages: + cross-python_emscripten-32-3.10.1-h60d57d3_8.tar.bz2: + build: h60d57d3_8 + build_number: 8 + depends: + - coreutils + - crossenv >=1.2 + - emscripten_emscripten-32 + - pip + - python 3.10.* + - rsync + - sed + - setuptools + md5: 07b8bdf69566cd63e6e49759033830ad + name: cross-python_emscripten-32 + purls: + - "pkg:pypi/cross-python@3.10.1" + sha256: 50064d294599734090b9a33a2194a94f5d3743325d5bae61fab3fa5151acf0f9 + size: 7069 + subdir: linux-64 + timestamp: 1679045249592 + version: 3.10.1 + emscripten_emscripten-32-3.1.27-h60d57d3_5.tar.bz2: + build: h60d57d3_5 + build_number: 5 + depends: + - emsdk + md5: 16f2daa96cb33b21c78f3dbc6d83546f + name: emscripten_emscripten-32 + sha256: ab5117990980e5243446c7e83a501cedfc9863c56bbf896e45c77155c84cc3ed + size: 6820 + subdir: linux-64 + timestamp: 1678986155405 + version: 3.1.27 +packages.conda: + cross-python_emscripten-32-3.10.1-h60d57d3_8.conda: + build: h60d57d3_8 + build_number: 8 + depends: + - coreutils + - crossenv >=1.2 + - emscripten_emscripten-32 + - pip + - python 3.10.* + - rsync + - sed + - setuptools + md5: 07b8bdf69566cd63e6e49759033830ad + name: cross-python_emscripten-32 + purls: + - "pkg:pypi/cross-python@3.10.1" + sha256: 12064d294599734090b9a33a2194a94f5d3743325d5bae61fab3fa5151acf0f9 + size: 7069 + subdir: linux-64 + timestamp: 1679045249592 + version: 3.10.1 + cross-python_emscripten-32-3.12.1-h60d57d3_8.conda: + build: h60d57d3_8 + build_number: 8 + depends: + - coreutils + - crossenv >=1.2 + - emscripten_emscripten-32 + - pip + - python 3.12.* + - rsync + - sed + - setuptools + md5: 07b8bdf69566cd63e6e49759033830ad + name: cross-python_emscripten-32 + sha256: 12064d294599734090b9a33a2194a94f5d3743325d5bae61fab3fa5151acf0f9 + size: 7069 + subdir: linux-64 + timestamp: 1679045249592 + version: 3.10.1 +repodata_version: 1 + diff --git a/test-data/channels/patch/linux-64/patch_instructions_4.json b/test-data/channels/patch/linux-64/patch_instructions_4.json new file mode 100644 index 000000000..b73ef38d7 --- /dev/null +++ b/test-data/channels/patch/linux-64/patch_instructions_4.json @@ -0,0 +1,9 @@ +{ + "packages": { + "cross-python_emscripten-32-3.10.1-h60d57d3_8.tar.bz2": { + "purls": [ + "pkg:pypi/cross-python@3.10.1" + ] + } + } +}