diff --git a/Cargo.lock b/Cargo.lock index 9c8750534..de5981af1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2580,6 +2580,7 @@ dependencies = [ "rattler", "rattler_conda_types", "rattler_digest", + "rattler_index", "rattler_networking", "rattler_package_streaming", "rattler_repodata_gateway", @@ -2661,6 +2662,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "rattler_index" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b41c19bcd60dbc9375b90fd8ce4cce7073d56cab42879a5728a49965aaa6d50" +dependencies = [ + "fs-err", + "rattler_conda_types", + "rattler_digest", + "rattler_package_streaming", + "serde_json", + "tracing", + "walkdir", +] + [[package]] name = "rattler_macros" version = "0.16.1" diff --git a/Cargo.toml b/Cargo.toml index 68262138c..4834a00de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ serde_yaml = "0.9.30" rattler = { version = "0.16.1", default-features = false } rattler_conda_types = { version = "0.16.1", default-features = false } rattler_digest = { version = "0.16.1", default-features = false } +rattler_index = { version = "0.16.1", default-features = false } rattler_networking = { version = "0.16.1", default-features = false } rattler_repodata_gateway = { version = "0.16.1", default-features = false, features = [ "sparse", diff --git a/src/build.rs b/src/build.rs index f80a4f43e..5cd0b8f6a 100644 --- a/src/build.rs +++ b/src/build.rs @@ -14,6 +14,7 @@ use std::process::{Command, Stdio}; use itertools::Itertools; use miette::IntoDiagnostic; +use rattler_index::index; use rattler_shell::shell; use crate::env_vars::write_env_script; @@ -23,7 +24,7 @@ use crate::packaging::{package_conda, record_files}; use crate::recipe::parser::{ScriptContent, TestType}; use crate::render::resolved_dependencies::{install_environments, resolve_dependencies}; use crate::source::fetch_sources; -use crate::{index, package_test, tool_configuration}; +use crate::{package_test, tool_configuration}; const BASH_PREAMBLE: &str = r#" ## Start of bash preamble @@ -207,7 +208,7 @@ pub async fn run_build( ) -> miette::Result { let directories = &output.build_configuration.directories; - index::index( + index( &directories.output_dir, Some(&output.build_configuration.target_platform), ) @@ -337,7 +338,7 @@ pub async fn run_build( fs::remove_dir_all(&directories.build_dir).into_diagnostic()?; } - index::index( + index( &directories.output_dir, Some(&output.build_configuration.target_platform), ) diff --git a/src/index.rs b/src/index.rs deleted file mode 100644 index 42a1816ee..000000000 --- a/src/index.rs +++ /dev/null @@ -1,195 +0,0 @@ -//! Indexing of packages in a output folder to create up to date repodata.json files -use rattler_conda_types::package::ArchiveType; -use rattler_conda_types::package::IndexJson; -use rattler_conda_types::package::PackageFile; -use rattler_conda_types::ChannelInfo; -use rattler_conda_types::PackageRecord; -use rattler_conda_types::Platform; -use rattler_conda_types::RepoData; -use rattler_package_streaming::read; -use rattler_package_streaming::seek; - -use fs_err::File; -use std::ffi::OsStr; -use std::io::Read; -use std::io::Write; -use std::path::Path; -use std::path::PathBuf; -use walkdir::WalkDir; - -fn package_record_from_index_json( - file: &Path, - index_json_reader: &mut T, -) -> Result { - let index = IndexJson::from_reader(index_json_reader)?; - - let sha256_result = rattler_digest::compute_file_digest::(file)?; - let md5_result = rattler_digest::compute_file_digest::(file)?; - let size = std::fs::metadata(file)?.len(); - - let package_record = PackageRecord { - name: index.name, - version: index.version, - build: index.build, - build_number: index.build_number, - subdir: index.subdir.unwrap_or_else(|| "unknown".to_string()), - md5: Some(md5_result), - sha256: Some(sha256_result), - size: Some(size), - arch: index.arch, - platform: index.platform, - depends: index.depends, - constrains: index.constrains, - track_features: index.track_features, - features: index.features, - noarch: index.noarch, - license: index.license, - license_family: index.license_family, - timestamp: index.timestamp, - legacy_bz2_md5: None, - legacy_bz2_size: None, - purls: Default::default(), - }; - Ok(package_record) -} - -fn package_record_from_tar_bz2(file: &Path) -> Result { - let reader = std::fs::File::open(file)?; - let mut archive = read::stream_tar_bz2(reader); - for entry in archive.entries()?.flatten() { - let mut entry = entry; - let path = entry.path()?; - if path.as_os_str().eq("info/index.json") { - return package_record_from_index_json(file, &mut entry); - } - } - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "No index.json found", - )) -} - -fn package_record_from_conda(file: &Path) -> Result { - let reader = std::fs::File::open(file)?; - let mut archive = seek::stream_conda_info(reader).expect("Could not open conda file"); - - for entry in archive.entries()?.flatten() { - let mut entry = entry; - let path = entry.path()?; - if path.as_os_str().eq("info/index.json") { - return package_record_from_index_json(file, &mut entry); - } - } - Err(std::io::Error::new( - std::io::ErrorKind::Other, - "No index.json found", - )) -} - -/// Create a new `repodata.json` for all packages in the given output folder. If `target_platform` is -/// `Some`, only that specific subdir is indexed. Otherwise indexes all subdirs and creates a -/// `repodata.json` for each. -pub fn index( - output_folder: &Path, - target_platform: Option<&Platform>, -) -> Result<(), std::io::Error> { - let entries = WalkDir::new(output_folder).into_iter(); - let entries: Vec<(PathBuf, ArchiveType)> = entries - .filter_entry(|e| e.depth() <= 2) - .filter_map(|e| e.ok()) - .filter_map(|e| { - ArchiveType::split_str(e.path().to_string_lossy().as_ref()) - .map(|(p, t)| (PathBuf::from(format!("{}{}", p, t.extension())), t)) - }) - .collect(); - - // find all subdirs - let mut platforms = entries - .iter() - .filter_map(|(p, _)| { - p.parent() - .and_then(|parent| parent.file_name()) - .and_then(|file_name| { - let name = file_name.to_string_lossy().to_string(); - if name != "src_cache" { - Some(name) - } else { - None - } - }) - }) - .collect::>(); - - // Always create noarch subdir - if !output_folder.join("noarch").exists() { - std::fs::create_dir(output_folder.join("noarch"))?; - platforms.insert("noarch".to_string()); - } - - // Create target platform dir if needed - if let Some(target_platform) = target_platform { - let platform_str = target_platform.to_string(); - if !output_folder.join(&platform_str).exists() { - std::fs::create_dir(output_folder.join(&platform_str))?; - platforms.insert(platform_str); - } - } - - for platform in platforms { - if let Some(target_platform) = target_platform { - if platform != target_platform.to_string() { - if platform != "noarch" { - continue; - } else { - // check that noarch is already indexed if it is not the target platform - if output_folder.join("noarch/repodata.json").exists() { - continue; - } - } - } - } - - let mut repodata = RepoData { - info: Some(ChannelInfo { - subdir: platform.clone(), - base_url: None, - }), - packages: Default::default(), - conda_packages: Default::default(), - removed: Default::default(), - version: Some(1), - }; - - for (p, t) in entries.iter().filter_map(|(p, t)| { - p.parent().and_then(|parent| { - parent.file_name().and_then(|file_name| { - if file_name == OsStr::new(&platform) { - // If the file_name is the platform we're looking for, return Some((p, t)) - Some((p, t)) - } else { - // Otherwise, we return None to filter out this item - None - } - }) - }) - }) { - let record = match t { - ArchiveType::TarBz2 => package_record_from_tar_bz2(p), - ArchiveType::Conda => package_record_from_conda(p), - }; - let (Ok(record), Some(file_name)) = (record, p.file_name()) else { - tracing::info!("Could not read package record from {:?}", p); - continue; - }; - repodata - .conda_packages - .insert(file_name.to_string_lossy().to_string(), record); - } - let out_file = output_folder.join(platform).join("repodata.json"); - File::create(&out_file)?.write_all(serde_json::to_string_pretty(&repodata)?.as_bytes())?; - } - - Ok(()) -} - -// TODO: write proper unit tests for above functions diff --git a/src/lib.rs b/src/lib.rs index ac554e988..932ce5db0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,6 @@ pub mod variant_config; mod env_vars; pub mod hash; -mod index; mod linux; mod macos; mod packaging; diff --git a/src/package_test.rs b/src/package_test.rs index 036a4c76c..88d7b0b1f 100644 --- a/src/package_test.rs +++ b/src/package_test.rs @@ -20,13 +20,14 @@ use rattler_conda_types::{ package::{ArchiveIdentifier, PathsJson}, MatchSpec, Platform, }; +use rattler_index::index; use rattler_shell::{ activation::{ActivationError, ActivationVariables, Activator}, shell::{Shell, ShellEnum, ShellScript}, }; use crate::{ - env_vars, index, + env_vars, recipe::parser::{CommandsTestRequirements, PackageContents, PythonTest}, render::solver::create_environment, tool_configuration, @@ -270,7 +271,7 @@ pub async fn run_test(package_file: &Path, config: &TestConfiguration) -> Result )?; // index the temporary channel - index::index(tmp_repo.path(), Some(&target_platform))?; + index(tmp_repo.path(), Some(&target_platform))?; let cache_dir = rattler::default_cache_dir()?;