diff --git a/crates/rattler_package_streaming/Cargo.toml b/crates/rattler_package_streaming/Cargo.toml index 1daf13d19..948b79fa4 100644 --- a/crates/rattler_package_streaming/Cargo.toml +++ b/crates/rattler_package_streaming/Cargo.toml @@ -24,6 +24,7 @@ rattler_conda_types = { version = "0.2.0", path = "../rattler_conda_types" } itertools = "0.10.5" serde_json = "1.0.94" url = "2.3.1" +chrono = "0.4.24" [features] tokio = ["dep:tokio", "bzip2/tokio", "tokio/fs", "tokio-util/io", "tokio-util/io-util", "reqwest?/stream", "futures-util"] diff --git a/crates/rattler_package_streaming/src/write.rs b/crates/rattler_package_streaming/src/write.rs index 2d04f6a00..a1420a95c 100644 --- a/crates/rattler_package_streaming/src/write.rs +++ b/crates/rattler_package_streaming/src/write.rs @@ -94,6 +94,7 @@ impl CompressionLevel { /// * `base_path` - the base path of the package. All paths in `paths` are relative to this path /// * `paths` - a list of paths to include in the package /// * `compression_level` - the compression level to use for the inner bzip2 encoded files +/// * `timestamp` - optional a timestamp to use for all archive files (useful for reproducible builds) /// /// # Errors /// @@ -109,7 +110,7 @@ impl CompressionLevel { /// /// let paths = vec![PathBuf::from("info/recipe/meta.yaml"), PathBuf::from("info/recipe/conda_build_config.yaml")]; /// let mut file = File::create("test.tar.bz2").unwrap(); -/// write_tar_bz2_package(&mut file, &PathBuf::from("test"), &paths, CompressionLevel::Default).unwrap(); +/// write_tar_bz2_package(&mut file, &PathBuf::from("test"), &paths, CompressionLevel::Default, None).unwrap(); /// ``` /// /// # See also @@ -120,6 +121,7 @@ pub fn write_tar_bz2_package( base_path: &Path, paths: &[PathBuf], compression_level: CompressionLevel, + timestamp: Option<&chrono::DateTime>, ) -> Result<(), std::io::Error> { let mut archive = tar::Builder::new(bzip2::write::BzEncoder::new( writer, @@ -130,7 +132,7 @@ pub fn write_tar_bz2_package( // sort paths alphabetically, and sort paths beginning with `info/` first let (info_paths, other_paths) = sort_paths(paths, base_path); for path in info_paths.chain(other_paths) { - append_path_to_archive(&mut archive, base_path, &path)?; + append_path_to_archive(&mut archive, base_path, &path, ×tamp)?; } archive.into_inner()?.finish()?; @@ -144,6 +146,7 @@ fn write_zst_archive( base_path: &Path, paths: impl Iterator, compression_level: CompressionLevel, + timestamp: &Option<&chrono::DateTime>, ) -> Result<(), std::io::Error> { // TODO figure out multi-threading for zstd let compression_level = compression_level.to_zstd_level()?; @@ -151,7 +154,7 @@ fn write_zst_archive( archive.follow_symlinks(false); for path in paths { - append_path_to_archive(&mut archive, base_path, &path)?; + append_path_to_archive(&mut archive, base_path, &path, timestamp)?; } archive.into_inner()?.finish()?; @@ -171,6 +174,7 @@ fn write_zst_archive( /// * `base_path` - the base path of the package. All paths in `paths` are relative to this path /// * `paths` - a list of paths to include in the package /// * `compression_level` - the compression level to use for the inner zstd encoded files +/// * `timestamp` - optional a timestamp to use for all archive files (useful for reproducible builds) /// /// # Errors /// @@ -182,6 +186,7 @@ pub fn write_conda_package( paths: &[PathBuf], compression_level: CompressionLevel, out_name: &str, + timestamp: Option<&chrono::DateTime>, ) -> Result<(), std::io::Error> { // first create the outer zip archive that uses no compression let mut outer_archive = zip::ZipWriter::new(writer); @@ -202,18 +207,28 @@ pub fn write_conda_package( base_path, other_paths, compression_level, + ×tamp, )?; // info paths come last outer_archive.start_file(format!("info-{out_name}.tar.zst"), options)?; - write_zst_archive(&mut outer_archive, base_path, info_paths, compression_level)?; + write_zst_archive( + &mut outer_archive, + base_path, + info_paths, + compression_level, + ×tamp, + )?; outer_archive.finish()?; Ok(()) } -fn prepare_header(path: &Path) -> Result { +fn prepare_header( + path: &Path, + timestamp: &Option<&chrono::DateTime>, +) -> Result { let mut header = tar::Header::new_gnu(); let name = b"././@LongLink"; header.as_gnu_mut().unwrap().name[..name.len()].clone_from_slice(&name[..]); @@ -227,6 +242,10 @@ fn prepare_header(path: &Path) -> Result { header.set_device_minor(0)?; header.set_device_major(0)?; + if let Some(timestamp) = timestamp { + header.set_mtime(timestamp.timestamp() as u64); + } + // let file_size = stat.len(); // TODO do we need this // + 1 to be compliant with GNU tar @@ -243,9 +262,10 @@ fn append_path_to_archive( archive: &mut tar::Builder, base_path: &Path, path: &Path, + timestamp: &Option<&chrono::DateTime>, ) -> Result<(), std::io::Error> { // create a tar header - let mut header = prepare_header(&base_path.join(path)) + let mut header = prepare_header(&base_path.join(path), timestamp) .map_err(|err| trace_file_error(&base_path.join(path), err))?; if header.entry_type().is_file() { diff --git a/crates/rattler_package_streaming/tests/write.rs b/crates/rattler_package_streaming/tests/write.rs index b36c5fbfe..7cb88938c 100644 --- a/crates/rattler_package_streaming/tests/write.rs +++ b/crates/rattler_package_streaming/tests/write.rs @@ -179,7 +179,8 @@ fn test_rewrite_tar_bz2() { let writer = File::create(&new_archive).unwrap(); let paths = find_all_package_files(&target_dir); - write_tar_bz2_package(writer, &target_dir, &paths, CompressionLevel::Default).unwrap(); + write_tar_bz2_package(writer, &target_dir, &paths, CompressionLevel::Default, None) + .unwrap(); // compare the two archives let mut f1 = File::open(&file_path).unwrap(); @@ -217,6 +218,7 @@ fn test_rewrite_conda() { &paths, CompressionLevel::Default, &name, + None, ) .unwrap();