Allow setting timestamp when writing archives

crates/rattler_package_streaming/Cargo.toml

@@ -24,6 +24,7 @@ rattler_conda_types = { version = "0.2.0", path = "../rattler_conda_types" }
 itertools = "0.10.5"
 serde_json = "1.0.94"
 url = "2.3.1"
+chrono = "0.4.24"
 
 [features]
 tokio = ["dep:tokio", "bzip2/tokio", "tokio/fs", "tokio-util/io", "tokio-util/io-util", "reqwest?/stream", "futures-util"]

crates/rattler_package_streaming/src/write.rs

@@ -94,6 +94,7 @@ impl CompressionLevel {
 /// * `base_path` - the base path of the package. All paths in `paths` are relative to this path
 /// * `paths` - a list of paths to include in the package
 /// * `compression_level` - the compression level to use for the inner bzip2 encoded files
+/// * `timestamp` - optional a timestamp to use for all archive files (useful for reproducible builds)
 ///
 /// # Errors
 ///
@@ -109,7 +110,7 @@ impl CompressionLevel {
 ///
 /// let paths = vec![PathBuf::from("info/recipe/meta.yaml"), PathBuf::from("info/recipe/conda_build_config.yaml")];
 /// let mut file = File::create("test.tar.bz2").unwrap();
-/// write_tar_bz2_package(&mut file, &PathBuf::from("test"), &paths, CompressionLevel::Default).unwrap();
+/// write_tar_bz2_package(&mut file, &PathBuf::from("test"), &paths, CompressionLevel::Default, None).unwrap();
 /// ```
 ///
 /// # See also
@@ -120,6 +121,7 @@ pub fn write_tar_bz2_package<W: Write>(
     base_path: &Path,
     paths: &[PathBuf],
     compression_level: CompressionLevel,
+    timestamp: Option<&chrono::DateTime<chrono::Utc>>,
 ) -> Result<(), std::io::Error> {
     let mut archive = tar::Builder::new(bzip2::write::BzEncoder::new(
         writer,
@@ -130,7 +132,7 @@ pub fn write_tar_bz2_package<W: Write>(
     // sort paths alphabetically, and sort paths beginning with `info/` first
     let (info_paths, other_paths) = sort_paths(paths, base_path);
     for path in info_paths.chain(other_paths) {
-        append_path_to_archive(&mut archive, base_path, &path)?;
+        append_path_to_archive(&mut archive, base_path, &path, &timestamp)?;
     }
 
     archive.into_inner()?.finish()?;
@@ -144,14 +146,15 @@ fn write_zst_archive<W: Write>(
     base_path: &Path,
     paths: impl Iterator<Item = PathBuf>,
     compression_level: CompressionLevel,
+    timestamp: &Option<&chrono::DateTime<chrono::Utc>>,
 ) -> Result<(), std::io::Error> {
     // TODO figure out multi-threading for zstd
     let compression_level = compression_level.to_zstd_level()?;
     let mut archive = tar::Builder::new(zstd::Encoder::new(writer, compression_level)?);
     archive.follow_symlinks(false);
 
     for path in paths {
-        append_path_to_archive(&mut archive, base_path, &path)?;
+        append_path_to_archive(&mut archive, base_path, &path, timestamp)?;
     }
 
     archive.into_inner()?.finish()?;
@@ -171,6 +174,7 @@ fn write_zst_archive<W: Write>(
 /// * `base_path` - the base path of the package. All paths in `paths` are relative to this path
 /// * `paths` - a list of paths to include in the package
 /// * `compression_level` - the compression level to use for the inner zstd encoded files
+/// * `timestamp` - optional a timestamp to use for all archive files (useful for reproducible builds)
 ///
 /// # Errors
 ///
@@ -182,6 +186,7 @@ pub fn write_conda_package<W: Write + Seek>(
     paths: &[PathBuf],
     compression_level: CompressionLevel,
     out_name: &str,
+    timestamp: Option<&chrono::DateTime<chrono::Utc>>,
 ) -> Result<(), std::io::Error> {
     // first create the outer zip archive that uses no compression
     let mut outer_archive = zip::ZipWriter::new(writer);
@@ -202,18 +207,28 @@ pub fn write_conda_package<W: Write + Seek>(
         base_path,
         other_paths,
         compression_level,
+        &timestamp,
     )?;
 
     // info paths come last
     outer_archive.start_file(format!("info-{out_name}.tar.zst"), options)?;
-    write_zst_archive(&mut outer_archive, base_path, info_paths, compression_level)?;
+    write_zst_archive(
+        &mut outer_archive,
+        base_path,
+        info_paths,
+        compression_level,
+        &timestamp,
+    )?;
 
     outer_archive.finish()?;
 
     Ok(())
 }
 
-fn prepare_header(path: &Path) -> Result<tar::Header, std::io::Error> {
+fn prepare_header(
+    path: &Path,
+    timestamp: &Option<&chrono::DateTime<chrono::Utc>>,
+) -> Result<tar::Header, std::io::Error> {
     let mut header = tar::Header::new_gnu();
     let name = b"././@LongLink";
     header.as_gnu_mut().unwrap().name[..name.len()].clone_from_slice(&name[..]);
@@ -227,6 +242,10 @@ fn prepare_header(path: &Path) -> Result<tar::Header, std::io::Error> {
     header.set_device_minor(0)?;
     header.set_device_major(0)?;
 
+    if let Some(timestamp) = timestamp {
+        header.set_mtime(timestamp.timestamp() as u64);
+    }
+
     // let file_size = stat.len();
     // TODO do we need this
     //  + 1 to be compliant with GNU tar
@@ -243,9 +262,10 @@ fn append_path_to_archive(
     archive: &mut tar::Builder<impl Write>,
     base_path: &Path,
     path: &Path,
+    timestamp: &Option<&chrono::DateTime<chrono::Utc>>,
 ) -> Result<(), std::io::Error> {
     // create a tar header
-    let mut header = prepare_header(&base_path.join(path))
+    let mut header = prepare_header(&base_path.join(path), timestamp)
         .map_err(|err| trace_file_error(&base_path.join(path), err))?;
 
     if header.entry_type().is_file() {

crates/rattler_package_streaming/tests/write.rs

@@ -179,7 +179,8 @@ fn test_rewrite_tar_bz2() {
 
         let writer = File::create(&new_archive).unwrap();
         let paths = find_all_package_files(&target_dir);
-        write_tar_bz2_package(writer, &target_dir, &paths, CompressionLevel::Default).unwrap();
+        write_tar_bz2_package(writer, &target_dir, &paths, CompressionLevel::Default, None)
+            .unwrap();
 
         // compare the two archives
         let mut f1 = File::open(&file_path).unwrap();
@@ -217,6 +218,7 @@ fn test_rewrite_conda() {
             &paths,
             CompressionLevel::Default,
             &name,
+            None,
         )
         .unwrap();