Support reading zstd-compressed initrds

.github/workflows/rust.yml

@@ -47,7 +47,7 @@ jobs:
           path: target/debug
           key: deps-${{ runner.os }}-${{ matrix.arch }}-${{ env.INSTALLED_TOOLCHAIN }}
       - name: Install dependencies
-        run: dnf install -y gcc git-core openssl-devel cpio diffutils jq xz
+        run: dnf install -y gcc git-core libzstd-devel openssl-devel cpio diffutils jq xz
       - name: Configure cargo
         run: |
           # Avoid OOM on emulated s390x
@@ -103,7 +103,7 @@ jobs:
           path: target/debug
           key: deps-${{ runner.os }}-${{ matrix.arch }}-${{ env.INSTALLED_TOOLCHAIN }}
       - name: Install dependencies
-        run: dnf install -y gcc git-core openssl-devel cpio diffutils jq xz
+        run: dnf install -y gcc git-core libzstd-devel openssl-devel cpio diffutils jq xz
       - name: Configure cargo
         run: |
           # Avoid OOM on emulated s390x
@@ -153,7 +153,7 @@ jobs:
           path: target/debug
           key: deps-${{ runner.os }}-${{ matrix.arch }}-${{ env.INSTALLED_TOOLCHAIN }}-lints
       - name: Install dependencies
-        run: dnf install -y gcc git-core openssl-devel
+        run: dnf install -y gcc git-core libzstd-devel openssl-devel
       - name: Configure cargo
         run: |
           # Avoid OOM on emulated s390x
@@ -198,7 +198,7 @@ jobs:
           path: target/debug
           key: deps-${{ runner.os }}-${{ matrix.arch }}-${{ env.INSTALLED_TOOLCHAIN }}
       - name: Install dependencies
-        run: dnf install -y gcc git-core openssl-devel cpio diffutils jq xz
+        run: dnf install -y gcc git-core libzstd-devel openssl-devel cpio diffutils jq xz
       - name: Configure cargo
         run: |
           # Avoid OOM on emulated s390x
@@ -234,7 +234,7 @@ jobs:
           ARCH: x86_64
           TOOLCHAIN: stable
       - name: Install dependencies
-        run: dnf install -y gcc git-core openssl-devel util-linux
+        run: dnf install -y gcc git-core libzstd-devel openssl-devel util-linux
       - name: cargo build
         run: cargo build
       - name: Help text line length

Cargo.toml

@@ -81,6 +81,7 @@ url = ">= 2.1, < 3.0"
 uuid = { version = ">= 0.8, < 2.0", features = ["v4"] }
 walkdir = "^2.3"
 xz2 = "^0.1"
+zstd = { version = ">= 0.10.0, < 0.12.0", features = ["pkg-config"] }
 
 [target.'cfg(target_arch = "s390x")'.dependencies]
 mbrman = { version = ">= 0.3, < 0.5", default-features = false }

Dockerfile

@@ -1,5 +1,5 @@
 FROM registry.fedoraproject.org/fedora:36 AS builder
-RUN dnf install -y cargo git-core openssl-devel xz-devel
+RUN dnf install -y cargo git-core libzstd-devel openssl-devel xz-devel
 WORKDIR /build
 COPY Cargo.* ./
 COPY src src/

docs/release-notes.md

@@ -8,6 +8,7 @@ nav_order: 8
 
 Major changes:
 
+- Support reading initrd images compressed with zstd
 
 Minor changes:
 
@@ -20,6 +21,7 @@ Internal changes:
 
 Packaging changes:
 
+- Add dependency on `zstd` crate and `libzstd` shared library
 - Remove non-Linux dependencies from vendor archive
 
 

fixtures/initrd/compressed-img.gen.sh

@@ -20,7 +20,10 @@ gzip -9 gzip.cpio
 make xz
 xz -9 xz.cpio
 
-cat uncompressed-1.cpio gzip.cpio.gz xz.cpio.xz uncompressed-2.cpio > compressed.img
+make zstd
+zstd -19 zstd.cpio
+
+cat uncompressed-1.cpio gzip.cpio.gz xz.cpio.xz zstd.cpio.zst uncompressed-2.cpio > compressed.img
 xz -9 compressed.img
 popd
 mv $dir/compressed.img.xz .

src/download.rs

@@ -531,6 +531,11 @@ mod tests {
             &include_bytes!("../fixtures/verify/1M.xz.sig")[..],
             &[0; 1 << 20][..],
         );
+        test_one_signed_file(
+            &include_bytes!("../fixtures/verify/1M.zst")[..],
+            &include_bytes!("../fixtures/verify/1M.zst.sig")[..],
+            &[0; 1 << 20][..],
+        );
     }
 
     fn test_one_signed_file(data: &[u8], sig: &[u8], decompressed_data: &[u8]) {

src/io/compress.rs

@@ -16,21 +16,22 @@ use anyhow::{Context, Result};
 use flate2::bufread::GzDecoder;
 use std::io::{self, ErrorKind, Read};
 
-use crate::io::{PeekReader, XzStreamDecoder};
+use crate::io::{is_zstd_magic, PeekReader, XzStreamDecoder, ZstdStreamDecoder};
 
-enum CompressDecoder<R: Read> {
+enum CompressDecoder<'a, R: Read> {
     Uncompressed(PeekReader<R>),
     Gzip(GzDecoder<PeekReader<R>>),
     Xz(XzStreamDecoder<PeekReader<R>>),
+    Zstd(ZstdStreamDecoder<'a, R>),
 }
 
-pub struct DecompressReader<R: Read> {
-    decoder: CompressDecoder<R>,
+pub struct DecompressReader<'a, R: Read> {
+    decoder: CompressDecoder<'a, R>,
     allow_trailing: bool,
 }
 
 /// Format-sniffing decompressor
-impl<R: Read> DecompressReader<R> {
+impl<R: Read> DecompressReader<'_, R> {
     pub fn new(source: PeekReader<R>) -> Result<Self> {
         Self::new_full(source, false)
     }
@@ -46,6 +47,8 @@ impl<R: Read> DecompressReader<R> {
             Gzip(GzDecoder::new(source))
         } else if sniff.len() >= 6 && &sniff[0..6] == b"\xfd7zXZ\x00" {
             Xz(XzStreamDecoder::new(source))
+        } else if sniff.len() > 4 && is_zstd_magic(sniff[0..4].try_into().unwrap()) {
+            Zstd(ZstdStreamDecoder::new(source)?)
         } else {
             Uncompressed(source)
         };
@@ -61,6 +64,7 @@ impl<R: Read> DecompressReader<R> {
             Uncompressed(d) => d,
             Gzip(d) => d.into_inner(),
             Xz(d) => d.into_inner(),
+            Zstd(d) => d.into_inner(),
         }
     }
 
@@ -70,6 +74,7 @@ impl<R: Read> DecompressReader<R> {
             Uncompressed(d) => d,
             Gzip(d) => d.get_mut(),
             Xz(d) => d.get_mut(),
+            Zstd(d) => d.get_mut(),
         }
     }
 
@@ -79,17 +84,19 @@ impl<R: Read> DecompressReader<R> {
             Uncompressed(_) => false,
             Gzip(_) => true,
             Xz(_) => true,
+            Zstd(_) => true,
         }
     }
 }
 
-impl<R: Read> Read for DecompressReader<R> {
+impl<R: Read> Read for DecompressReader<'_, R> {
     fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
         use CompressDecoder::*;
         let count = match &mut self.decoder {
             Uncompressed(d) => d.read(buf)?,
             Gzip(d) => d.read(buf)?,
             Xz(d) => d.read(buf)?,
+            Zstd(d) => d.read(buf)?,
         };
         if count == 0 && !buf.is_empty() && self.compressed() && !self.allow_trailing {
             // Decompressors stop reading as soon as they encounter the
@@ -121,6 +128,9 @@ mod tests {
         test_decompress_reader_trailing_data_one(
             &include_bytes!("../../fixtures/verify/1M.xz")[..],
         );
+        test_decompress_reader_trailing_data_one(
+            &include_bytes!("../../fixtures/verify/1M.zst")[..],
+        );
     }
 
     fn test_decompress_reader_trailing_data_one(input: &[u8]) {

src/io/initrd.rs

@@ -234,6 +234,8 @@ mod tests {
                 "gzip/world".into() => b"WORLD\n".to_vec(),
                 "xz/hello".into() => b"HELLO\n".to_vec(),
                 "xz/world".into() => b"WORLD\n".to_vec(),
+                "zstd/hello".into() => b"HELLO\n".to_vec(),
+                "zstd/world".into() => b"WORLD\n".to_vec(),
             }
         );
     }
@@ -268,8 +270,8 @@ mod tests {
         let initrd = Initrd::from_reader(&*archive).unwrap();
         assert_eq!(initrd.find(&matcher("gzip/hello")).len(), 1);
         assert_eq!(initrd.find(&matcher("gzip/*")).len(), 2);
-        assert_eq!(initrd.find(&matcher("*/hello")).len(), 4);
-        assert_eq!(initrd.find(&matcher("*")).len(), 8);
+        assert_eq!(initrd.find(&matcher("*/hello")).len(), 5);
+        assert_eq!(initrd.find(&matcher("*")).len(), 10);
         assert_eq!(initrd.find(&matcher("z")).len(), 0);
 
         // filtered initrd

src/io/mod.rs

@@ -26,6 +26,7 @@ mod peek;
 mod tee;
 mod verify;
 mod xz;
+mod zstd;
 
 pub use self::bls::*;
 pub use self::compress::*;
@@ -37,6 +38,7 @@ pub use self::peek::*;
 pub use self::tee::*;
 pub use self::verify::*;
 pub use self::xz::*;
+pub use self::zstd::*;
 
 // The default BufReader/BufWriter buffer size is 8 KiB, which isn't large
 // enough to fully amortize system call overhead.

src/io/peek.rs

@@ -105,7 +105,7 @@ mod tests {
     fn make_peek() -> PeekReader<Cursor<&'static [u8]>> {
         // use BufReader capacity larger than input; we're not testing
         // BufReader's buffering behavior
-        PeekReader::with_capacity(64, Cursor::new("abcdefghijklmnopqrstuvwxyz".as_bytes()))
+        PeekReader::with_capacity(64, Cursor::new(b"abcdefghijklmnopqrstuvwxyz"))
     }
 
     fn read_bytes<R: Read>(peek: &mut PeekReader<R>, amt: usize) -> Vec<u8> {