From fd6be9fb4e4ba4bf94c9be98fa3732c4d320724d Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 5 Nov 2021 16:26:24 -0400 Subject: [PATCH 1/4] live: factor out helper to check stdout isn't a TTY Prep for calling it in a future patch. --- src/live.rs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/live.rs b/src/live.rs index 3ed837e29..1e9769423 100644 --- a/src/live.rs +++ b/src/live.rs @@ -124,10 +124,8 @@ pub fn iso_ignition_remove(config: &IsoIgnitionRemoveConfig) -> Result<()> { } pub fn pxe_ignition_wrap(config: &PxeIgnitionWrapConfig) -> Result<()> { - if config.output.is_none() - && isatty(io::stdout().as_raw_fd()).context("checking if stdout is a TTY")? - { - bail!("Refusing to write binary data to terminal"); + if config.output.is_none() { + verify_stdout_not_tty()?; } let ignition = match config.ignition_file { @@ -232,9 +230,7 @@ fn write_live_iso(iso: &IsoConfig, input: &mut File, output_path: Option<&String iso.write(input)?; } Some("-") => { - if isatty(io::stdout().as_raw_fd()).context("checking if stdout is a TTY")? { - bail!("Refusing to write binary data to terminal"); - } + verify_stdout_not_tty()?; iso.stream(input, &mut io::stdout().lock())?; } Some(output_path) => { @@ -772,6 +768,13 @@ fn copy_file_from_iso(iso: &mut IsoFs, file: &iso9660::File, output_path: &Path) Ok(()) } +fn verify_stdout_not_tty() -> Result<()> { + if isatty(io::stdout().as_raw_fd()).context("checking if stdout is a TTY")? { + bail!("Refusing to write binary data to terminal"); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; From 7e348db03c174e0c8277a558f68471f751f4dbe8 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 5 Nov 2021 16:27:27 -0400 Subject: [PATCH 2/4] live: add IsoConfig::for_iso() Prep for using it in a future patch. --- src/live.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/live.rs b/src/live.rs index 1e9769423..9edac28b4 100644 --- a/src/live.rs +++ b/src/live.rs @@ -264,9 +264,13 @@ impl IsoConfig { pub fn for_file(file: &mut File) -> Result { let mut iso = IsoFs::from_file(file.try_clone().context("cloning file")?) .context("parsing ISO9660 image")?; + IsoConfig::for_iso(&mut iso) + } + + pub fn for_iso(iso: &mut IsoFs) -> Result { Ok(Self { - ignition: ignition_embed_area(&mut iso)?, - kargs: KargEmbedAreas::for_iso(&mut iso)?, + ignition: ignition_embed_area(iso)?, + kargs: KargEmbedAreas::for_iso(iso)?, }) } From f775dca9f9a8c07e9f941385eb1c1d3eaec67859 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 5 Nov 2021 16:28:48 -0400 Subject: [PATCH 3/4] live: add KargEmbedInfo::for_iso() Prep for using it in a future patch. --- src/live.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/live.rs b/src/live.rs index 9edac28b4..8fe0306f3 100644 --- a/src/live.rs +++ b/src/live.rs @@ -470,15 +470,13 @@ struct KargEmbedLocation { offset: u64, } -impl KargEmbedAreas { - // Return Ok(None) if no kargs embed areas exist. +impl KargEmbedInfo { + // Returns Ok(None) if `kargs.json` doesn't exist. pub fn for_iso(iso: &mut IsoFs) -> Result> { let iso_file = match iso.get_path(COREOS_KARG_EMBED_INFO_PATH) { Ok(record) => record.try_into_file()?, // old ISO without info JSON - Err(e) if e.is::() => { - return Self::for_file_via_system_area(iso.as_file()?) - } + Err(e) if e.is::() => return Ok(None), Err(e) => return Err(e), }; let info: KargEmbedInfo = serde_json::from_reader( @@ -486,6 +484,17 @@ impl KargEmbedAreas { .context("reading kargs embed area info")?, ) .context("decoding kargs embed area info")?; + Ok(Some(info)) + } +} + +impl KargEmbedAreas { + // Return Ok(None) if no kargs embed areas exist. + pub fn for_iso(iso: &mut IsoFs) -> Result> { + let info = match KargEmbedInfo::for_iso(iso)? { + Some(info) => info, + None => return Self::for_file_via_system_area(iso.as_file()?), + }; // sanity-check size against a reasonable limit if info.size > COREOS_KARG_EMBED_AREA_MAX_SIZE { From fd6b487e5b5475c8cf457bf4497ef5eb0c406723 Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Fri, 5 Nov 2021 16:30:27 -0400 Subject: [PATCH 4/4] Add support for minimal ISO packing/unpacking This command allows packing into the live ISO a minimal version of itself which does not contain the root squashfs. The use case for minimal ISOs has come up multiple times, notably in: https://github.com/coreos/fedora-coreos-tracker/issues/661 This is a sort of intermediate approach where we don't officially ship a new artifact, but allow users to derive the minimal ISO from the official live ISO. The way this works is similar to osmet. It requires a "packing" step at build time which adds an osmet-like file (the miniso data file) on the full ISO itself. The data file describes how to construct a minimal ISO from the full ISO. It weighs in at a few kilobytes, so the impact of the size of the live ISO is negligible. If we ever officially do ship a minimal ISO, it also allows for the possibility of matching its checksum exactly. The method should be arch-independent and does not rely on e.g. modifying ISO9660 structures or moving the GPT backup header, etc... We support a `--rootfs-url` option which uses the `iso kargs` code to inject `coreos.live.rootfs_url` since that's the primary use case. That way, the minimal ISO can be prepared in a single step. One of the primary goals is to ideally satisfy the needs of the Assisted Installer (https://github.com/openshift/assisted-installer), though there's more things needed before we get there (notably #545). --- .cci.jenkinsfile | 5 +- src/cmdline.rs | 36 +++++++ src/lib.rs | 1 + src/live.rs | 204 ++++++++++++++++++++++++++++++++++- src/main.rs | 2 + src/miniso.rs | 271 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 513 insertions(+), 6 deletions(-) create mode 100644 src/miniso.rs diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 5bc6d5719..efeb607a5 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -22,7 +22,8 @@ cosaPod(buildroot: true, runAsUser: 0) { stage("Build metal+live") { shwrap("cd /srv/fcos && cosa buildextend-metal") shwrap("cd /srv/fcos && cosa buildextend-metal4k") - shwrap("cd /srv/fcos && cosa buildextend-live --fast") + // Enable --miniso until it's the default + shwrap("cd /srv/fcos && cosa buildextend-live --fast --miniso") // Test metal with an uncompressed image and metal4k with a // compressed one shwrap("cd /srv/fcos && cosa compress --fast --artifact=metal4k") @@ -31,7 +32,7 @@ cosaPod(buildroot: true, runAsUser: 0) { // No need to run the iso-live-login/iso-as-disk scenarios fcosKolaTestIso( cosaDir: "/srv/fcos", - scenarios: "pxe-install,pxe-offline-install,iso-install,iso-offline-install", + scenarios: "pxe-install,pxe-offline-install,iso-install,iso-offline-install,miniso-install", scenarios4k: "iso-install,iso-offline-install", skipUEFI: true ) diff --git a/src/cmdline.rs b/src/cmdline.rs index 5cacebc57..b35ea8824 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -100,6 +100,13 @@ pub enum IsoKargsCmd { pub enum IsoExtractCmd { /// Extract PXE files from an ISO image Pxe(IsoExtractPxeConfig), + /// Extract a minimal ISO from a CoreOS live ISO image + MinimalIso(IsoExtractMinimalIsoConfig), + // This doesn't really make sense under `extract`, but it's hidden and conceptually feels + // cleaner being alongside `coreos-installer iso extract minimal-iso`. + /// Pack a minimal ISO into a CoreOS live ISO image + #[structopt(setting(AppSettings::Hidden))] + PackMinimalIso(IsoExtractPackMinimalIsoConfig), } #[derive(Debug, StructOpt)] @@ -409,6 +416,35 @@ pub struct IsoExtractPxeConfig { pub output_dir: String, } +#[derive(Debug, StructOpt)] +pub struct IsoExtractMinimalIsoConfig { + /// ISO image + #[structopt(value_name = "ISO")] + pub input: String, + /// Extract rootfs image as well + #[structopt(long, value_name = "PATH")] + pub output_rootfs: Option, + /// Minimal ISO output file + #[structopt(value_name = "OUTPUT_ISO", default_value = "-")] + pub output: String, + /// Inject rootfs URL karg into minimal ISO + #[structopt(long, value_name = "URL")] + pub rootfs_url: Option, +} + +#[derive(Debug, StructOpt)] +pub struct IsoExtractPackMinimalIsoConfig { + /// ISO image + #[structopt(value_name = "FULL_ISO")] + pub full: String, + /// Minimal ISO image + #[structopt(value_name = "MINIMAL_ISO")] + pub minimal: String, + /// Delete minimal ISO after packing + #[structopt(long)] + pub consume: bool, +} + #[derive(Debug, StructOpt)] pub struct OsmetPackConfig { /// Path to osmet file to write diff --git a/src/lib.rs b/src/lib.rs index 276b8edfc..09707eca2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod install; pub mod io; pub mod iso9660; pub mod live; +pub mod miniso; pub mod osmet; #[cfg(target_arch = "s390x")] pub mod s390x; diff --git a/src/live.rs b/src/live.rs index 8fe0306f3..2dad77441 100644 --- a/src/live.rs +++ b/src/live.rs @@ -18,17 +18,19 @@ use cpio::{write_cpio, NewcBuilder, NewcReader}; use nix::unistd::isatty; use openat_ext::FileExt; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::convert::TryInto; use std::fs::{read, write, File, OpenOptions}; use std::io::{self, copy, BufReader, BufWriter, Cursor, Read, Seek, SeekFrom, Write}; use std::iter::repeat; use std::os::unix::io::AsRawFd; -use std::path::Path; +use std::path::{Path, PathBuf}; use crate::cmdline::*; use crate::install::*; use crate::io::*; use crate::iso9660::{self, IsoFs}; +use crate::miniso; const FILENAME: &str = "config.ign"; const COREOS_IGNITION_EMBED_PATH: &str = "IMAGES/IGNITION.IMG"; @@ -39,6 +41,8 @@ const COREOS_KARG_EMBED_AREA_HEADER_MAX_OFFSETS: usize = 6; const COREOS_KARG_EMBED_AREA_MAX_SIZE: usize = 2048; const COREOS_KARG_EMBED_INFO_PATH: &str = "COREOS/KARGS.JSO"; const COREOS_ISO_PXEBOOT_DIR: &str = "IMAGES/PXEBOOT"; +const COREOS_ISO_ROOTFS_IMG: &str = "IMAGES/PXEBOOT/ROOTFS.IMG"; +const COREOS_ISO_MINISO_FILE: &str = "COREOS/MINISO.DAT"; pub fn iso_embed(config: &IsoEmbedConfig) -> Result<()> { eprintln!("`iso embed` is deprecated; use `iso ignition embed`. Continuing."); @@ -457,14 +461,14 @@ struct KargEmbedAreas { args: String, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize)] struct KargEmbedInfo { default: String, files: Vec, size: usize, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize)] struct KargEmbedLocation { path: String, offset: u64, @@ -486,6 +490,29 @@ impl KargEmbedInfo { .context("decoding kargs embed area info")?; Ok(Some(info)) } + + pub fn update_iso(&self, iso: &mut IsoFs) -> Result<()> { + let iso_file = iso.get_path(COREOS_KARG_EMBED_INFO_PATH)?.try_into_file()?; + let mut w = iso.overwrite_file(&iso_file)?; + let new_json = serde_json::to_string_pretty(&self).context("serializing object")?; + if new_json.len() > iso_file.length as usize { + // This really shouldn't happen. It's only used by the miniso stuff, and there we + // strictly *remove* kargs from the default set. + bail!( + "New version of {} does not fit in space ({} vs {})", + COREOS_KARG_EMBED_INFO_PATH, + new_json.len(), + iso_file.length, + ); + } + + let mut contents = vec![b' '; iso_file.length as usize]; + contents[..new_json.len()].copy_from_slice(new_json.as_bytes()); + w.write_all(&contents) + .with_context(|| format!("failed to update {}", COREOS_KARG_EMBED_INFO_PATH))?; + w.flush().context("flushing ISO")?; + Ok(()) + } } impl KargEmbedAreas { @@ -777,7 +804,176 @@ fn copy_file_from_iso(iso: &mut IsoFs, file: &iso9660::File, output_path: &Path) .with_context(|| format!("opening {}", output_path.display()))?; let mut bufw = BufWriter::with_capacity(BUFFER_SIZE, &mut outf); copy(&mut iso.read_file(file)?, &mut bufw)?; - bufw.flush()?; + bufw.flush().context("flushing buffer")?; + Ok(()) +} + +pub fn iso_extract_minimal_iso(config: &IsoExtractMinimalIsoConfig) -> Result<()> { + // Note we don't support overwriting the input ISO. Unlike other commands, this operation is + // non-reversible, so let's make it harder for users to shoot themselves in the foot. + let mut full_iso = IsoFs::from_file(open_live_iso(&config.input, None)?)?; + + // For now, we require the full ISO to be completely vanilla. Otherwise, the hashes won't + // match. + let iso = IsoConfig::for_iso(&mut full_iso)?; + if iso.have_ignition() { + bail!("Cannot operate on ISO with embedded Ignition config. Reset it and try again."); + } else if iso.kargs()? != iso.kargs_default()? { + bail!("Cannot operate on ISO with non-default kargs. Reset it and try again."); + } + + // do this early so we exit immediately if stdout is a TTY + let output_dir: PathBuf = if &config.output == "-" { + verify_stdout_not_tty()?; + std::env::temp_dir() + } else { + Path::new(&config.output) + .parent() + .with_context(|| format!("no parent directory of {}", &config.output))? + .into() + }; + + if let Some(ref path) = config.output_rootfs { + let rootfs = full_iso + .get_path(COREOS_ISO_ROOTFS_IMG) + .with_context(|| format!("looking up '{}'", COREOS_ISO_ROOTFS_IMG))? + .try_into_file()?; + copy_file_from_iso(&mut full_iso, &rootfs, Path::new(path))?; + } + + let miniso_data_file = full_iso + .get_path(COREOS_ISO_MINISO_FILE) + .with_context(|| format!("looking up '{}'", COREOS_ISO_MINISO_FILE))? + .try_into_file()?; + + let data = { + let mut f = full_iso.read_file(&miniso_data_file)?; + miniso::Data::deserialize(&mut f).context("reading miniso data file")? + }; + let mut outf = tempfile::Builder::new() + .prefix(".coreos-installer-temp-") + .tempfile_in(&output_dir) + .context("creating temporary file")?; + data.unxzpack(full_iso.as_file()?, &mut outf) + .context("unpacking miniso")?; + outf.seek(SeekFrom::Start(0)) + .context("seeking back to start of miniso tempfile")?; + + modify_miniso_kargs(outf.as_file_mut(), config.rootfs_url.as_ref()) + .context("modifying miniso kernel args")?; + + if &config.output == "-" { + copy(&mut outf, &mut io::stdout().lock()).context("writing output")?; + } else { + outf.persist_noclobber(&config.output) + .map_err(|e| e.error)?; + } + + Ok(()) +} + +pub fn iso_pack_minimal_iso(config: &IsoExtractPackMinimalIsoConfig) -> Result<()> { + let mut full_iso = IsoFs::from_file(open_live_iso(&config.full, Some(None))?)?; + let mut minimal_iso = IsoFs::from_file(open_live_iso(&config.minimal, None)?)?; + + let full_files = collect_iso_files(&mut full_iso) + .with_context(|| format!("collecting files from {}", &config.full))?; + let minimal_files = collect_iso_files(&mut minimal_iso) + .with_context(|| format!("collecting files from {}", &config.minimal))?; + if full_files.is_empty() { + bail!("No files found in {}", &config.full); + } else if minimal_files.is_empty() { + bail!("No files found in {}", &config.minimal); + } + + eprintln!("Packing minimal ISO"); + let (data, matches, skipped, written, written_compressed) = + miniso::Data::xzpack(minimal_iso.as_file()?, &full_files, &minimal_files) + .context("packing miniso")?; + eprintln!("Matched {} files of {}", matches, minimal_files.len()); + + eprintln!("Total bytes skipped: {}", skipped); + eprintln!("Total bytes written: {}", written); + eprintln!("Total bytes written (compressed): {}", written_compressed); + + eprintln!("Verifying that packed image matches digest"); + data.unxzpack(full_iso.as_file()?, std::io::sink()) + .context("unpacking miniso for verification")?; + + let miniso_entry = full_iso + .get_path(COREOS_ISO_MINISO_FILE) + .with_context(|| format!("looking up '{}'", COREOS_ISO_MINISO_FILE))? + .try_into_file()?; + let mut w = full_iso.overwrite_file(&miniso_entry)?; + data.serialize(&mut w).context("writing miniso data file")?; + w.flush().context("flushing full ISO")?; + + if config.consume { + std::fs::remove_file(&config.minimal) + .with_context(|| format!("consuming {}", &config.minimal))?; + } + + eprintln!("Packing successful!"); + Ok(()) +} + +fn collect_iso_files(iso: &mut IsoFs) -> Result> { + iso.walk()? + .filter_map(|r| match r { + Err(e) => Some(Err(e)), + Ok((s, iso9660::DirectoryRecord::File(f))) => Some(Ok((s, f))), + Ok(_) => None, + }) + .collect::>>() + .context("while walking ISO filesystem") +} + +fn modify_miniso_kargs(f: &mut File, rootfs_url: Option<&String>) -> Result<()> { + let mut iso = IsoFs::from_file(f.try_clone().context("cloning a file")?)?; + let mut cfg = IsoConfig::for_file(f)?; + + let kargs = cfg.kargs()?; + + // same disclaimer as `modify_kargs()` here re. whitespace/quoting + let liveiso_karg = kargs + .split_ascii_whitespace() + .find(|&karg| karg.starts_with("coreos.liveiso=")) + .ok_or_else(|| anyhow!("minimal ISO does not have coreos.liveiso= karg"))? + .to_string(); + + let new_default_kargs = modify_kargs(kargs, &[], &[], &[], &[liveiso_karg])?; + cfg.set_kargs(&new_default_kargs)?; + + if let Some(url) = rootfs_url { + if url.split_ascii_whitespace().count() > 1 { + bail!("forbidden whitespace found in '{}'", url); + } + let final_kargs = modify_kargs( + &new_default_kargs, + vec![format!("coreos.live.rootfs_url={}", url)].as_slice(), + &[], + &[], + &[], + )?; + + cfg.set_kargs(&final_kargs)?; + } + + // update kargs + write_live_iso(&cfg, f, None)?; + + // also modify the default kargs because we don't want `coreos-installer iso kargs reset` to + // re-add `coreos.liveiso` + let mut kargs_info = KargEmbedInfo::for_iso(&mut iso)?.ok_or_else(|| { + // should be impossible; we only support new-style CoreOS ISOs with kargs.json + anyhow!("minimal ISO does not have kargs.json; please report this as a bug") + })?; + + // NB: We don't need to update the length for this; it's a fixed property of the kargs files. + // (Though its original value did depend on the original default kargs at build time.) + kargs_info.default = new_default_kargs; + kargs_info.update_iso(&mut iso)?; + Ok(()) } diff --git a/src/main.rs b/src/main.rs index 3cc3ca048..1be376de2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -41,6 +41,8 @@ fn main() -> Result<()> { IsoCmd::Inspect(c) => live::iso_inspect(&c), IsoCmd::Extract(c) => match c { IsoExtractCmd::Pxe(c) => live::iso_extract_pxe(&c), + IsoExtractCmd::MinimalIso(c) => live::iso_extract_minimal_iso(&c), + IsoExtractCmd::PackMinimalIso(c) => live::iso_pack_minimal_iso(&c), }, }, Cmd::Osmet(c) => match c { diff --git a/src/miniso.rs b/src/miniso.rs new file mode 100644 index 000000000..63bea6c06 --- /dev/null +++ b/src/miniso.rs @@ -0,0 +1,271 @@ +// Copyright 2021 Red Hat, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::convert::TryInto; +use std::fs::File; +use std::io::{copy, Read, Seek, SeekFrom, Write}; + +use anyhow::{anyhow, bail, Context, Result}; +use bincode::Options; +use serde::{Deserialize, Serialize}; +use structopt::clap::crate_version; +use xz2::read::XzDecoder; +use xz2::write::XzEncoder; + +use crate::io::*; +use crate::iso9660; + +/// Magic header value for miniso data file. +const HEADER_MAGIC: [u8; 8] = *b"MINISO\0\0"; + +/// Basic versioning. Used as a safety check that we're unpacking a miniso data file we understand. +/// Bump this when making changes to the format. +const HEADER_VERSION: u32 = 1; + +/// Maximum size of miniso data file we'll agree to deserialize. FCOS is currently +/// at 2892 bytes, so this is generous. +const DATA_MAX_SIZE: u64 = 1024 * 1024; + +#[derive(Serialize, Deserialize, Debug)] +struct Table { + entries: Vec, +} + +impl Table { + fn new( + full_files: &HashMap, + minimal_files: &HashMap, + ) -> Result { + let mut entries: Vec = Vec::new(); + for (path, minimal_entry) in minimal_files { + let full_entry = full_files + .get(path) + .ok_or_else(|| anyhow!("missing minimal file {} in full ISO", path))?; + if full_entry.length != minimal_entry.length { + bail!( + "File {} has different lengths in full and minimal ISOs", + path + ); + } + entries.push(TableEntry { + minimal: minimal_entry.address, + full: full_entry.address, + length: full_entry.length, + }); + } + + entries.sort_by_key(|e| e.minimal.as_sector()); + let table = Table { entries }; + table.validate().context("validating table")?; + Ok(table) + } + + fn validate(&self) -> Result<()> { + let n = self.entries.len(); + if n == 0 { + bail!("table is empty; ISOs have no files in common?"); + } + for (e, next_e) in self.entries[..n - 1].iter().zip(self.entries[1..n].iter()) { + if e.minimal.as_offset() + e.length as u64 > next_e.minimal.as_offset() { + bail!( + "Files at offsets {} and {} overlap", + e.minimal.as_offset(), + next_e.minimal.as_offset(), + ); + } + } + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug)] +struct TableEntry { + minimal: iso9660::Address, + full: iso9660::Address, + length: u32, +} + +// Version-agnostic header. Frozen. +#[derive(Serialize, Deserialize, Debug)] +struct Header { + magic: [u8; 8], + version: u32, + /// For informational purposes only. + app_version: String, +} + +impl Default for Header { + fn default() -> Self { + Self { + magic: HEADER_MAGIC, + version: HEADER_VERSION, + app_version: crate_version!().into(), + } + } +} + +impl Header { + pub fn validate(&self) -> Result<()> { + if self.magic != HEADER_MAGIC { + bail!("not a miniso file!"); + } + if self.version != HEADER_VERSION { + bail!( + "incompatible miniso file version: {} vs {} (created by {})", + HEADER_VERSION, + self.version, + self.app_version, + ); + } + Ok(()) + } +} + +// Version-specific payload. Evolvable. +#[derive(Serialize, Deserialize, Debug)] +pub struct Data { + table: Table, + digest: Sha256Digest, + xzpacked: Vec, +} + +impl Data { + pub fn xzpack( + miniso: &mut File, + full_files: &HashMap, + minimal_files: &HashMap, + ) -> Result<(Self, usize, u64, u64, u64)> { + let table = Table::new(full_files, minimal_files)?; + + // A `ReadHasher` here would let us wrap the miniso so we calculate the digest as we read. + let digest = Sha256Digest::from_file(miniso)?; + let mut offset = miniso + .seek(SeekFrom::Start(0)) + .context("seeking back to miniso start")?; + + let mut xzw = XzEncoder::new(Vec::new(), 9); + let mut buf = [0u8; BUFFER_SIZE]; + let mut skipped: u64 = 0; + for entry in &table.entries { + let addr: u64 = entry.minimal.as_offset(); + assert!(offset <= addr); + if addr > offset { + copy_exactly_n(miniso, &mut xzw, addr - offset, &mut buf).with_context(|| { + format!( + "copying {} miniso bytes at offset {}", + addr - offset, + offset + ) + })?; + } + // I tested trying to be smarter here and rounding to the nearest 2k block so we can + // skip padding, but zeroes compress so well that it only saved a grand total of 4 + // bytes after xz. So not worth the complexity. + offset = miniso + .seek(SeekFrom::Current(entry.length as i64)) + .with_context(|| format!("skipping miniso file at offset {}", addr))?; + skipped += entry.length as u64; + } + + copy(miniso, &mut xzw).context("copying remaining miniso bytes")?; + + xzw.try_finish().context("trying to finish xz stream")?; + let matches = table.entries.len(); + let written = xzw.total_in(); + let written_compressed = xzw.total_out(); + Ok(( + Self { + table, + digest, + xzpacked: xzw.finish().context("finishing xz stream")?, + }, + matches, + skipped, + written, + written_compressed, + )) + } + + pub fn serialize(&self, w: impl Write) -> Result<()> { + let mut limiter = LimitWriter::new(w, DATA_MAX_SIZE, "data size limit".into()); + + let header = Header::default(); + let coder = &mut bincoder(); + coder + .serialize_into(&mut limiter, &header) + .context("failed to serialize header")?; + coder + .serialize_into(&mut limiter, &self) + .context("failed to serialize data")?; + + Ok(()) + } + + pub fn deserialize(r: impl Read) -> Result { + let mut limiter = LimitReader::new(r, DATA_MAX_SIZE, "data size limit".into()); + + let coder = &mut bincoder(); + let header: Header = coder + .deserialize_from(&mut limiter) + .context("failed to deserialize header")?; + header.validate().context("validating header")?; + + let data: Self = coder + .deserialize_from(&mut limiter) + .context("failed to deserialize data")?; + data.table.validate().context("validating table")?; + + Ok(data) + } + + pub fn unxzpack(&self, fulliso: &mut File, w: impl Write) -> Result<()> { + let mut xzr = XzDecoder::new(self.xzpacked.as_slice()); + let mut w = WriteHasher::new_sha256(w)?; + let mut buf = [0u8; BUFFER_SIZE]; + let mut offset = 0; + for entry in &self.table.entries { + let minimal_addr = entry.minimal.as_offset(); + let fulliso_addr = entry.full.as_offset(); + if minimal_addr > offset { + offset += copy_exactly_n(&mut xzr, &mut w, minimal_addr - offset, &mut buf) + .with_context(|| { + format!( + "copying {} packed bytes at offset {}", + minimal_addr - offset, + offset + ) + })?; + } + fulliso + .seek(SeekFrom::Start(fulliso_addr)) + .with_context(|| format!("seeking to full ISO file at offset {}", fulliso_addr))?; + offset += copy_exactly_n(fulliso, &mut w, entry.length as u64, &mut buf) + .with_context(|| format!("copying full ISO file at offset {}", fulliso_addr))?; + } + + copy(&mut xzr, &mut w).context("copying remaining packed bytes")?; + let digest = w.try_into()?; + if self.digest != digest { + bail!( + "wrong final digest: expected {}, found {}", + self.digest.to_hex_string()?, + digest.to_hex_string()? + ); + } + + Ok(()) + } +}