From 7b0f07f2f010a0be4bdd26ea518197ad77f1ccab Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 23 Jan 2025 15:05:09 -0500 Subject: [PATCH] compose: Drop rpmdb sqlite journaling files if rpmdb-normalize See https://github.com/rpm-software-management/rpm/issues/2219 This is one of the things that makes builds unreproducible in general, which is worth fixing alone. But the thing immediately driving this now for me is that I think we're getting some ill-defined behavior because we may have these files hardlinked (via ostree) and depending on the container build environment, we may or may not see modifications "through" the hardlink: https://docs.kernel.org/filesystems/overlayfs.html#index If we happen to mutate the `rpmdb.sqlite-shm` file in one path but not the other confusion could easily result. (Actually what we want to do really is drop our other hardlinked copies of the rpmdb entirely, but that's a bigger change) Out of conservatism for now, we only do this if `rpmdb-normalize` is set (which none of the Fedora derivatives set today AFAICS). I do think we should likely do this in client side layering too, but this reduces the blast radius for now. I plan to enable this in fedora-bootc. Signed-off-by: Colin Walters --- Cargo.lock | 61 +++++++++++++++++++++++++++++++ Cargo.toml | 1 + rust/src/normalization.rs | 24 +++++++++++- tests/compose/libbasic-test.sh | 2 + tests/compose/test-misc-tweaks.sh | 7 ++++ 5 files changed, 94 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 7998ad2907..4967d8e7f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,18 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -853,6 +865,18 @@ dependencies = [ "rand", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -1248,6 +1272,9 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -1255,6 +1282,15 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.4.1" @@ -1776,6 +1812,16 @@ dependencies = [ "redox_syscall", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "libsystemd" version = "0.7.0" @@ -2677,6 +2723,7 @@ dependencies = [ "regex", "reqwest 0.12.12", "rpmostree-client", + "rusqlite", "rust-ini", "rustix", "serde", @@ -2695,6 +2742,20 @@ dependencies = [ "xmlrpc", ] +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags 2.8.0", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rust-ini" version = "0.21.1" diff --git a/Cargo.toml b/Cargo.toml index bb3df996fa..868281a622 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,6 +79,7 @@ phf = { version = "0.11", features = ["macros"] } rand = "0.8.5" rayon = "1.10.0" regex = "1.10" +rusqlite = "0.32.1" reqwest = { version = "0.12", features = ["native-tls", "blocking", "gzip"] } rpmostree-client = { path = "rust/rpmostree-client", version = "0.1.0" } rust-ini = "0.21.1" diff --git a/rust/src/normalization.rs b/rust/src/normalization.rs index 85d3f452c4..708e544429 100644 --- a/rust/src/normalization.rs +++ b/rust/src/normalization.rs @@ -7,13 +7,14 @@ use crate::bwrap::Bubblewrap; use crate::nameservice::shadow::parse_shadow_content; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; use cap_std::fs::OpenOptionsExt; use cap_std::fs::{Dir, OpenOptions}; use cap_std_ext::cap_std; use fn_error_context::context; use ostree_ext::gio; use std::io::{BufReader, Read, Seek, SeekFrom, Write}; +use std::os::fd::{AsFd, AsRawFd}; use std::path::Path; pub(crate) fn source_date_epoch() -> Option { @@ -129,6 +130,27 @@ pub(crate) fn rewrite_rpmdb_timestamps(rpmdb: &mut F) -> #[context("Rewriting rpmdb database files for build stability")] pub(crate) fn normalize_rpmdb(rootfs: &Dir, rpmdb_path: impl AsRef) -> Result<()> { + let rpmdb_path = rpmdb_path.as_ref(); + { + // Unconditionally clean up the sqlite SHM file if it exists. + // https://github.com/rpm-software-management/rpm/issues/2219 + const RPMDB_SQLITE: &str = "rpmdb.sqlite"; + const RPMDB_SQLITE_SHM: &str = "rpmdb.sqlite-shm"; + let subdir = rootfs.open_dir(rpmdb_path)?; + if subdir.try_exists(RPMDB_SQLITE_SHM)? { + tracing::debug!("Cleaning up {RPMDB_SQLITE_SHM}"); + let procpath = format!( + "/proc/self/fd/{}/{RPMDB_SQLITE}", + subdir.as_fd().as_raw_fd() + ); + let conn = rusqlite::Connection::open(&procpath) + .with_context(|| format!("Opening {RPMDB_SQLITE}"))?; + conn.pragma_update(None, "journal_mode", "DELETE")?; + conn.close().map_err(|r| r.1)?; + } + } + + // If SOURCE_DATE_EPOCH isn't set then we don't attempt to rewrite the database. let source_date = if let Some(source_date) = source_date_epoch() { source_date as u32 } else { diff --git a/tests/compose/libbasic-test.sh b/tests/compose/libbasic-test.sh index 3f7c6d8ae5..83e433e41e 100644 --- a/tests/compose/libbasic-test.sh +++ b/tests/compose/libbasic-test.sh @@ -43,6 +43,8 @@ echo "ok etc/default/useradd" for path in /usr/share/rpm /usr/lib/sysimage/rpm-ostree-base-db; do ostree --repo=${repo} ls -R ${treeref} ${path} > db.txt assert_file_has_content_literal db.txt rpmdb.sqlite + # Verify we *aren't* normalizing yet + assert_file_has_content_literal db.txt rpmdb.sqlite-shm done ostree --repo=${repo} ls ${treeref} /usr/lib/sysimage/rpm >/dev/null echo "ok db" diff --git a/tests/compose/test-misc-tweaks.sh b/tests/compose/test-misc-tweaks.sh index fa04441926..079c80f861 100755 --- a/tests/compose/test-misc-tweaks.sh +++ b/tests/compose/test-misc-tweaks.sh @@ -50,6 +50,7 @@ cat > config/other.yaml <<'EOF' recommends: true selinux-label-version: 1 readonly-executables: true +rpmdb-normalize: true container-cmd: - /usr/bin/bash opt-usrlocal: "root" @@ -188,6 +189,12 @@ ostree --repo=${repo} ls ${treeref} /usr/etc > out.txt assert_file_has_content out.txt 'etc/sharedfile' echo "ok remove-from-packages" +# Verify rpmdb-normalize +ostree --repo=${repo} ls -R ${treeref} /usr/share/rpm > db.txt +assert_file_has_content_literal db.txt rpmdb.sqlite +assert_not_file_has_content_literal db.txt rpmdb.sqlite-shm +echo "ok db" + ostree --repo=${repo} ls ${treeref} /opt > ls.txt assert_file_has_content ls.txt '^d0' ostree --repo=${repo} ls ${treeref} /usr/local > ls.txt