From db150b1d888e7bd3a3d784566273da1a3f62ebb5 Mon Sep 17 00:00:00 2001 From: llenotre Date: Mon, 6 May 2024 23:48:17 +0200 Subject: [PATCH] refactor: cleanup archive decompression --- Cargo.lock | 123 ++++++++++++++++++++++++++++--------------- builder/src/build.rs | 2 - builder/src/desc.rs | 13 ++--- builder/src/main.rs | 2 +- common/Cargo.toml | 1 + common/src/util.rs | 80 ++++++++-------------------- 6 files changed, 108 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 583b848..71c8b9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -258,47 +258,48 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.13" +version = "0.6.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", + "is_terminal_polyfill", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" [[package]] name = "anstyle-parse" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +checksum = "a64c907d4e79225ac72e2a354c9ce84d50ebb4586dee56c82b3ee73004f537f5" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.2" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -306,15 +307,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" [[package]] name = "autocfg" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" @@ -339,9 +340,9 @@ checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "base64" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" @@ -424,6 +425,12 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" version = "1.6.0" @@ -462,15 +469,26 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.95" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +checksum = "099a5357d84c4c61eb35fc8eafa9a79a902c2f76911e5747ced4e032edd8d9b4" dependencies = [ "jobserver", "libc", "once_cell", ] +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -479,9 +497,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "colorchoice" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" [[package]] name = "common" @@ -493,6 +511,7 @@ dependencies = [ "flate2", "futures", "futures-util", + "infer", "reqwest", "serde", "serde_json", @@ -801,9 +820,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -1011,12 +1030,27 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "infer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" +dependencies = [ + "cfb", +] + [[package]] name = "ipnet" version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itoa" version = "1.0.11" @@ -1055,9 +1089,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.153" +version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" [[package]] name = "linux-raw-sys" @@ -1431,7 +1465,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "bytes", "encoding_rs", "futures-channel", @@ -1504,7 +1538,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.22.0", + "base64 0.22.1", "rustls-pki-types", ] @@ -1537,11 +1571,11 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -1550,9 +1584,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -1566,18 +1600,18 @@ checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "serde" -version = "1.0.199" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" +checksum = "ddc6f9cc94d67c0e21aaf7eda3a010fd3af78ebf6e096aa6e2e13c79749cce4f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.199" +version = "1.0.200" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" +checksum = "856f046b9400cee3c8c94ed572ecdb752444c24528c035cd35882aad6f492bcb" dependencies = [ "proc-macro2", "quote", @@ -1812,16 +1846,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -1931,6 +1964,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" +[[package]] +name = "uuid" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" + [[package]] name = "v_htmlescape" version = "0.15.8" @@ -2224,18 +2263,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "087eca3c1eaf8c47b94d02790dd086cd594b912d2043d4de4bfdd466b3befb7c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "6f4b6c273f496d8fd4eaf18853e6b448760225dc030ff2c485a786859aea6393" dependencies = [ "proc-macro2", "quote", diff --git a/builder/src/build.rs b/builder/src/build.rs index f611bfb..40cc302 100644 --- a/builder/src/build.rs +++ b/builder/src/build.rs @@ -114,7 +114,6 @@ impl BuildProcess { /// `output_path` is the path at which the package's archive will be created. pub fn create_archive(&self, output_path: &Path) -> io::Result<()> { let build_desc_path = self.input_path.join("package.json"); - let tar_gz = File::create(output_path)?; let enc = GzEncoder::new(tar_gz, Compression::default()); let mut tar = tar::Builder::new(enc); @@ -122,7 +121,6 @@ impl BuildProcess { tar.append_path_with_name(build_desc_path, "package.json")?; tar.append_dir_all("data", &self.sysroot)?; // TODO add install/update/remove hooks - tar.finish() } diff --git a/builder/src/desc.rs b/builder/src/desc.rs index 00dcf81..eefbe4c 100644 --- a/builder/src/desc.rs +++ b/builder/src/desc.rs @@ -61,7 +61,7 @@ pub struct Source { } impl Source { - /// Fetches files from the source and uncompresses them if necessary. + /// Fetches files from the source and decompress them if necessary. /// /// Files are placed into the build directory `build_dir` according to the specified location. pub async fn fetch(&self, build_dir: &Path) -> Result<()> { @@ -75,9 +75,8 @@ impl Source { if metadata.is_dir() { common::util::recursive_copy(path, &dest_path)?; } else { - // TODO uncompress only if it is an actual archive - // Uncompress tarball - common::util::uncompress(path, &dest_path, self.unwrap)?; + // TODO decompress only if it is an actual archive + common::util::decompress(path, &dest_path, self.unwrap)?; } } @@ -97,12 +96,8 @@ impl Source { let mut download_task = DownloadTask::new(url, &path).await?; // TODO progress bar while download_task.next().await? {} - // TODO check integrity with hash if specified - - // Uncompress the archive - common::util::uncompress(&path, &dest_path, self.unwrap)?; - + common::util::decompress(&path, &dest_path, self.unwrap)?; // TODO remove archive? } diff --git a/builder/src/main.rs b/builder/src/main.rs index 86497a7..ef4c67f 100644 --- a/builder/src/main.rs +++ b/builder/src/main.rs @@ -6,8 +6,8 @@ mod util; use crate::build::BuildProcess; use crate::util::{get_build_triplet, get_jobs_count}; -use anyhow::{anyhow, bail}; use anyhow::Result; +use anyhow::{anyhow, bail}; use common::repository::Repository; use std::env; use std::fs; diff --git a/common/Cargo.toml b/common/Cargo.toml index be743fb..3181342 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -13,6 +13,7 @@ bzip2 = "0.4.4" flate2 = "1.0.30" futures = "0.3.30" futures-util = "0.3.30" +infer = "0.15.0" reqwest = { version = "0.12.4", features = ["blocking", "json", "stream"], optional = true } serde = { version = "1.0.199", features = ["derive"] } serde_json = "1.0.116" diff --git a/common/src/util.rs b/common/src/util.rs index 727ea65..ede9547 100644 --- a/common/src/util.rs +++ b/common/src/util.rs @@ -12,9 +12,8 @@ use std::io::BufReader; use std::io::BufWriter; use std::io::Read; use std::os::unix; -use std::path::Component::Normal; -use std::path::Path; use std::path::PathBuf; +use std::path::{Component, Path}; use std::process::Command; use tar::Archive; use xz2::read::XzDecoder; @@ -59,84 +58,57 @@ pub fn create_tmp_file() -> io::Result<(PathBuf, File)> { } } -fn uncompress_(mut archive: Archive, dest: &Path, unwrap: bool) -> io::Result<()> { +fn decompress_impl(stream: R, dest: &Path, unwrap: bool) -> io::Result<()> { + let mut archive = Archive::new(stream); archive.set_overwrite(true); archive.set_preserve_permissions(true); - // TODO undo on fail? if unwrap { for entry in archive.entries()? { let mut entry = entry?; - let path: PathBuf = entry .path()? .components() .skip(1) - .filter(|c| matches!(c, Normal(_))) + .filter(|c| matches!(c, Component::Normal(_))) .collect(); let path = dest.join(path); - entry.unpack(path)?; } } else { archive.unpack(dest)?; } - Ok(()) } -/// Uncompresses the given archive file `src` to the given location `dest`. +/// Decompresses the given archive file `src` to the given location `dest`. /// /// `unwrap` tells whether the tarball shall be unwrapped. /// /// If the tarball contains directories at the root, the unwrap operation unwraps their content /// instead of the directories themselves. -pub fn uncompress(src: &Path, dest: &Path, unwrap: bool) -> io::Result<()> { - // Try to uncompress .tar.gz - { - let file = File::open(src)?; - let tar = GzDecoder::new(file); - let archive = Archive::new(tar); - - if uncompress_(archive, dest, unwrap).is_ok() { - return Ok(()); - } - } - - // Try to uncompress .tar.xz - { - let file = File::open(src)?; - let tar = XzDecoder::new(file); - let archive = Archive::new(tar); - - if uncompress_(archive, dest, unwrap).is_ok() { - return Ok(()); - } - } - - // Try to uncompress .tar.bz2 - { - let file = File::open(src)?; - let tar = BzDecoder::new(file); - let archive = Archive::new(tar); - - uncompress_(archive, dest, unwrap) +pub fn decompress(src: &Path, dest: &Path, unwrap: bool) -> io::Result<()> { + let file_type = infer::get_from_path(src)?.map(|t| t.mime_type()); + let file = File::open(src)?; + match file_type { + Some("application/gzip") => decompress_impl(GzDecoder::new(file), dest, unwrap), + Some("application/x-xz") => decompress_impl(XzDecoder::new(file), dest, unwrap), + Some("application/x-bzip2") => decompress_impl(BzDecoder::new(file), dest, unwrap), + _ => Err(io::Error::new( + io::ErrorKind::Other, + "Invalid or unsupported archive format", + )), } } -/// Uncompresses the given .tar.gz file `archive` into a temporary directory, executes the given +/// Decompresses the given .tar.gz file `archive` into a temporary directory, executes the given /// function `f` with the path to the temporary directory as argument, then removes the directory /// and returns the result of the call to `f`. -pub fn uncompress_wrap T>(archive: &Path, f: F) -> io::Result { - // Uncompressing +pub fn decompress_wrap T>(archive: &Path, f: F) -> io::Result { let tmp_dir = create_tmp_dir()?; - uncompress(archive, &tmp_dir, false)?; - + decompress(archive, &tmp_dir, false)?; let v = f(&tmp_dir); - - // Remove temporary directory fs::remove_dir_all(&tmp_dir)?; - Ok(v) } @@ -160,11 +132,9 @@ pub fn run_hook(hook_path: &Path, sysroot: &Path) -> io::Result { if !Path::new(hook_path).exists() { return Ok(true); } - let status = Command::new(hook_path) .env("SYSROOT", sysroot.as_os_str()) .status()?; - Ok(status.success()) } @@ -221,22 +191,14 @@ pub fn print_size(mut size: u64) { pub fn read_json Deserialize<'a>>(file: &Path) -> io::Result { let file = File::open(file)?; let reader = BufReader::new(file); - - serde_json::from_reader(reader).map_err(|e| { - let msg = format!("JSON deserializing failed: {e}"); - io::Error::new(io::ErrorKind::Other, msg) - }) + Ok(serde_json::from_reader(reader)?) } /// Writes a JSON file. pub fn write_json(file: &Path, data: &T) -> io::Result<()> { let file = File::create(file)?; let writer = BufWriter::new(file); - - serde_json::to_writer_pretty(writer, &data).map_err(|e| { - let msg = format!("JSON serializing failed: {e}"); - io::Error::new(io::ErrorKind::Other, msg) - }) + Ok(serde_json::to_writer_pretty(writer, &data)?) } /// Concatenates the given paths.