From b31ad516eb24358aed178b546f12427e8ffb52a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Thu, 31 Aug 2023 15:07:25 +0200 Subject: [PATCH 01/11] style(xtask): explicitly specify self for module imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 2f9202f090..908923ac34 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -9,10 +9,11 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; use anyhow::Result; -use arch::Arch; -use archive::Archive; use xshell::{cmd, Shell}; +use self::arch::Arch; +use self::archive::Archive; + fn main() -> Result<()> { flags::Xtask::from_env()?.run() } From 4a958e348030e26d0ec2735fb5874a1e59d6f6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Thu, 31 Aug 2023 15:11:19 +0200 Subject: [PATCH 02/11] refactor(xtask): move build and clippy tasks to new modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/build.rs | 176 ++++++++++++++++++++++++++++++++++++ xtask/src/clippy.rs | 38 ++++++++ xtask/src/main.rs | 213 ++------------------------------------------ 3 files changed, 219 insertions(+), 208 deletions(-) create mode 100644 xtask/src/build.rs create mode 100644 xtask/src/clippy.rs diff --git a/xtask/src/build.rs b/xtask/src/build.rs new file mode 100644 index 0000000000..ba092e737d --- /dev/null +++ b/xtask/src/build.rs @@ -0,0 +1,176 @@ +use std::env::{self, VarError}; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use xshell::cmd; + +use crate::archive::Archive; +use crate::flags; + +impl flags::Build { + pub fn run(self) -> Result<()> { + let sh = crate::sh()?; + + eprintln!("Building kernel"); + cmd!(sh, "cargo build") + .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) + .args(self.arch.cargo_args()) + .args(self.target_dir_args()) + .args(self.no_default_features_args()) + .args(self.features_args()) + .args(self.profile_args()) + .run()?; + + let build_archive = self.build_archive(); + let dist_archive = self.dist_archive(); + eprintln!( + "Copying {} to {}", + build_archive.as_ref().display(), + dist_archive.as_ref().display() + ); + sh.create_dir(dist_archive.as_ref().parent().unwrap())?; + sh.copy_file(&build_archive, &dist_archive)?; + + eprintln!("Exporting symbols"); + self.export_syms()?; + + eprintln!("Building hermit-builtins"); + cmd!(sh, "cargo build") + .arg("--manifest-path=hermit-builtins/Cargo.toml") + .args(self.arch.builtins_cargo_args()) + .args(self.target_dir_args()) + .args(self.profile_args()) + .run()?; + + eprintln!("Exporting hermit-builtins symbols"); + let builtins = self.builtins_archive(); + let builtin_symbols = sh.read_file("hermit-builtins/exports")?; + builtins.retain_symbols(builtin_symbols.lines())?; + + dist_archive.append(&builtins)?; + + eprintln!("Setting OSABI"); + dist_archive.set_osabi()?; + + eprintln!("Kernel available at {}", dist_archive.as_ref().display()); + Ok(()) + } + + fn cargo_encoded_rustflags(&self) -> Result { + let outer_rustflags = match env::var("CARGO_ENCODED_RUSTFLAGS") { + Ok(s) => Some(s), + Err(VarError::NotPresent) => None, + Err(err) => return Err(err.into()), + }; + let mut rustflags = outer_rustflags + .as_deref() + .map(|s| vec![s]) + .unwrap_or_default(); + + // TODO: Re-enable mutable-noalias + // https://github.com/hermitcore/kernel/issues/200 + rustflags.push("-Zmutable-noalias=no"); + + if self.instrument_mcount { + rustflags.push("-Zinstrument-mcount"); + } + + if self.randomize_layout { + rustflags.push("-Zrandomize-layout") + } + + rustflags.extend(self.arch.rustflags()); + + Ok(rustflags.join("\x1f")) + } + + fn target_dir_args(&self) -> [&OsStr; 2] { + ["--target-dir".as_ref(), self.target_dir().as_ref()] + } + + fn no_default_features_args(&self) -> &[&str] { + if self.no_default_features { + &["--no-default-features"] + } else { + &[] + } + } + + fn features_args(&self) -> impl Iterator { + self.features + .iter() + .flat_map(|feature| ["--features", feature.as_str()]) + } + + fn profile_args(&self) -> [&str; 2] { + ["--profile", self.profile()] + } + + fn export_syms(&self) -> Result<()> { + let archive = self.dist_archive(); + + let syscall_symbols = archive.syscall_symbols()?; + let explicit_exports = [ + "_start", + "__bss_start", + "runtime_entry", + // lwIP functions (C runtime) + "init_lwip", + "lwip_read", + "lwip_write", + // lwIP rtl8139 driver + "init_rtl8139_netif", + "irq_install_handler", + "virt_to_phys", + "eoi", + ] + .into_iter(); + + let symbols = explicit_exports.chain(syscall_symbols.iter().map(String::as_str)); + + archive.retain_symbols(symbols)?; + + Ok(()) + } + + fn profile(&self) -> &str { + self.profile + .as_deref() + .unwrap_or(if self.release { "release" } else { "dev" }) + } + + fn target_dir(&self) -> &Path { + self.target_dir + .as_deref() + .unwrap_or_else(|| Path::new("target")) + } + + fn out_dir(&self, triple: impl AsRef) -> PathBuf { + let mut out_dir = self.target_dir().to_path_buf(); + out_dir.push(triple); + out_dir.push(match self.profile() { + "dev" => "debug", + profile => profile, + }); + out_dir + } + + fn builtins_archive(&self) -> Archive { + let mut builtins_archive = self.out_dir(self.arch.hermit_triple()); + builtins_archive.push("libhermit_builtins.a"); + builtins_archive.into() + } + + fn build_archive(&self) -> Archive { + let mut built_archive = self.out_dir(self.arch.triple()); + built_archive.push("libhermit.a"); + built_archive.into() + } + + fn dist_archive(&self) -> Archive { + let mut dist_archive = self.out_dir(self.arch.name()); + dist_archive.push("libhermit.a"); + dist_archive.into() + } +} diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs new file mode 100644 index 0000000000..47a29f7a82 --- /dev/null +++ b/xtask/src/clippy.rs @@ -0,0 +1,38 @@ +use anyhow::Result; +use xshell::cmd; + +use crate::arch::Arch; +use crate::flags; + +impl flags::Clippy { + pub fn run(self) -> Result<()> { + let sh = crate::sh()?; + + for target in [Arch::X86_64, Arch::AArch64] { + let target_args = target.cargo_args(); + cmd!(sh, "cargo clippy {target_args...}").run()?; + cmd!(sh, "cargo clippy {target_args...}") + .arg("--no-default-features") + .run()?; + cmd!(sh, "cargo clippy {target_args...}") + .arg("--no-default-features") + .arg("--features=acpi,fsgsbase,pci,smp,vga") + .run()?; + // TODO: Enable clippy for newlib + // https://github.com/hermitcore/kernel/issues/470 + // cmd!(sh, "cargo clippy {target_args...}") + // .arg("--no-default-features") + // .arg("--features=acpi,fsgsbase,newlib,smp,vga") + // .run()?; + } + + cmd!(sh, "cargo clippy") + .arg("--manifest-path=hermit-builtins/Cargo.toml") + .arg("--target=x86_64-unknown-none") + .run()?; + + cmd!(sh, "cargo clippy --package xtask").run()?; + + Ok(()) + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 908923ac34..927de8faca 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,17 +2,14 @@ mod arch; mod archive; +mod build; +mod clippy; mod flags; -use std::env::{self, VarError}; -use std::ffi::OsStr; -use std::path::{Path, PathBuf}; +use std::path::Path; use anyhow::Result; -use xshell::{cmd, Shell}; - -use self::arch::Arch; -use self::archive::Archive; +use xshell::Shell; fn main() -> Result<()> { flags::Xtask::from_env()?.run() @@ -27,207 +24,7 @@ impl flags::Xtask { } } -impl flags::Build { - fn run(self) -> Result<()> { - let sh = sh()?; - - eprintln!("Building kernel"); - cmd!(sh, "cargo build") - .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) - .args(self.arch.cargo_args()) - .args(self.target_dir_args()) - .args(self.no_default_features_args()) - .args(self.features_args()) - .args(self.profile_args()) - .run()?; - - let build_archive = self.build_archive(); - let dist_archive = self.dist_archive(); - eprintln!( - "Copying {} to {}", - build_archive.as_ref().display(), - dist_archive.as_ref().display() - ); - sh.create_dir(dist_archive.as_ref().parent().unwrap())?; - sh.copy_file(&build_archive, &dist_archive)?; - - eprintln!("Exporting symbols"); - self.export_syms()?; - - eprintln!("Building hermit-builtins"); - cmd!(sh, "cargo build") - .arg("--manifest-path=hermit-builtins/Cargo.toml") - .args(self.arch.builtins_cargo_args()) - .args(self.target_dir_args()) - .args(self.profile_args()) - .run()?; - - eprintln!("Exporting hermit-builtins symbols"); - let builtins = self.builtins_archive(); - let builtin_symbols = sh.read_file("hermit-builtins/exports")?; - builtins.retain_symbols(builtin_symbols.lines())?; - - dist_archive.append(&builtins)?; - - eprintln!("Setting OSABI"); - dist_archive.set_osabi()?; - - eprintln!("Kernel available at {}", dist_archive.as_ref().display()); - Ok(()) - } - - fn cargo_encoded_rustflags(&self) -> Result { - let outer_rustflags = match env::var("CARGO_ENCODED_RUSTFLAGS") { - Ok(s) => Some(s), - Err(VarError::NotPresent) => None, - Err(err) => return Err(err.into()), - }; - let mut rustflags = outer_rustflags - .as_deref() - .map(|s| vec![s]) - .unwrap_or_default(); - - // TODO: Re-enable mutable-noalias - // https://github.com/hermitcore/kernel/issues/200 - rustflags.push("-Zmutable-noalias=no"); - - if self.instrument_mcount { - rustflags.push("-Zinstrument-mcount"); - } - - if self.randomize_layout { - rustflags.push("-Zrandomize-layout") - } - - rustflags.extend(self.arch.rustflags()); - - Ok(rustflags.join("\x1f")) - } - - fn target_dir_args(&self) -> [&OsStr; 2] { - ["--target-dir".as_ref(), self.target_dir().as_ref()] - } - - fn no_default_features_args(&self) -> &[&str] { - if self.no_default_features { - &["--no-default-features"] - } else { - &[] - } - } - - fn features_args(&self) -> impl Iterator { - self.features - .iter() - .flat_map(|feature| ["--features", feature.as_str()]) - } - - fn profile_args(&self) -> [&str; 2] { - ["--profile", self.profile()] - } - - fn export_syms(&self) -> Result<()> { - let archive = self.dist_archive(); - - let syscall_symbols = archive.syscall_symbols()?; - let explicit_exports = [ - "_start", - "__bss_start", - "runtime_entry", - // lwIP functions (C runtime) - "init_lwip", - "lwip_read", - "lwip_write", - // lwIP rtl8139 driver - "init_rtl8139_netif", - "irq_install_handler", - "virt_to_phys", - "eoi", - ] - .into_iter(); - - let symbols = explicit_exports.chain(syscall_symbols.iter().map(String::as_str)); - - archive.retain_symbols(symbols)?; - - Ok(()) - } - - fn profile(&self) -> &str { - self.profile - .as_deref() - .unwrap_or(if self.release { "release" } else { "dev" }) - } - - fn target_dir(&self) -> &Path { - self.target_dir - .as_deref() - .unwrap_or_else(|| Path::new("target")) - } - - fn out_dir(&self, triple: impl AsRef) -> PathBuf { - let mut out_dir = self.target_dir().to_path_buf(); - out_dir.push(triple); - out_dir.push(match self.profile() { - "dev" => "debug", - profile => profile, - }); - out_dir - } - - fn builtins_archive(&self) -> Archive { - let mut builtins_archive = self.out_dir(self.arch.hermit_triple()); - builtins_archive.push("libhermit_builtins.a"); - builtins_archive.into() - } - - fn build_archive(&self) -> Archive { - let mut built_archive = self.out_dir(self.arch.triple()); - built_archive.push("libhermit.a"); - built_archive.into() - } - - fn dist_archive(&self) -> Archive { - let mut dist_archive = self.out_dir(self.arch.name()); - dist_archive.push("libhermit.a"); - dist_archive.into() - } -} - -impl flags::Clippy { - fn run(self) -> Result<()> { - let sh = sh()?; - - for target in [Arch::X86_64, Arch::AArch64] { - let target_args = target.cargo_args(); - cmd!(sh, "cargo clippy {target_args...}").run()?; - cmd!(sh, "cargo clippy {target_args...}") - .arg("--no-default-features") - .run()?; - cmd!(sh, "cargo clippy {target_args...}") - .arg("--no-default-features") - .arg("--features=acpi,fsgsbase,pci,smp,vga") - .run()?; - // TODO: Enable clippy for newlib - // https://github.com/hermitcore/kernel/issues/470 - // cmd!(sh, "cargo clippy {target_args...}") - // .arg("--no-default-features") - // .arg("--features=acpi,fsgsbase,newlib,smp,vga") - // .run()?; - } - - cmd!(sh, "cargo clippy") - .arg("--manifest-path=hermit-builtins/Cargo.toml") - .arg("--target=x86_64-unknown-none") - .run()?; - - cmd!(sh, "cargo clippy --package xtask").run()?; - - Ok(()) - } -} - -fn sh() -> Result { +pub fn sh() -> Result { let sh = Shell::new()?; let project_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); sh.change_dir(project_root); From 104091df341cf34626475c70d4ff6cd4829c403e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 10:49:19 +0200 Subject: [PATCH 03/11] style(xtask): rename AArch64 enum variant to Aarch64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/arch.rs | 16 ++++++++-------- xtask/src/clippy.rs | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs index 01e07ed547..a44bf65e44 100644 --- a/xtask/src/arch.rs +++ b/xtask/src/arch.rs @@ -5,28 +5,28 @@ use anyhow::anyhow; #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum Arch { X86_64, - AArch64, + Aarch64, } impl Arch { pub fn name(&self) -> &'static str { match self { Self::X86_64 => "x86_64", - Self::AArch64 => "aarch64", + Self::Aarch64 => "aarch64", } } pub fn triple(&self) -> &'static str { match self { Self::X86_64 => "x86_64-unknown-none", - Self::AArch64 => "aarch64-unknown-none-softfloat", + Self::Aarch64 => "aarch64-unknown-none-softfloat", } } pub fn hermit_triple(&self) -> &'static str { match self { Arch::X86_64 => "x86_64-unknown-hermit", - Arch::AArch64 => "aarch64-unknown-hermit", + Arch::Aarch64 => "aarch64-unknown-hermit", } } @@ -37,7 +37,7 @@ impl Arch { "-Zbuild-std=core", "-Zbuild-std-features=compiler-builtins-mem", ], - Arch::AArch64 => &[ + Arch::Aarch64 => &[ "--target=aarch64-unknown-hermit", "-Zbuild-std=core", "-Zbuild-std-features=compiler-builtins-mem", @@ -48,7 +48,7 @@ impl Arch { pub fn cargo_args(&self) -> &'static [&'static str] { match self { Self::X86_64 => &["--target=x86_64-unknown-none"], - Self::AArch64 => &[ + Self::Aarch64 => &[ "--target=aarch64-unknown-none-softfloat", // We can't use prebuilt std for aarch64 because it is built with // relocation-model=static and we need relocation-model=pic @@ -61,7 +61,7 @@ impl Arch { pub fn rustflags(&self) -> &'static [&'static str] { match self { Self::X86_64 => &[], - Self::AArch64 => &["-Crelocation-model=pic"], + Self::Aarch64 => &["-Crelocation-model=pic"], } } } @@ -72,7 +72,7 @@ impl FromStr for Arch { fn from_str(s: &str) -> Result { match s { "x86_64" => Ok(Self::X86_64), - "aarch64" => Ok(Self::AArch64), + "aarch64" => Ok(Self::Aarch64), s => Err(anyhow!("Unsupported arch: {s}")), } } diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index 47a29f7a82..e791c47453 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -8,7 +8,7 @@ impl flags::Clippy { pub fn run(self) -> Result<()> { let sh = crate::sh()?; - for target in [Arch::X86_64, Arch::AArch64] { + for target in [Arch::X86_64, Arch::Aarch64] { let target_args = target.cargo_args(); cmd!(sh, "cargo clippy {target_args...}").run()?; cmd!(sh, "cargo clippy {target_args...}") From d04641fde195c14c6cf411733925edbe51429532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 10:58:33 +0200 Subject: [PATCH 04/11] feat(xtask): migrate to clap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- Cargo.lock | 129 ++++++++++++++++++++++++++++++++++++++------ xtask/Cargo.toml | 2 +- xtask/src/arch.rs | 22 +++----- xtask/src/build.rs | 41 +++++++++++++- xtask/src/clippy.rs | 8 ++- xtask/src/flags.rs | 81 ---------------------------- xtask/src/main.rs | 21 +++++--- 7 files changed, 179 insertions(+), 125 deletions(-) delete mode 100644 xtask/src/flags.rs diff --git a/Cargo.lock b/Cargo.lock index be49467e74..68703c77ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,6 +38,54 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d39c09fbfba977f4842e1f6bdc48b979112b64f8886993a34e051bc5f3c5c288" +[[package]] +name = "anstream" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.75" @@ -113,6 +161,52 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "clap_lex" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "critical-section" version = "1.1.1" @@ -272,6 +366,12 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-dtb" version = "0.1.1" @@ -819,6 +919,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.109" @@ -930,6 +1036,12 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "version_check" version = "0.9.4" @@ -1037,21 +1149,6 @@ dependencies = [ "volatile", ] -[[package]] -name = "xflags" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4554b580522d0ca238369c16b8f6ce34524d61dafe7244993754bbd05f2c2ea" -dependencies = [ - "xflags-macros", -] - -[[package]] -name = "xflags-macros" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58e7b3ca8977093aae6b87b6a7730216fc4c53a6530bab5c43a783cd810c1a8" - [[package]] name = "xshell" version = "0.2.5" @@ -1072,9 +1169,9 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "clap", "goblin", "llvm-tools", - "xflags", "xshell", ] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 5e5fd8d80d..cd97c7da52 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" [dependencies] anyhow = "1.0" +clap = { version = "4", features = ["derive"] } goblin = { version = "0.7", default-features = false, features = ["archive", "elf32", "elf64", "std"] } llvm-tools = "0.1" -xflags = "0.3" xshell = "0.2" diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs index a44bf65e44..8afa88b83a 100644 --- a/xtask/src/arch.rs +++ b/xtask/src/arch.rs @@ -1,10 +1,12 @@ -use std::str::FromStr; +use clap::ValueEnum; -use anyhow::anyhow; - -#[derive(Clone, Copy, PartialEq, Eq, Debug)] +/// Target architecture. +#[derive(ValueEnum, Clone, Copy, PartialEq, Eq, Debug)] +#[value(rename_all = "snake_case")] pub enum Arch { + /// x86-64 X86_64, + /// AArch64 Aarch64, } @@ -65,15 +67,3 @@ impl Arch { } } } - -impl FromStr for Arch { - type Err = anyhow::Error; - - fn from_str(s: &str) -> Result { - match s { - "x86_64" => Ok(Self::X86_64), - "aarch64" => Ok(Self::Aarch64), - s => Err(anyhow!("Unsupported arch: {s}")), - } - } -} diff --git a/xtask/src/build.rs b/xtask/src/build.rs index ba092e737d..01c9cdd709 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -3,12 +3,49 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; use anyhow::Result; +use clap::Args; use xshell::cmd; +use crate::arch::Arch; use crate::archive::Archive; -use crate::flags; -impl flags::Build { +/// Build the kernel. +#[derive(Args)] +pub struct Build { + /// Build for the architecture. + #[arg(value_enum, long)] + pub arch: Arch, + + /// Directory for all generated artifacts. + #[arg(long)] + pub target_dir: Option, + + /// Do not activate the `default` feature. + #[arg(long)] + pub no_default_features: bool, + + /// Space or comma separated list of features to activate. + #[arg(long)] + pub features: Vec, + + /// Build artifacts in release mode, with optimizations. + #[arg(short, long)] + pub release: bool, + + /// Build artifacts with the specified profile. + #[arg(long)] + pub profile: Option, + + /// Enable the `-Z instrument-mcount` flag. + #[arg(long)] + pub instrument_mcount: bool, + + /// Enable the `-Z randomize-layout` flag. + #[arg(long)] + pub randomize_layout: bool, +} + +impl Build { pub fn run(self) -> Result<()> { let sh = crate::sh()?; diff --git a/xtask/src/clippy.rs b/xtask/src/clippy.rs index e791c47453..831e4beb3f 100644 --- a/xtask/src/clippy.rs +++ b/xtask/src/clippy.rs @@ -1,10 +1,14 @@ use anyhow::Result; +use clap::Args; use xshell::cmd; use crate::arch::Arch; -use crate::flags; -impl flags::Clippy { +/// Run Clippy for all targets. +#[derive(Args)] +pub struct Clippy; + +impl Clippy { pub fn run(self) -> Result<()> { let sh = crate::sh()?; diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs deleted file mode 100644 index 78e54ceac9..0000000000 --- a/xtask/src/flags.rs +++ /dev/null @@ -1,81 +0,0 @@ -use std::path::PathBuf; - -use crate::arch::Arch; - -xflags::xflags! { - src "./src/flags.rs" - - /// Run custom build command. - cmd xtask { - /// Build the kernel. - cmd build - { - /// Build for the architecture. - required --arch arch: Arch - /// Directory for all generated artifacts. - optional --target-dir target_dir: PathBuf - /// Do not activate the `default` feature. - optional --no-default-features - /// Space or comma separated list of features to activate. - repeated --features features: String - /// Build artifacts in release mode, with optimizations. - optional -r, --release - /// Build artifacts with the specified profile. - optional --profile profile: String - /// Enable the `-Z instrument-mcount` flag. - optional --instrument-mcount - /// Enable the `-Z randomize-layout` flag. - optional --randomize-layout - } - - /// Run clippy for all targets. - cmd clippy {} - } -} - -// generated start -// The following code is generated by `xflags` macro. -// Run `env UPDATE_XFLAGS=1 cargo build` to regenerate. -#[derive(Debug)] -pub struct Xtask { - pub subcommand: XtaskCmd, -} - -#[derive(Debug)] -pub enum XtaskCmd { - Build(Build), - Clippy(Clippy), -} - -#[derive(Debug)] -pub struct Build { - pub arch: Arch, - pub target_dir: Option, - pub no_default_features: bool, - pub features: Vec, - pub release: bool, - pub profile: Option, - pub instrument_mcount: bool, - pub randomize_layout: bool, -} - -#[derive(Debug)] -pub struct Clippy; - -impl Xtask { - #[allow(dead_code)] - pub fn from_env_or_exit() -> Self { - Self::from_env_or_exit_() - } - - #[allow(dead_code)] - pub fn from_env() -> xflags::Result { - Self::from_env_() - } - - #[allow(dead_code)] - pub fn from_vec(args: Vec) -> xflags::Result { - Self::from_vec_(args) - } -} -// generated end diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 927de8faca..40eec9677e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -4,26 +4,33 @@ mod arch; mod archive; mod build; mod clippy; -mod flags; use std::path::Path; use anyhow::Result; +use clap::Parser; use xshell::Shell; -fn main() -> Result<()> { - flags::Xtask::from_env()?.run() +#[derive(Parser)] +enum Cli { + Build(build::Build), + Clippy(clippy::Clippy), } -impl flags::Xtask { +impl Cli { fn run(self) -> Result<()> { - match self.subcommand { - flags::XtaskCmd::Build(build) => build.run(), - flags::XtaskCmd::Clippy(clippy) => clippy.run(), + match self { + Self::Build(build) => build.run(), + Self::Clippy(clippy) => clippy.run(), } } } +fn main() -> Result<()> { + let cli = Cli::parse(); + cli.run() +} + pub fn sh() -> Result { let sh = Shell::new()?; let project_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); From 7890a9e89fc9f71de8accc4bca3643e0db5cf3e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 11:51:51 +0200 Subject: [PATCH 05/11] refactor(xtask): extract artifact args MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/artifact.rs | 67 ++++++++++++++++++++++++++++++++++++ xtask/src/build.rs | 79 +++++++------------------------------------ xtask/src/main.rs | 1 + 3 files changed, 80 insertions(+), 67 deletions(-) create mode 100644 xtask/src/artifact.rs diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs new file mode 100644 index 0000000000..2205329413 --- /dev/null +++ b/xtask/src/artifact.rs @@ -0,0 +1,67 @@ +use std::path::{Path, PathBuf}; + +use clap::Args; + +use crate::arch::Arch; +use crate::archive::Archive; + +#[derive(Args)] +pub struct Artifact { + /// Target architecture. + #[arg(value_enum, long)] + pub arch: Arch, + + /// Directory for all generated artifacts. + #[arg(long, id = "DIRECTORY")] + pub target_dir: Option, + + /// Build artifacts in release mode, with optimizations. + #[arg(short, long)] + pub release: bool, + + /// Build artifacts with the specified profile. + #[arg(long, id = "PROFILE-NAME")] + pub profile: Option, +} + +impl Artifact { + pub fn profile(&self) -> &str { + self.profile + .as_deref() + .unwrap_or(if self.release { "release" } else { "dev" }) + } + + pub fn target_dir(&self) -> &Path { + self.target_dir + .as_deref() + .unwrap_or_else(|| Path::new("target")) + } + + fn out_dir(&self, triple: impl AsRef) -> PathBuf { + let mut out_dir = self.target_dir().to_path_buf(); + out_dir.push(triple); + out_dir.push(match self.profile() { + "dev" => "debug", + profile => profile, + }); + out_dir + } + + pub fn builtins_archive(&self) -> Archive { + let mut builtins_archive = self.out_dir(self.arch.hermit_triple()); + builtins_archive.push("libhermit_builtins.a"); + builtins_archive.into() + } + + pub fn build_archive(&self) -> Archive { + let mut built_archive = self.out_dir(self.arch.triple()); + built_archive.push("libhermit.a"); + built_archive.into() + } + + pub fn dist_archive(&self) -> Archive { + let mut dist_archive = self.out_dir(self.arch.name()); + dist_archive.push("libhermit.a"); + dist_archive.into() + } +} diff --git a/xtask/src/build.rs b/xtask/src/build.rs index 01c9cdd709..6383467644 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -1,24 +1,17 @@ use std::env::{self, VarError}; use std::ffi::OsStr; -use std::path::{Path, PathBuf}; use anyhow::Result; use clap::Args; use xshell::cmd; -use crate::arch::Arch; -use crate::archive::Archive; +use crate::artifact::Artifact; /// Build the kernel. #[derive(Args)] pub struct Build { - /// Build for the architecture. - #[arg(value_enum, long)] - pub arch: Arch, - - /// Directory for all generated artifacts. - #[arg(long)] - pub target_dir: Option, + #[command(flatten)] + artifact: Artifact, /// Do not activate the `default` feature. #[arg(long)] @@ -28,14 +21,6 @@ pub struct Build { #[arg(long)] pub features: Vec, - /// Build artifacts in release mode, with optimizations. - #[arg(short, long)] - pub release: bool, - - /// Build artifacts with the specified profile. - #[arg(long)] - pub profile: Option, - /// Enable the `-Z instrument-mcount` flag. #[arg(long)] pub instrument_mcount: bool, @@ -52,15 +37,15 @@ impl Build { eprintln!("Building kernel"); cmd!(sh, "cargo build") .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) - .args(self.arch.cargo_args()) + .args(self.artifact.arch.cargo_args()) .args(self.target_dir_args()) .args(self.no_default_features_args()) .args(self.features_args()) .args(self.profile_args()) .run()?; - let build_archive = self.build_archive(); - let dist_archive = self.dist_archive(); + let build_archive = self.artifact.build_archive(); + let dist_archive = self.artifact.dist_archive(); eprintln!( "Copying {} to {}", build_archive.as_ref().display(), @@ -75,13 +60,13 @@ impl Build { eprintln!("Building hermit-builtins"); cmd!(sh, "cargo build") .arg("--manifest-path=hermit-builtins/Cargo.toml") - .args(self.arch.builtins_cargo_args()) + .args(self.artifact.arch.builtins_cargo_args()) .args(self.target_dir_args()) .args(self.profile_args()) .run()?; eprintln!("Exporting hermit-builtins symbols"); - let builtins = self.builtins_archive(); + let builtins = self.artifact.builtins_archive(); let builtin_symbols = sh.read_file("hermit-builtins/exports")?; builtins.retain_symbols(builtin_symbols.lines())?; @@ -117,13 +102,13 @@ impl Build { rustflags.push("-Zrandomize-layout") } - rustflags.extend(self.arch.rustflags()); + rustflags.extend(self.artifact.arch.rustflags()); Ok(rustflags.join("\x1f")) } fn target_dir_args(&self) -> [&OsStr; 2] { - ["--target-dir".as_ref(), self.target_dir().as_ref()] + ["--target-dir".as_ref(), self.artifact.target_dir().as_ref()] } fn no_default_features_args(&self) -> &[&str] { @@ -141,11 +126,11 @@ impl Build { } fn profile_args(&self) -> [&str; 2] { - ["--profile", self.profile()] + ["--profile", self.artifact.profile()] } fn export_syms(&self) -> Result<()> { - let archive = self.dist_archive(); + let archive = self.artifact.dist_archive(); let syscall_symbols = archive.syscall_symbols()?; let explicit_exports = [ @@ -170,44 +155,4 @@ impl Build { Ok(()) } - - fn profile(&self) -> &str { - self.profile - .as_deref() - .unwrap_or(if self.release { "release" } else { "dev" }) - } - - fn target_dir(&self) -> &Path { - self.target_dir - .as_deref() - .unwrap_or_else(|| Path::new("target")) - } - - fn out_dir(&self, triple: impl AsRef) -> PathBuf { - let mut out_dir = self.target_dir().to_path_buf(); - out_dir.push(triple); - out_dir.push(match self.profile() { - "dev" => "debug", - profile => profile, - }); - out_dir - } - - fn builtins_archive(&self) -> Archive { - let mut builtins_archive = self.out_dir(self.arch.hermit_triple()); - builtins_archive.push("libhermit_builtins.a"); - builtins_archive.into() - } - - fn build_archive(&self) -> Archive { - let mut built_archive = self.out_dir(self.arch.triple()); - built_archive.push("libhermit.a"); - built_archive.into() - } - - fn dist_archive(&self) -> Archive { - let mut dist_archive = self.out_dir(self.arch.name()); - dist_archive.push("libhermit.a"); - dist_archive.into() - } } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 40eec9677e..7c2ce1ce0c 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,6 +2,7 @@ mod arch; mod archive; +mod artifact; mod build; mod clippy; From ccc3e1aab9af7d2abd43dec73b9b3ba328a43aa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 11:59:53 +0200 Subject: [PATCH 06/11] feat(xtask): always build hermit-builtins as release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/artifact.rs | 4 +++- xtask/src/build.rs | 9 ++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs index 2205329413..3364bea983 100644 --- a/xtask/src/artifact.rs +++ b/xtask/src/artifact.rs @@ -48,7 +48,9 @@ impl Artifact { } pub fn builtins_archive(&self) -> Archive { - let mut builtins_archive = self.out_dir(self.arch.hermit_triple()); + let mut builtins_archive = self.target_dir().to_path_buf(); + builtins_archive.push(self.arch.hermit_triple()); + builtins_archive.push("release"); builtins_archive.push("libhermit_builtins.a"); builtins_archive.into() } diff --git a/xtask/src/build.rs b/xtask/src/build.rs index 6383467644..57bbe455dc 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -37,11 +37,11 @@ impl Build { eprintln!("Building kernel"); cmd!(sh, "cargo build") .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) + .args(&["--profile", self.artifact.profile()]) .args(self.artifact.arch.cargo_args()) .args(self.target_dir_args()) .args(self.no_default_features_args()) .args(self.features_args()) - .args(self.profile_args()) .run()?; let build_archive = self.artifact.build_archive(); @@ -58,11 +58,10 @@ impl Build { self.export_syms()?; eprintln!("Building hermit-builtins"); - cmd!(sh, "cargo build") + cmd!(sh, "cargo build --release") .arg("--manifest-path=hermit-builtins/Cargo.toml") .args(self.artifact.arch.builtins_cargo_args()) .args(self.target_dir_args()) - .args(self.profile_args()) .run()?; eprintln!("Exporting hermit-builtins symbols"); @@ -125,10 +124,6 @@ impl Build { .flat_map(|feature| ["--features", feature.as_str()]) } - fn profile_args(&self) -> [&str; 2] { - ["--profile", self.artifact.profile()] - } - fn export_syms(&self) -> Result<()> { let archive = self.artifact.dist_archive(); From 4cb7302dbf5873b3f885f19fdc80a42c5edd7d03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 13:44:53 +0200 Subject: [PATCH 07/11] refactor(xtask): extract profile_path_component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/artifact.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs index 3364bea983..07d509a537 100644 --- a/xtask/src/artifact.rs +++ b/xtask/src/artifact.rs @@ -31,6 +31,13 @@ impl Artifact { .unwrap_or(if self.release { "release" } else { "dev" }) } + pub fn profile_path_component(&self) -> &str { + match self.profile() { + "dev" => "debug", + profile => profile, + } + } + pub fn target_dir(&self) -> &Path { self.target_dir .as_deref() @@ -40,10 +47,7 @@ impl Artifact { fn out_dir(&self, triple: impl AsRef) -> PathBuf { let mut out_dir = self.target_dir().to_path_buf(); out_dir.push(triple); - out_dir.push(match self.profile() { - "dev" => "debug", - profile => profile, - }); + out_dir.push(self.profile_path_component()); out_dir } From c073def28fe75e74f455c5b05806c71adafcd417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 13:50:32 +0200 Subject: [PATCH 08/11] refactor(xtask): inline out_dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/artifact.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs index 07d509a537..2632b1ee03 100644 --- a/xtask/src/artifact.rs +++ b/xtask/src/artifact.rs @@ -44,13 +44,6 @@ impl Artifact { .unwrap_or_else(|| Path::new("target")) } - fn out_dir(&self, triple: impl AsRef) -> PathBuf { - let mut out_dir = self.target_dir().to_path_buf(); - out_dir.push(triple); - out_dir.push(self.profile_path_component()); - out_dir - } - pub fn builtins_archive(&self) -> Archive { let mut builtins_archive = self.target_dir().to_path_buf(); builtins_archive.push(self.arch.hermit_triple()); @@ -60,13 +53,17 @@ impl Artifact { } pub fn build_archive(&self) -> Archive { - let mut built_archive = self.out_dir(self.arch.triple()); + let mut built_archive = self.target_dir().to_path_buf(); + built_archive.push(self.arch.triple()); + built_archive.push(self.profile_path_component()); built_archive.push("libhermit.a"); built_archive.into() } pub fn dist_archive(&self) -> Archive { - let mut dist_archive = self.out_dir(self.arch.name()); + let mut dist_archive = self.target_dir().to_path_buf(); + dist_archive.push(self.arch.name()); + dist_archive.push(self.profile_path_component()); dist_archive.push("libhermit.a"); dist_archive.into() } From 1c511d72ee41ca80330aaa1bc019f37981949eb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Fri, 1 Sep 2023 13:56:42 +0200 Subject: [PATCH 09/11] refactor(xtask): collect artifact paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/artifact.rs | 42 +++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs index 2632b1ee03..6b0b22935c 100644 --- a/xtask/src/artifact.rs +++ b/xtask/src/artifact.rs @@ -45,26 +45,38 @@ impl Artifact { } pub fn builtins_archive(&self) -> Archive { - let mut builtins_archive = self.target_dir().to_path_buf(); - builtins_archive.push(self.arch.hermit_triple()); - builtins_archive.push("release"); - builtins_archive.push("libhermit_builtins.a"); - builtins_archive.into() + [ + self.target_dir(), + self.arch.hermit_triple().as_ref(), + "release".as_ref(), + "libhermit_builtins.a".as_ref(), + ] + .iter() + .collect::() + .into() } pub fn build_archive(&self) -> Archive { - let mut built_archive = self.target_dir().to_path_buf(); - built_archive.push(self.arch.triple()); - built_archive.push(self.profile_path_component()); - built_archive.push("libhermit.a"); - built_archive.into() + [ + self.target_dir(), + self.arch.triple().as_ref(), + self.profile_path_component().as_ref(), + "libhermit.a".as_ref(), + ] + .iter() + .collect::() + .into() } pub fn dist_archive(&self) -> Archive { - let mut dist_archive = self.target_dir().to_path_buf(); - dist_archive.push(self.arch.name()); - dist_archive.push(self.profile_path_component()); - dist_archive.push("libhermit.a"); - dist_archive.into() + [ + self.target_dir(), + self.arch.name().as_ref(), + self.profile_path_component().as_ref(), + "libhermit.a".as_ref(), + ] + .iter() + .collect::() + .into() } } From 7fbc7eb86bff152f54d8ba13b5859447774e2bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 5 Sep 2023 10:55:36 +0200 Subject: [PATCH 10/11] feat(xtask): extract CargoBuild This improves generated commands. --- xtask/src/artifact.rs | 1 + xtask/src/build.rs | 52 ++++++---------------------- xtask/src/cargo_build.rs | 73 ++++++++++++++++++++++++++++++++++++++++ xtask/src/main.rs | 1 + 4 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 xtask/src/cargo_build.rs diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs index 6b0b22935c..ddb9981fc2 100644 --- a/xtask/src/artifact.rs +++ b/xtask/src/artifact.rs @@ -46,6 +46,7 @@ impl Artifact { pub fn builtins_archive(&self) -> Archive { [ + "hermit-builtins".as_ref(), self.target_dir(), self.arch.hermit_triple().as_ref(), "release".as_ref(), diff --git a/xtask/src/build.rs b/xtask/src/build.rs index 57bbe455dc..534c6c3290 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -1,25 +1,16 @@ use std::env::{self, VarError}; -use std::ffi::OsStr; use anyhow::Result; use clap::Args; use xshell::cmd; -use crate::artifact::Artifact; +use crate::cargo_build::{CargoBuild, CmdExt}; /// Build the kernel. #[derive(Args)] pub struct Build { #[command(flatten)] - artifact: Artifact, - - /// Do not activate the `default` feature. - #[arg(long)] - pub no_default_features: bool, - - /// Space or comma separated list of features to activate. - #[arg(long)] - pub features: Vec, + cargo_build: CargoBuild, /// Enable the `-Z instrument-mcount` flag. #[arg(long)] @@ -37,15 +28,12 @@ impl Build { eprintln!("Building kernel"); cmd!(sh, "cargo build") .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) - .args(&["--profile", self.artifact.profile()]) - .args(self.artifact.arch.cargo_args()) - .args(self.target_dir_args()) - .args(self.no_default_features_args()) - .args(self.features_args()) + .args(self.cargo_build.artifact.arch.cargo_args()) + .cargo_build_args(&self.cargo_build) .run()?; - let build_archive = self.artifact.build_archive(); - let dist_archive = self.artifact.dist_archive(); + let build_archive = self.cargo_build.artifact.build_archive(); + let dist_archive = self.cargo_build.artifact.dist_archive(); eprintln!( "Copying {} to {}", build_archive.as_ref().display(), @@ -60,12 +48,12 @@ impl Build { eprintln!("Building hermit-builtins"); cmd!(sh, "cargo build --release") .arg("--manifest-path=hermit-builtins/Cargo.toml") - .args(self.artifact.arch.builtins_cargo_args()) - .args(self.target_dir_args()) + .args(self.cargo_build.artifact.arch.builtins_cargo_args()) + .target_dir_args(&self.cargo_build) .run()?; eprintln!("Exporting hermit-builtins symbols"); - let builtins = self.artifact.builtins_archive(); + let builtins = self.cargo_build.artifact.builtins_archive(); let builtin_symbols = sh.read_file("hermit-builtins/exports")?; builtins.retain_symbols(builtin_symbols.lines())?; @@ -101,31 +89,13 @@ impl Build { rustflags.push("-Zrandomize-layout") } - rustflags.extend(self.artifact.arch.rustflags()); + rustflags.extend(self.cargo_build.artifact.arch.rustflags()); Ok(rustflags.join("\x1f")) } - fn target_dir_args(&self) -> [&OsStr; 2] { - ["--target-dir".as_ref(), self.artifact.target_dir().as_ref()] - } - - fn no_default_features_args(&self) -> &[&str] { - if self.no_default_features { - &["--no-default-features"] - } else { - &[] - } - } - - fn features_args(&self) -> impl Iterator { - self.features - .iter() - .flat_map(|feature| ["--features", feature.as_str()]) - } - fn export_syms(&self) -> Result<()> { - let archive = self.artifact.dist_archive(); + let archive = self.cargo_build.artifact.dist_archive(); let syscall_symbols = archive.syscall_symbols()?; let explicit_exports = [ diff --git a/xtask/src/cargo_build.rs b/xtask/src/cargo_build.rs new file mode 100644 index 0000000000..0570a51458 --- /dev/null +++ b/xtask/src/cargo_build.rs @@ -0,0 +1,73 @@ +use std::ffi::OsStr; + +use clap::Args; +use xshell::Cmd; + +use crate::artifact::Artifact; + +#[derive(Args)] +pub struct CargoBuild { + #[command(flatten)] + pub artifact: Artifact, + + /// Do not activate the `default` feature. + #[arg(long)] + no_default_features: bool, + + /// Space or comma separated list of features to activate. + #[arg(long)] + features: Vec, +} + +pub trait CmdExt { + fn cargo_build_args(self, cargo_build: &CargoBuild) -> Self; + fn target_dir_args(self, cargo_build: &CargoBuild) -> Self; +} + +impl CmdExt for Cmd<'_> { + fn cargo_build_args(self, cargo_build: &CargoBuild) -> Self { + let cmd = self + .target_dir_args(cargo_build) + .args(cargo_build.no_default_features_args()) + .args(cargo_build.features_args()) + .args(cargo_build.release_args()); + + if let Some(profile) = &cargo_build.artifact.profile { + cmd.args(&["--profile", profile]) + } else { + cmd + } + } + + fn target_dir_args(self, cargo_build: &CargoBuild) -> Self { + if let Some(target_dir) = &cargo_build.artifact.target_dir { + self.args::<&[&OsStr]>(&["--target-dir".as_ref(), target_dir.as_ref()]) + } else { + self + } + } +} + +impl CargoBuild { + fn release_args(&self) -> &'static [&'static str] { + if self.artifact.release { + &["--release"] + } else { + &[] + } + } + + fn no_default_features_args(&self) -> &'static [&'static str] { + if self.no_default_features { + &["--no-default-features"] + } else { + &[] + } + } + + fn features_args(&self) -> impl Iterator { + self.features + .iter() + .flat_map(|feature| ["--features", feature.as_str()]) + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 7c2ce1ce0c..0a089c9a4c 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -4,6 +4,7 @@ mod arch; mod archive; mod artifact; mod build; +mod cargo_build; mod clippy; use std::path::Path; From f91f321e5331d03e1d4c479005ec40777c86efea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Mon, 4 Sep 2023 13:23:34 +0200 Subject: [PATCH 11/11] feat(xtask): add ci subcommand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- .github/workflows/ci.yml | 239 +++---------- Cargo.lock | 432 +++++++++++++++++++++++- fc-config.json | 13 - xtask/Cargo.toml | 3 + xtask/src/arch.rs | 13 + xtask/src/artifact.rs | 12 + xtask/src/ci/build.rs | 45 +++ xtask/src/ci/firecracker.rs | 42 +++ xtask/src/ci/firecracker_vm_config.json | 13 + xtask/src/ci/mod.rs | 31 ++ xtask/src/ci/qemu.rs | 286 ++++++++++++++++ xtask/src/ci/uhyve.rs | 41 +++ xtask/src/main.rs | 4 + 13 files changed, 969 insertions(+), 205 deletions(-) delete mode 100644 fc-config.json create mode 100644 xtask/src/ci/build.rs create mode 100644 xtask/src/ci/firecracker.rs create mode 100644 xtask/src/ci/firecracker_vm_config.json create mode 100644 xtask/src/ci/mod.rs create mode 100644 xtask/src/ci/qemu.rs create mode 100644 xtask/src/ci/uhyve.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4af80c298d..4a8033a2c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,6 +117,9 @@ jobs: run-x86_64: name: Run Hermit for Rust (x86_64) runs-on: ubuntu-latest + defaults: + run: + working-directory: kernel steps: - name: Checkout hermit-rs uses: actions/checkout@v4 @@ -125,6 +128,7 @@ jobs: submodules: true - name: Remove hermit-kernel submodule run: git rm -r kernel + working-directory: . - name: Checkout hermit-kernel uses: actions/checkout@v4 with: @@ -146,131 +150,31 @@ jobs: file: rusty-loader-x86_64 - name: Install virtiofsd run: cargo install virtiofsd - - name: Build dev profile - run: cargo build -Zbuild-std=std,panic_abort --package rusty_demo --target x86_64-unknown-hermit --features pci-ids - - name: Test dev profile - run: | - mkdir -p foo - virtiofsd --socket-path=./vhostqemu --shared-dir ./foo --announce-submounts --sandbox none --seccomp none --inode-file-handles=never & - qemu-system-x86_64 -display none -smp 1 -m 1G -serial stdio \ - -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ - -kernel rusty-loader-x86_64 \ - -chardev socket,id=char0,path=./vhostqemu \ - -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=root \ - -object memory-backend-file,id=mem,size=1G,mem-path=/dev/shm,share=on -numa node,memdev=mem \ - -initrd target/x86_64-unknown-hermit/debug/rusty_demo - - name: Build release profile - run: cargo build -Zbuild-std=std,panic_abort --package rusty_demo --target x86_64-unknown-hermit --release --features pci-ids - - name: Test release profile - run: | - mkdir -p foo - virtiofsd --socket-path=./vhostqemu --shared-dir ./foo --announce-submounts --sandbox none --seccomp none --inode-file-handles=never & - qemu-system-x86_64 -display none -smp 1 -m 1G -serial stdio \ - -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ - -kernel rusty-loader-x86_64 \ - -chardev socket,id=char0,path=./vhostqemu \ - -device vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=root \ - -object memory-backend-file,id=mem,size=1G,mem-path=/dev/shm,share=on -numa node,memdev=mem \ - -initrd target/x86_64-unknown-hermit/release/rusty_demo - - name: Build httpd with DHCP support (debug, rtl8139) - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --package httpd --features ci,dhcpv4,rtl8139 - - name: Test httpd with DHCP support (debug, rtl8139) - run: | - qemu-system-x86_64 -smp 1 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 -display none -m 128M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/debug/httpd \ - -netdev user,id=u1,hostfwd=tcp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device rtl8139,netdev=u1 & - sleep 5 - curl http://127.0.0.1:9975/help - sleep 1 - - name: Build httpd with DHCP support (debug, virtio-net) - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --package httpd --features ci,dhcpv4 - - name: Test httpd with DHCP support (debug, virtio-net) - run: | - qemu-system-x86_64 -smp 1 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 -display none -m 512M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/debug/httpd \ - -netdev user,id=u1,hostfwd=tcp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device virtio-net-pci,netdev=u1,disable-legacy=on & - sleep 5 - curl http://127.0.0.1:9975/help - sleep 1 - - name: Build httpd with DHCP support (release, rtl8139) - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --package httpd --release --features ci,dhcpv4,rtl8139 - - name: Test httpd with DHCP support (release, rtl8139) - run: | - qemu-system-x86_64 -smp 1 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 -display none -m 128M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/release/httpd \ - -netdev user,id=u1,hostfwd=tcp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device rtl8139,netdev=u1 & - sleep 5 - curl http://127.0.0.1:9975/help - sleep 1 - - name: Build httpd with DHCP support (release, virtio-net) - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --package httpd --release --features ci,dhcpv4 - - name: Test httpd with DHCP support (release, virtio-net) - run: | - qemu-system-x86_64 -smp 1 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 -display none -m 512M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/release/httpd \ - -netdev user,id=u1,hostfwd=tcp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device virtio-net-pci,netdev=u1,disable-legacy=on & - sleep 5 - curl http://127.0.0.1:9975/help - sleep 1 - - name: Build minimal profile - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --no-default-features --release --package hello_world - - name: Test minimal profile - run: | - FREQ=`grep 'cpu MHz' /proc/cpuinfo | head -1 | awk -F: '{print $2}' | awk '{printf("%d\n",$1 + 0.5)}'` - echo "FREQ = $FREQ" - qemu-system-x86_64 \ - -M microvm,x-option-roms=off,pit=off,pic=off,rtc=on \ - -global virtio-mmio.force-legacy=on -nodefaults -no-user-config \ - -display none -smp 1 -m 64M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/release/hello_world \ - -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ - -append "-freq $FREQ" \ - || qemu_status=$? - test $qemu_status -eq 3 - - name: Build udp tests (debug, virtio-net) - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --package testudp --features udp,dhcpv4 - - name: Test udp profile - run: | - qemu-system-x86_64 -smp 1 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 -display none -m 512M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/debug/testudp \ - -netdev user,id=u1,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device virtio-net-pci,netdev=u1,disable-legacy=on & - sleep 5 - echo "exit" | socat -d -d -d - UDP:localhost:9975 - - name: Build udp tests (release, virtio-net) - run: cargo build -Zbuild-std=std,panic_abort --release --target x86_64-unknown-hermit --package testudp --features udp,dhcpv4 - - name: Test udp profile - run: | - qemu-system-x86_64 -smp 1 -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -device isa-debug-exit,iobase=0xf4,iosize=0x04 -display none -m 512M -serial stdio \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/release/testudp \ - -netdev user,id=u1,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device virtio-net-pci,netdev=u1,disable-legacy=on & - sleep 5 - echo "exit" | socat -d -d -d - UDP:localhost:9975 + - name: rusty_demo with pci-ids and virtiofsd + run: cargo xtask ci qemu --arch x86_64 --package rusty_demo --features pci-ids --virtiofsd + - name: rusty_demo with pci-ids and virtiofsd (release) + run: cargo xtask ci qemu --arch x86_64 --package rusty_demo --features pci-ids --virtiofsd --release + - name: httpd with virtio-net-pci and dhcpv4 + run: cargo xtask ci qemu --arch x86_64 --package httpd --features ci,dhcpv4 --netdev virtio-net-pci + - name: httpd with virtio-net-pci and dhcpv4 (release) + run: cargo xtask ci qemu --arch x86_64 --package httpd --features ci,dhcpv4 --netdev virtio-net-pci --release + - name: httpd with rtl8139 and dhcpv4 + run: cargo xtask ci qemu --arch x86_64 --package httpd --features ci,dhcpv4,rtl8139 --netdev rtl8139 + - name: httpd with rtl8139 and dhcpv4 (release) + run: cargo xtask ci qemu --arch x86_64 --package httpd --features ci,dhcpv4,rtl8139 --netdev rtl8139 --release + - name: hello_world on microVM (release) + run: cargo xtask ci qemu --arch x86_64 --package hello_world --no-default-features --microvm --release + - name: testudp with virtio-net-pci and dhcpv4 + run: cargo xtask ci qemu --arch x86_64 --package testudp --features udp,dhcpv4 --netdev virtio-net-pci + - name: testudp with virtio-net-pci and dhcpv4 (release) + run: cargo xtask ci qemu --arch x86_64 --package testudp --features udp,dhcpv4 --netdev virtio-net-pci --release run-aarch64: name: Run Hermit for Rust (aarch64) runs-on: ubuntu-latest + defaults: + run: + working-directory: kernel steps: - name: Checkout hermit-rs uses: actions/checkout@v4 @@ -279,6 +183,7 @@ jobs: submodules: true - name: Remove hermit-kernel submodule run: git rm -r kernel + working-directory: . - name: Checkout hermit-kernel uses: actions/checkout@v4 with: @@ -299,52 +204,21 @@ jobs: . kernel kernel/hermit-builtins - - name: Build dev profile - run: cargo build -Zbuild-std=std,panic_abort --target aarch64-unknown-hermit --package rusty_demo --features pci-ids - - name: Test dev kernel - run: | - qemu-system-aarch64 -semihosting \ - -kernel rusty-loader-aarch64 -machine virt,gic-version=3 \ - -m 512M -cpu cortex-a72 -smp 1 -display none -serial stdio \ - -device guest-loader,addr=0x48000000,initrd=target/aarch64-unknown-hermit/debug/rusty_demo - - name: Build release profile - run: cargo build -Zbuild-std=std,panic_abort --target aarch64-unknown-hermit --package rusty_demo --release --features pci-ids - - name: Test release kernel - run: | - qemu-system-aarch64 -semihosting \ - -kernel rusty-loader-aarch64 -machine virt,gic-version=3 \ - -m 512M -cpu cortex-a72 -smp 1 -display none -serial stdio \ - -device guest-loader,addr=0x48000000,initrd=target/aarch64-unknown-hermit/release/rusty_demo - - name: Build httpd with DHCP support (debug) - run: cargo build -Zbuild-std=std,panic_abort --target aarch64-unknown-hermit --package httpd --features ci,dhcpv4 - - name: Test httpd with DHCP support (debug, virtio-net) - run: | - qemu-system-aarch64 -semihosting \ - -kernel rusty-loader-aarch64 -machine virt,gic-version=3 \ - -m 512M -cpu cortex-a72 -smp 1 -display none -serial stdio \ - -device guest-loader,addr=0x48000000,initrd=target/aarch64-unknown-hermit/debug/httpd \ - -netdev user,id=u1,hostfwd=tcp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device virtio-net-pci,netdev=u1,disable-legacy=on & - sleep 5 - curl http://127.0.0.1:9975/help - sleep 1 - - name: Build httpd with DHCP support (release) - run: cargo build -Zbuild-std=std,panic_abort --target aarch64-unknown-hermit --package httpd --release --features ci,dhcpv4 - - name: Test httpd with DHCP support (release, virtio-net) - run: | - qemu-system-aarch64 -semihosting \ - -kernel rusty-loader-aarch64 -machine virt,gic-version=3 \ - -m 512M -cpu cortex-a72 -smp 1 -display none -serial stdio \ - -device guest-loader,addr=0x48000000,initrd=target/aarch64-unknown-hermit/release/httpd \ - -netdev user,id=u1,hostfwd=tcp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9 \ - -device virtio-net-pci,netdev=u1,disable-legacy=on & - sleep 5 - curl http://127.0.0.1:9975/help - sleep 1 + - name: rusty_demo with pci-ids + run: cargo xtask ci qemu --arch aarch64 --package rusty_demo --features pci-ids + - name: rusty_demo with pci-ids (release) + run: cargo xtask ci qemu --arch aarch64 --package rusty_demo --features pci-ids --release + - name: httpd with dhcpv4 + run: cargo xtask ci qemu --arch aarch64 --package httpd --features ci,dhcpv4 --netdev virtio-net-pci + - name: httpd with dhcpv4 (release) + run: cargo xtask ci qemu --arch aarch64 --package httpd --features ci,dhcpv4 --netdev virtio-net-pci --release run-x86_64-kvm: name: Run Hermit for Rust (x86_64 + kvm) runs-on: [self-hosted] + defaults: + run: + working-directory: kernel steps: - name: Checkout hermit-rs uses: actions/checkout@v4 @@ -353,6 +227,7 @@ jobs: submodules: true - name: Remove hermit-kernel submodule run: git rm -r kernel + working-directory: . - name: Checkout hermit-kernel uses: actions/checkout@v4 with: @@ -392,31 +267,13 @@ jobs: - run: cargo install uhyve - uses: mkroening/rust-toolchain-toml@main - uses: Swatinem/rust-cache@v2 - - name: Build dev profile - run: cargo build -Zbuild-std=std,panic_abort --package rusty_demo --target x86_64-unknown-hermit - - name: Test debug version (Uhyve) - run: uhyve --verbose -c 1 target/x86_64-unknown-hermit/debug/rusty_demo - env: - RUST_LOG: debug - - name: Test debug profile (Qemu) - run: | - qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio \ - -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/debug/rusty_demo - - name: Build release profile - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --package rusty_demo --release - - name: Test release version (Uhyve) - run: uhyve --verbose -c 1 target/x86_64-unknown-hermit/release/rusty_demo - env: - RUST_LOG: debug - - name: Test release profile (Qemu) - run: | - qemu-system-x86_64 -display none -smp 1 -m 128M -serial stdio \ - -enable-kvm -cpu qemu64,apic,fsgsbase,rdtscp,xsave,xsaveopt,fxsr,rdrand \ - -kernel rusty-loader-x86_64 \ - -initrd target/x86_64-unknown-hermit/release/rusty_demo - - name: Build minimal profile (debug) - run: cargo build -Zbuild-std=std,panic_abort --target x86_64-unknown-hermit --no-default-features --package hello_world - - name: Test debug profile (Firecracker) - run: firecracker --no-api --config-file ./kernel/fc-config.json + - name: rusty_demo on Uhyve + run: cargo xtask ci uhyve --arch x86_64 --package rusty_demo + - name: rusty_demo on Uhyve (release) + run: cargo xtask ci uhyve --arch x86_64 --package rusty_demo --release + - name: rusty_demo on QEMU (with KVM) + run: cargo xtask ci qemu --arch x86_64 --package rusty_demo --accel + - name: rusty_demo on QEMU (with KVM, release) + run: cargo xtask ci qemu --arch x86_64 --package rusty_demo --accel --release + - name: hello_world on Firecracker + run: cargo xtask ci firecracker --arch x86_64 --package hello_world --no-default-features diff --git a/Cargo.lock b/Cargo.lock index 68703c77ae..55aa889711 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,6 +21,12 @@ dependencies = [ "tock-registers", ] +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.8.3" @@ -116,6 +122,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" + [[package]] name = "bit_field" version = "0.10.2" @@ -134,6 +146,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +[[package]] +name = "bumpalo" +version = "3.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" + [[package]] name = "byteorder" version = "1.4.3" @@ -207,12 +225,61 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" +[[package]] +name = "crossbeam-channel" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-utils" version = "0.8.16" @@ -266,6 +333,12 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +[[package]] +name = "either" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" + [[package]] name = "embedded-hal" version = "0.2.7" @@ -309,6 +382,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +[[package]] +name = "flate2" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "float-cmp" version = "0.9.0" @@ -318,6 +401,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + [[package]] name = "generic_once_cell" version = "0.1.1" @@ -362,7 +454,7 @@ dependencies = [ "atomic-polyfill", "hash32", "rustc_version", - "spin", + "spin 0.9.8", "stable_deref_trait", ] @@ -372,6 +464,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "hermit-abi" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" + [[package]] name = "hermit-dtb" version = "0.1.1" @@ -445,6 +543,16 @@ dependencies = [ "x86_64", ] +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "include-transformed" version = "0.2.2" @@ -457,6 +565,15 @@ dependencies = [ "tempfile", ] +[[package]] +name = "js-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.147" @@ -503,12 +620,30 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "multiboot" version = "0.8.0" @@ -543,6 +678,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + [[package]] name = "num" version = "0.4.1" @@ -617,6 +761,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -651,6 +805,12 @@ dependencies = [ "bit_field", ] +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + [[package]] name = "pflock" version = "0.2.0" @@ -792,6 +952,28 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "rayon" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.3.5" @@ -801,6 +983,21 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "riscv" version = "0.10.1" @@ -834,6 +1031,38 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustls" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.4", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.14" @@ -866,6 +1095,16 @@ dependencies = [ "syn 2.0.28", ] +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "semver" version = "1.0.18" @@ -904,6 +1143,12 @@ dependencies = [ "managed", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.8" @@ -946,6 +1191,21 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sysinfo" +version = "0.29.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8d0e9cc2273cc8d31377bdd638d72e3ac3e5607b18621062b169d02787f1bab" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "winapi", +] + [[package]] name = "take-static" version = "0.1.2" @@ -1013,6 +1273,21 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tock-registers" version = "0.8.1" @@ -1030,12 +1305,60 @@ dependencies = [ "x86", ] +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + [[package]] name = "unicode-ident" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "rustls-webpki 0.100.2", + "url", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "utf8parse" version = "0.2.1" @@ -1060,6 +1383,110 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "442887c63f2c839b346c192d047a7c87e73d0689c9157b00b53dcc27dd5ea793" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.28", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" +dependencies = [ + "rustls-webpki 0.100.2", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-sys" version = "0.48.0" @@ -1172,6 +1599,9 @@ dependencies = [ "clap", "goblin", "llvm-tools", + "sysinfo", + "ureq", + "wait-timeout", "xshell", ] diff --git a/fc-config.json b/fc-config.json deleted file mode 100644 index 71f9f708ed..0000000000 --- a/fc-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "boot-source": { - "kernel_image_path": "./rusty-loader-x86_64-fc", - "initrd_path": "target/x86_64-unknown-hermit/debug/hello_world", - "boot_args": "" - }, - "drives": [], - "machine-config": { - "vcpu_count": 1, - "mem_size_mib": 256, - "smt": false - } -} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index cd97c7da52..26dfed366a 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -8,4 +8,7 @@ anyhow = "1.0" clap = { version = "4", features = ["derive"] } goblin = { version = "0.7", default-features = false, features = ["archive", "elf32", "elf64", "std"] } llvm-tools = "0.1" +sysinfo = "0.29" +ureq = "2" +wait-timeout = "0.2" xshell = "0.2" diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs index 8afa88b83a..77f841ab74 100644 --- a/xtask/src/arch.rs +++ b/xtask/src/arch.rs @@ -60,6 +60,19 @@ impl Arch { } } + pub fn ci_cargo_args(&self) -> &'static [&'static str] { + match self { + Arch::X86_64 => &[ + "--target=x86_64-unknown-hermit", + "-Zbuild-std=std,panic_abort", + ], + Arch::Aarch64 => &[ + "--target=aarch64-unknown-hermit", + "-Zbuild-std=std,panic_abort", + ], + } + } + pub fn rustflags(&self) -> &'static [&'static str] { match self { Self::X86_64 => &[], diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs index ddb9981fc2..c769c6baa2 100644 --- a/xtask/src/artifact.rs +++ b/xtask/src/artifact.rs @@ -80,4 +80,16 @@ impl Artifact { .collect::() .into() } + + pub fn ci_image(&self, package: &str) -> PathBuf { + [ + "..".as_ref(), + self.target_dir(), + self.arch.hermit_triple().as_ref(), + self.profile_path_component().as_ref(), + package.as_ref(), + ] + .iter() + .collect() + } } diff --git a/xtask/src/ci/build.rs b/xtask/src/ci/build.rs new file mode 100644 index 0000000000..84cd3047a0 --- /dev/null +++ b/xtask/src/ci/build.rs @@ -0,0 +1,45 @@ +use std::path::PathBuf; + +use anyhow::Result; +use clap::Args; +use xshell::cmd; + +use crate::cargo_build::{CargoBuild, CmdExt}; + +/// Build hermit-rs images. +#[derive(Args)] +#[command(next_help_heading = "Build options")] +pub struct Build { + #[command(flatten)] + pub cargo_build: CargoBuild, + + /// Package to build (see `cargo help pkgid`) + #[arg(short, long, id = "SPEC")] + pub package: String, +} + +impl Build { + pub fn run(&self) -> Result<()> { + if super::in_ci() { + eprintln!("::group::cargo build") + } + + let sh = crate::sh()?; + + cmd!(sh, "cargo build --manifest-path ../Cargo.toml") + .args(self.cargo_build.artifact.arch.ci_cargo_args()) + .cargo_build_args(&self.cargo_build) + .args(&["--package", self.package.as_str()]) + .run()?; + + if super::in_ci() { + eprintln!("::endgroup::") + } + + Ok(()) + } + + pub fn image(&self) -> PathBuf { + self.cargo_build.artifact.ci_image(&self.package) + } +} diff --git a/xtask/src/ci/firecracker.rs b/xtask/src/ci/firecracker.rs new file mode 100644 index 0000000000..8f843b7633 --- /dev/null +++ b/xtask/src/ci/firecracker.rs @@ -0,0 +1,42 @@ +use std::path::Path; + +use anyhow::Result; +use clap::Args; +use xshell::cmd; + +use super::build::Build; + +/// Run hermit-rs images on Firecracker. +#[derive(Args)] +pub struct Firecracker { + #[command(flatten)] + build: Build, +} + +impl Firecracker { + pub fn run(self) -> Result<()> { + self.build.run()?; + + let sh = crate::sh()?; + + let config = format!( + include_str!("firecracker_vm_config.json"), + kernel_image_path = "../rusty-loader-x86_64-fc", + initrd_path = self.build.image().display() + ); + eprintln!("firecracker config"); + eprintln!("{config}"); + let config_path = Path::new("firecracker_vm_config.json"); + sh.write_file(config_path, config)?; + + let log_path = Path::new("firecracker.log"); + sh.write_file(log_path, "")?; + cmd!(sh, "firecracker --no-api --config-file {config_path} --log-path {log_path} --level Info --show-level --show-log-origin").run()?; + let log = sh.read_file(log_path)?; + + eprintln!("firecracker log"); + eprintln!("{log}"); + + Ok(()) + } +} diff --git a/xtask/src/ci/firecracker_vm_config.json b/xtask/src/ci/firecracker_vm_config.json new file mode 100644 index 0000000000..14c7b8e29d --- /dev/null +++ b/xtask/src/ci/firecracker_vm_config.json @@ -0,0 +1,13 @@ +{{ + "boot-source": {{ + "kernel_image_path": "{kernel_image_path}", + "initrd_path": "{initrd_path}", + "boot_args": "" + }}, + "drives": [], + "machine-config": {{ + "vcpu_count": 1, + "mem_size_mib": 256, + "smt": false + }} +}} diff --git a/xtask/src/ci/mod.rs b/xtask/src/ci/mod.rs new file mode 100644 index 0000000000..1a497241b3 --- /dev/null +++ b/xtask/src/ci/mod.rs @@ -0,0 +1,31 @@ +use anyhow::Result; +use clap::Subcommand; + +mod build; +mod firecracker; +mod qemu; +mod uhyve; + +/// Run CI tasks. +#[derive(Subcommand)] +pub enum Ci { + Build(build::Build), + Firecracker(firecracker::Firecracker), + Qemu(qemu::Qemu), + Uhyve(uhyve::Uhyve), +} + +impl Ci { + pub fn run(self) -> Result<()> { + match self { + Self::Build(build) => build.run(), + Self::Firecracker(firecracker) => firecracker.run(), + Self::Qemu(qemu) => qemu.run(), + Self::Uhyve(uhyve) => uhyve.run(), + } + } +} + +fn in_ci() -> bool { + std::env::var_os("CI") == Some("true".into()) +} diff --git a/xtask/src/ci/qemu.rs b/xtask/src/ci/qemu.rs new file mode 100644 index 0000000000..704371153a --- /dev/null +++ b/xtask/src/ci/qemu.rs @@ -0,0 +1,286 @@ +use std::net::UdpSocket; +use std::process::{Child, Command, ExitStatus}; +use std::time::Duration; +use std::{env, thread}; + +use anyhow::{bail, ensure, Context, Result}; +use clap::{Args, ValueEnum}; +use sysinfo::{CpuExt, CpuRefreshKind, System, SystemExt}; +use wait_timeout::ChildExt; +use xshell::cmd; + +use super::build::Build; +use crate::arch::Arch; + +/// Run hermit-rs images on QEMU. +#[derive(Args)] +pub struct Qemu { + /// Enable hardware acceleration. + #[arg(long)] + accel: bool, + + /// Enable the `microvm` machine type. + #[arg(long)] + microvm: bool, + + /// Enable a network device. + #[arg(long)] + netdev: Option, + + /// Create multiple vCPUs. + #[arg(long)] + smp: bool, + + /// Enable the `virtiofsd` virtio-fs vhost-user device daemon. + #[arg(long)] + virtiofsd: bool, + + #[command(flatten)] + build: Build, +} + +#[derive(ValueEnum, Clone, Copy)] +pub enum NetworkDevice { + VirtioNetPci, + Rtl8139, +} + +impl Qemu { + pub fn run(self) -> Result<()> { + self.build.run()?; + + let sh = crate::sh()?; + + let virtiofsd = self.virtiofsd.then(spawn_virtiofsd).transpose()?; + thread::sleep(Duration::from_millis(100)); + + let arch = self.build.cargo_build.artifact.arch.name(); + let qemu = env::var_os("QEMU").unwrap_or_else(|| format!("qemu-system-{arch}").into()); + + let qemu = cmd!(sh, "{qemu}") + .args(&["-display", "none"]) + .args(&["-serial", "stdio"]) + .args(&["-kernel", format!("../rusty-loader-{arch}").as_ref()]) + .args(self.machine_args()) + .args(self.cpu_args()) + .args(self.smp_args()) + .args(self.memory_args()) + .args(self.netdev_args()) + .args(self.virtiofsd_args()); + + eprintln!("$ {qemu}"); + let mut qemu = KillChildOnDrop( + Command::from(qemu) + .spawn() + .context("Failed to spawn QEMU")?, + ); + + thread::sleep(Duration::from_millis(100)); + if let Some(status) = qemu.0.try_wait()? { + ensure!(status.qemu_success(), "QEMU exit code: {:?}", status.code()); + } + + match self.build.package.as_str() { + "httpd" => test_httpd()?, + "testudp" => test_testudp()?, + _ => {} + } + + let status = qemu.0.wait_timeout(Duration::from_secs(60 * 2))?; + let Some(status) = status else { + bail!("QEMU timeout") + }; + ensure!(status.qemu_success(), "QEMU exit code: {:?}", status.code()); + + if let Some(mut virtiofsd) = virtiofsd { + let status = virtiofsd.0.wait()?; + assert!(status.success()); + } + + Ok(()) + } + + fn machine_args(&self) -> Vec { + if self.microvm { + let frequency = get_frequency(); + vec![ + "-M".to_string(), + "microvm,x-option-roms=off,pit=off,pic=off,rtc=on".to_string(), + "-global".to_string(), + "virtio-mmio.force-legacy=on".to_string(), + "-nodefaults".to_string(), + "-no-user-config".to_string(), + "-append".to_string(), + format!("-freq {frequency}"), + ] + } else if self.build.cargo_build.artifact.arch == Arch::Aarch64 { + vec!["-machine".to_string(), "virt,gic-version=3".to_string()] + } else { + vec![] + } + } + + fn cpu_args(&self) -> Vec { + match self.build.cargo_build.artifact.arch { + Arch::X86_64 => { + let mut cpu_args = if self.accel { + if cfg!(target_os = "linux") { + vec![ + "-enable-kvm".to_string(), + "-cpu".to_string(), + "host".to_string(), + ] + } else { + todo!() + } + } else { + vec!["-cpu".to_string(), "Skylake-Client".to_string()] + }; + cpu_args.push("-device".to_string()); + cpu_args.push("isa-debug-exit,iobase=0xf4,iosize=0x04".to_string()); + cpu_args.push("-initrd".to_string()); + cpu_args.push(self.build.image().into_os_string().into_string().unwrap()); + cpu_args + } + Arch::Aarch64 => { + let mut cpu_args = if self.accel { + todo!() + } else { + vec!["-cpu".to_string(), "cortex-a72".to_string()] + }; + cpu_args.push("-semihosting".to_string()); + cpu_args.push("-device".to_string()); + cpu_args.push(format!( + "guest-loader,addr=0x48000000,initrd={}", + self.build.image().display() + )); + cpu_args + } + } + } + + fn smp_args(&self) -> &'static [&'static str] { + if self.smp { + &["-smp", "4"] + } else { + &["-smp", "1"] + } + } + + fn memory(&self) -> usize { + let mut memory = 32usize; + if self.build.cargo_build.artifact.profile() == "dev" { + memory *= 4; + } + if self.netdev.is_some() { + memory *= 4; + } + if self.smp { + memory *= 4; + } + if self.build.cargo_build.artifact.arch == Arch::Aarch64 { + memory = memory.max(256); + } + memory + } + + fn memory_args(&self) -> [String; 2] { + ["-m".to_string(), format!("{}M", self.memory())] + } + + fn netdev_args(&self) -> &'static [&'static str] { + match self.netdev { + Some(NetworkDevice::VirtioNetPci) => &[ + "-netdev", + "user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9", + "-device", + "virtio-net-pci,netdev=u1,disable-legacy=on" + ], + Some(NetworkDevice::Rtl8139) => &[ + "-netdev", + "user,id=u1,hostfwd=tcp::9975-:9975,hostfwd=udp::9975-:9975,net=192.168.76.0/24,dhcpstart=192.168.76.9", + "-device", + "rtl8139,netdev=u1" + ], + None => &[], + } + } + + fn virtiofsd_args(&self) -> Vec { + if self.virtiofsd { + let memory = self.memory(); + vec![ + "-chardev".to_string(), + "socket,id=char0,path=./vhostqemu".to_string(), + "-device".to_string(), + "vhost-user-fs-pci,queue-size=1024,chardev=char0,tag=root".to_string(), + "-object".to_string(), + format!("memory-backend-file,id=mem,size={memory}M,mem-path=/dev/shm,share=on"), + "-numa".to_string(), + "node,memdev=mem".to_string(), + ] + } else { + vec![] + } + } +} + +fn spawn_virtiofsd() -> Result { + let sh = crate::sh()?; + + sh.create_dir("foo")?; + + let cmd = cmd!(sh, "virtiofsd --socket-path=./vhostqemu --shared-dir ./foo --announce-submounts --sandbox none --seccomp none --inode-file-handles=never"); + + eprintln!("$ {cmd}"); + + Ok(KillChildOnDrop(Command::from(cmd).spawn()?)) +} + +fn get_frequency() -> u64 { + let mut sys = System::new(); + sys.refresh_cpu_specifics(CpuRefreshKind::new().with_frequency()); + let frequency = sys.cpus().first().unwrap().frequency(); + assert!(sys.cpus().iter().all(|cpu| cpu.frequency() == frequency)); + frequency +} + +fn test_httpd() -> Result<()> { + thread::sleep(Duration::from_secs(10)); + eprintln!("[CI] GET http://127.0.0.1:9975"); + let body = ureq::get("http://127.0.0.1:9975") + .timeout(Duration::from_secs(3)) + .call()? + .into_string()?; + eprintln!("[CI] {body}"); + assert_eq!(body.lines().next(), Some("Hello from Hermit! 🦀")); + Ok(()) +} + +fn test_testudp() -> Result<()> { + thread::sleep(Duration::from_secs(10)); + let buf = "exit"; + eprintln!("[CI] send {buf:?} via UDP to 127.0.0.1:9975"); + let socket = UdpSocket::bind("127.0.0.1:0")?; + socket.connect("127.0.0.1:9975")?; + socket.send(buf.as_bytes())?; + Ok(()) +} + +struct KillChildOnDrop(Child); + +impl Drop for KillChildOnDrop { + fn drop(&mut self) { + self.0.kill().ok(); + } +} + +trait ExitStatusExt { + fn qemu_success(&self) -> bool; +} + +impl ExitStatusExt for ExitStatus { + fn qemu_success(&self) -> bool { + self.success() || self.code() == Some(3) + } +} diff --git a/xtask/src/ci/uhyve.rs b/xtask/src/ci/uhyve.rs new file mode 100644 index 0000000000..a26d13daf4 --- /dev/null +++ b/xtask/src/ci/uhyve.rs @@ -0,0 +1,41 @@ +use anyhow::Result; +use clap::Args; +use xshell::cmd; + +use super::build::Build; + +/// Run hermit-rs images on Uhyve. +#[derive(Args)] +pub struct Uhyve { + /// Run with multiple vCPUs. + #[arg(long)] + smp: bool, + + #[command(flatten)] + build: Build, +} + +impl Uhyve { + pub fn run(self) -> Result<()> { + self.build.run()?; + + let sh = crate::sh()?; + + let image = self.build.image(); + + cmd!(sh, "uhyve --verbose {image}") + .env("RUST_LOG", "debug") + .args(self.cpu_count_args()) + .run()?; + + Ok(()) + } + + fn cpu_count_args(&self) -> &'static [&'static str] { + if self.smp { + &["--cpu-count=4"] + } else { + &[] + } + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 0a089c9a4c..133db08239 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -5,6 +5,7 @@ mod archive; mod artifact; mod build; mod cargo_build; +mod ci; mod clippy; use std::path::Path; @@ -16,6 +17,8 @@ use xshell::Shell; #[derive(Parser)] enum Cli { Build(build::Build), + #[command(subcommand)] + Ci(ci::Ci), Clippy(clippy::Clippy), } @@ -23,6 +26,7 @@ impl Cli { fn run(self) -> Result<()> { match self { Self::Build(build) => build.run(), + Self::Ci(ci) => ci.run(), Self::Clippy(clippy) => clippy.run(), } }