Skip to content

Commit

Permalink
integration-test: Implement running on VMs
Browse files Browse the repository at this point in the history
Implements running integration tests on multiple VMs with arbitrary
kernel images using `cargo xtask integration-test vm ...`.

This changes our coverage from 6.2 to 6.1 and 6.4.
  • Loading branch information
tamird committed Aug 9, 2023
1 parent b6a6a81 commit 82a77bc
Show file tree
Hide file tree
Showing 10 changed files with 709 additions and 644 deletions.
118 changes: 69 additions & 49 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,14 @@ jobs:
--target ${{ matrix.target }} \
-Z build-std=core
build-integration-test:
runs-on: ubuntu-22.04
run-integration-test:
strategy:
fail-fast: false
matrix:
runner:
- macos-12
- ubuntu-22.04
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -150,13 +156,12 @@ jobs:
with:
toolchain: nightly
components: rust-src
targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl

- uses: Swatinem/rust-cache@v2

- name: bpf-linker
run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git

- name: Install dependencies
- name: Install prerequisites
if: runner.os == 'Linux'
# ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
# enum values which was added in clang 15[1].
#
Expand All @@ -171,63 +176,78 @@ jobs:
set -euxo pipefail
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update
sudo apt-get -y install clang gcc-multilib llvm
sudo apt update
sudo apt -y install clang gcc-multilib llvm locate qemu-system-{arm,x86}
- name: Build
- name: bpf-linker
if: runner.os == 'Linux'
run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git

- name: Install prerequisites
if: runner.os == 'macOS'
# The clang shipped on macOS doesn't support BPF, so we need LLVM from brew.
#
# We also need LLVM for bpf-linker, see comment below.
run: |
set -euxo pipefail
mkdir -p integration-test-binaries
# See https://doc.rust-lang.org/cargo/reference/profiles.html for the
# names of the builtin profiles. Note that dev builds "debug" targets.
cargo xtask build-integration-test --cargo-arg=--profile=dev | xargs -I % cp % integration-test-binaries/dev
cargo xtask build-integration-test --cargo-arg=--profile=release | xargs -I % cp % integration-test-binaries/release
- uses: actions/upload-artifact@v3
with:
name: integration-test-binaries
path: integration-test-binaries
brew install qemu dpkg pkg-config llvm
echo /usr/local/opt/llvm/bin >> $GITHUB_PATH
run-integration-test:
runs-on: macos-latest
needs: ["build-integration-test"]
steps:
- uses: actions/checkout@v3
with:
sparse-checkout: |
test/run.sh
test/cloud-localds
- name: bpf-linker
if: runner.os == 'macOS'
# NB: rustc doesn't ship libLLVM.so on macOS, so disable proxying (default feature).
run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features

- name: Install Pre-requisites
- name: Download debian kernels
if: runner.arch == 'ARM64'
run: |
brew install qemu gnu-getopt coreutils cdrtools
- name: Cache tmp files
uses: actions/cache@v3
with:
path: |
.tmp/*.qcow2
.tmp/test_rsa
.tmp/test_rsa.pub
key: tmp-files-${{ hashFiles('test/run.sh') }}

- uses: actions/download-artifact@v3
with:
name: integration-test-binaries
path: integration-test-binaries

- name: Run integration tests
set -euxo pipefail
mkdir -p test/.tmp/debian-kernels/arm64
# NB: a 4.19 kernel image for arm64 was not available.
# TODO: enable tests on kernels before 6.0.
# linux-image-5.10.0-23-cloud-arm64-unsigned_5.10.179-3_arm64.deb \
printf '%s\0' \
linux-image-6.1.0-10-cloud-arm64-unsigned_6.1.38-2_arm64.deb \
linux-image-6.4.0-1-cloud-arm64-unsigned_6.4.4-2_arm64.deb \
| xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/arm64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{}
- name: Download debian kernels
if: runner.arch == 'X64'
run: |
set -euxo pipefail
find integration-test-binaries -type f -exec chmod +x {} \;
test/run.sh integration-test-binaries
mkdir -p test/.tmp/debian-kernels/amd64
# TODO: enable tests on kernels before 6.0.
# linux-image-4.19.0-21-cloud-amd64-unsigned_4.19.249-2_amd64.deb \
# linux-image-5.10.0-23-cloud-amd64-unsigned_5.10.179-3_amd64.deb \
printf '%s\0' \
linux-image-6.1.0-10-cloud-amd64-unsigned_6.1.38-2_amd64.deb \
linux-image-6.4.0-1-cloud-amd64-unsigned_6.4.4-2_amd64.deb \
| xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/amd64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{}
- name: Alias gtar as tar
if: runner.os == 'macOS'
# macOS tar doesn't support --wildcards which we use below.
run: mkdir tar-is-gtar && ln -s "$(which gtar)" tar-is-gtar/tar && echo "$PWD"/tar-is-gtar >> $GITHUB_PATH

- name: Extract debian kernels
run: |
set -euxo pipefail
find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
- name: Run integration tests
run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm

# Provides a single status check for the entire build workflow.
# This is used for merge automation, like Mergify, since GH actions
# has no concept of "when all status checks pass".
# https://docs.mergify.com/conditions/#validating-all-status-checks
build-workflow-complete:
needs: ["lint", "build-test-aya", "build-test-aya-bpf", "run-integration-test"]
needs:
- lint
- build-test-aya
- build-test-aya-bpf
- run-integration-test
runs-on: ubuntu-latest
steps:
- name: Build Complete
Expand Down
3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ members = [
"aya-log-parser",
"aya-obj",
"aya-tool",
"init",
"test/integration-test",
"xtask",

Expand All @@ -29,6 +30,7 @@ default-members = [
"aya-log-parser",
"aya-obj",
"aya-tool",
"init",
# test/integration-test is omitted; including it in this list causes `cargo test` to run its
# tests, and that doesn't work unless they've been built with `cargo xtask`.
"xtask",
Expand Down Expand Up @@ -72,6 +74,7 @@ lazy_static = { version = "1", default-features = false }
libc = { version = "0.2.105", default-features = false }
log = { version = "0.4", default-features = false }
netns-rs = { version = "0.1", default-features = false }
nix = { version = "0.26.2", default-features = false }
num_enum = { version = "0.6", default-features = false }
object = { version = "0.31", default-features = false }
parking_lot = { version = "0.12.0", default-features = false }
Expand Down
19 changes: 13 additions & 6 deletions bpf/aya-log-ebpf/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
#![no_std]
#![warn(clippy::cast_lossless, clippy::cast_sign_loss)]

use aya_bpf::{
macros::map,
maps::{PerCpuArray, PerfEventByteArray},
};
#[cfg(target_arch = "bpf")]
use aya_bpf::macros::map;
use aya_bpf::maps::{PerCpuArray, PerfEventByteArray};
pub use aya_log_common::{write_record_header, Level, WriteToBuf, LOG_BUF_CAPACITY};
pub use aya_log_ebpf_macros::{debug, error, info, log, trace, warn};

Expand All @@ -15,11 +14,19 @@ pub struct LogBuf {
}

#[doc(hidden)]
#[map]
// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't
// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so
// because the integration-test crate depends on this crate transitively. See comment in
// test/integration-test/Cargo.toml.
#[cfg_attr(target_arch = "bpf", map)]
pub static mut AYA_LOG_BUF: PerCpuArray<LogBuf> = PerCpuArray::with_max_entries(1, 0);

#[doc(hidden)]
#[map]
// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't
// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so
// because the integration-test crate depends on this crate transitively. See comment in
// test/integration-test/Cargo.toml.
#[cfg_attr(target_arch = "bpf", map)]
pub static mut AYA_LOGS: PerfEventByteArray = PerfEventByteArray::new(0);

#[doc(hidden)]
Expand Down
10 changes: 10 additions & 0 deletions init/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[package]
name = "init"
version = "0.1.0"
authors = ["Tamir Duberstein <tamird@gmail.com>"]
edition = "2021"
publish = false

[dependencies]
anyhow = { workspace = true, features = ["std"] }
nix = { workspace = true, features = ["fs", "mount", "reboot"] }
166 changes: 166 additions & 0 deletions init/src/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
//! init is the first process started by the kernel.
//!
//! This implementation creates the minimal mounts required to run BPF programs, runs all binaries
//! in /bin, prints a final message ("init: success|failure"), and powers off the machine.

use anyhow::Context as _;

#[derive(Debug)]
struct Errors(Vec<anyhow::Error>);

impl std::fmt::Display for Errors {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let Self(errors) = self;
for (i, error) in errors.iter().enumerate() {
if i != 0 {
writeln!(f)?;
}
write!(f, "{:?}", error)?;
}
Ok(())
}
}

impl std::error::Error for Errors {}

fn run() -> anyhow::Result<()> {
const RXRXRX: nix::sys::stat::Mode = nix::sys::stat::Mode::empty()
.union(nix::sys::stat::Mode::S_IRUSR)
.union(nix::sys::stat::Mode::S_IXUSR)
.union(nix::sys::stat::Mode::S_IRGRP)
.union(nix::sys::stat::Mode::S_IXGRP)
.union(nix::sys::stat::Mode::S_IROTH)
.union(nix::sys::stat::Mode::S_IXOTH);

struct Mount {
source: &'static str,
target: &'static str,
fstype: &'static str,
flags: nix::mount::MsFlags,
data: Option<&'static str>,
target_mode: Option<nix::sys::stat::Mode>,
}

for Mount {
source,
target,
fstype,
flags,
data,
target_mode,
} in [
Mount {
source: "proc",
target: "/proc",
fstype: "proc",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: Some(RXRXRX),
},
Mount {
source: "sysfs",
target: "/sys",
fstype: "sysfs",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: Some(RXRXRX),
},
Mount {
source: "debugfs",
target: "/sys/kernel/debug",
fstype: "debugfs",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: None,
},
Mount {
source: "bpffs",
target: "/sys/fs/bpf",
fstype: "bpf",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: None,
},
] {
match target_mode {
None => {
// Must exist.
let nix::sys::stat::FileStat { st_mode, .. } = nix::sys::stat::stat(target)
.with_context(|| format!("stat({target}) failed"))?;
let s_flag = nix::sys::stat::SFlag::from_bits_truncate(st_mode);

if !s_flag.contains(nix::sys::stat::SFlag::S_IFDIR) {
anyhow::bail!("{target} is not a directory");
}
}
Some(target_mode) => {
// Must not exist.
nix::unistd::mkdir(target, target_mode)
.with_context(|| format!("mkdir({target}) failed"))?;
}
}
nix::mount::mount(Some(source), target, Some(fstype), flags, data).with_context(|| {
format!("mount({source}, {target}, {fstype}, {flags:?}, {data:?}) failed")
})?;
}

// By contract we run everything in /bin and assume they're rust test binaries.
//
// If the user requested command line arguments, they're named init.arg={}.

// Read kernel parameters from /proc/cmdline. They're space separated on a single line.
let cmdline = std::fs::read_to_string("/proc/cmdline")
.with_context(|| "read_to_string(/proc/cmdline) failed")?;
let args = cmdline
.split_whitespace()
.filter_map(|parameter| {
parameter
.strip_prefix("init.arg=")
.map(std::ffi::OsString::from)
})
.collect::<Vec<_>>();

// Iterate files in /bin.
let read_dir = std::fs::read_dir("/bin").context("read_dir(/bin) failed")?;
let errors = read_dir
.filter_map(|entry| {
match (|| {
let entry = entry.context("read_dir(/bin) failed")?;
let path = entry.path();
let status = std::process::Command::new(&path)
.args(&args)
.status()
.with_context(|| format!("failed to execute {}", path.display()))?;

if status.code() == Some(0) {
Ok(())
} else {
Err(anyhow::anyhow!("{} failed: {status:?}", path.display()))
}
})() {
Ok(()) => None,
Err(err) => Some(err),
}
})
.collect::<Vec<_>>();
if errors.is_empty() {
Ok(())
} else {
Err(Errors(errors).into())
}
}

fn main() {
match run() {
Ok(()) => {
println!("init: success");
}
Err(err) => {
println!("{err:?}");
println!("init: failure");
}
}
let how = nix::sys::reboot::RebootMode::RB_POWER_OFF;
let _: std::convert::Infallible = nix::sys::reboot::reboot(how)
.unwrap_or_else(|err| panic!("reboot({how:?}) failed: {err:?}"));
}
Loading

0 comments on commit 82a77bc

Please sign in to comment.