Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

integration-test: Implement running on VMs #733

Merged
merged 2 commits into from
Aug 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 69 additions & 49 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,8 +139,14 @@ jobs:
--target ${{ matrix.target }} \
-Z build-std=core

build-integration-test:
runs-on: ubuntu-22.04
run-integration-test:
strategy:
fail-fast: false
matrix:
runner:
- macos-12
- ubuntu-22.04
runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -150,13 +156,12 @@ jobs:
with:
toolchain: nightly
components: rust-src
targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl

- uses: Swatinem/rust-cache@v2

- name: bpf-linker
run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git

- name: Install dependencies
- name: Install prerequisites
if: runner.os == 'Linux'
# ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
# enum values which was added in clang 15[1].
#
Expand All @@ -171,63 +176,78 @@ jobs:
set -euxo pipefail
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
echo deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy main | sudo tee /etc/apt/sources.list.d/llvm.list
sudo apt-get update
sudo apt-get -y install clang gcc-multilib llvm
sudo apt update
sudo apt -y install clang gcc-multilib llvm locate qemu-system-{arm,x86}

- name: Build
- name: bpf-linker
if: runner.os == 'Linux'
run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git

- name: Install prerequisites
if: runner.os == 'macOS'
# The clang shipped on macOS doesn't support BPF, so we need LLVM from brew.
#
# We also need LLVM for bpf-linker, see comment below.
run: |
set -euxo pipefail
mkdir -p integration-test-binaries
# See https://doc.rust-lang.org/cargo/reference/profiles.html for the
# names of the builtin profiles. Note that dev builds "debug" targets.
cargo xtask build-integration-test --cargo-arg=--profile=dev | xargs -I % cp % integration-test-binaries/dev
cargo xtask build-integration-test --cargo-arg=--profile=release | xargs -I % cp % integration-test-binaries/release

- uses: actions/upload-artifact@v3
with:
name: integration-test-binaries
path: integration-test-binaries
brew install qemu dpkg pkg-config llvm
echo /usr/local/opt/llvm/bin >> $GITHUB_PATH

run-integration-test:
runs-on: macos-latest
needs: ["build-integration-test"]
steps:
- uses: actions/checkout@v3
with:
sparse-checkout: |
test/run.sh
test/cloud-localds
- name: bpf-linker
if: runner.os == 'macOS'
# NB: rustc doesn't ship libLLVM.so on macOS, so disable proxying (default feature).
run: cargo install bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features

- name: Install Pre-requisites
- name: Download debian kernels
if: runner.arch == 'ARM64'
run: |
brew install qemu gnu-getopt coreutils cdrtools

- name: Cache tmp files
uses: actions/cache@v3
with:
path: |
.tmp/*.qcow2
.tmp/test_rsa
.tmp/test_rsa.pub
key: tmp-files-${{ hashFiles('test/run.sh') }}

- uses: actions/download-artifact@v3
with:
name: integration-test-binaries
path: integration-test-binaries

- name: Run integration tests
set -euxo pipefail
mkdir -p test/.tmp/debian-kernels/arm64
# NB: a 4.19 kernel image for arm64 was not available.
# TODO: enable tests on kernels before 6.0.
# linux-image-5.10.0-23-cloud-arm64-unsigned_5.10.179-3_arm64.deb \
printf '%s\0' \
linux-image-6.1.0-10-cloud-arm64-unsigned_6.1.38-2_arm64.deb \
linux-image-6.4.0-1-cloud-arm64-unsigned_6.4.4-2_arm64.deb \
| xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/arm64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{}

- name: Download debian kernels
if: runner.arch == 'X64'
run: |
set -euxo pipefail
find integration-test-binaries -type f -exec chmod +x {} \;
test/run.sh integration-test-binaries
mkdir -p test/.tmp/debian-kernels/amd64
# TODO: enable tests on kernels before 6.0.
# linux-image-4.19.0-21-cloud-amd64-unsigned_4.19.249-2_amd64.deb \
# linux-image-5.10.0-23-cloud-amd64-unsigned_5.10.179-3_amd64.deb \
printf '%s\0' \
linux-image-6.1.0-10-cloud-amd64-unsigned_6.1.38-2_amd64.deb \
linux-image-6.4.0-1-cloud-amd64-unsigned_6.4.4-2_amd64.deb \
| xargs -0 -t -P0 -I {} wget -nd -q -P test/.tmp/debian-kernels/amd64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{}

- name: Alias gtar as tar
if: runner.os == 'macOS'
# macOS tar doesn't support --wildcards which we use below.
run: mkdir tar-is-gtar && ln -s "$(which gtar)" tar-is-gtar/tar && echo "$PWD"/tar-is-gtar >> $GITHUB_PATH

- name: Extract debian kernels
run: |
set -euxo pipefail
find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"

- name: Run integration tests
run: find test/.tmp -name 'vmlinuz-*' | xargs -t cargo xtask integration-test vm

# Provides a single status check for the entire build workflow.
# This is used for merge automation, like Mergify, since GH actions
# has no concept of "when all status checks pass".
# https://docs.mergify.com/conditions/#validating-all-status-checks
build-workflow-complete:
needs: ["lint", "build-test-aya", "build-test-aya-bpf", "run-integration-test"]
needs:
- lint
- build-test-aya
- build-test-aya-bpf
- run-integration-test
runs-on: ubuntu-latest
steps:
- name: Build Complete
Expand Down
3 changes: 3 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ members = [
"aya-log-parser",
"aya-obj",
"aya-tool",
"init",
"test/integration-test",
"xtask",

Expand All @@ -29,6 +30,7 @@ default-members = [
"aya-log-parser",
"aya-obj",
"aya-tool",
"init",
# test/integration-test is omitted; including it in this list causes `cargo test` to run its
# tests, and that doesn't work unless they've been built with `cargo xtask`.
"xtask",
Expand Down Expand Up @@ -72,6 +74,7 @@ lazy_static = { version = "1", default-features = false }
libc = { version = "0.2.105", default-features = false }
log = { version = "0.4", default-features = false }
netns-rs = { version = "0.1", default-features = false }
nix = { version = "0.26.2", default-features = false }
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I for one am very pro nix and have 0 objections to using it in xtask.
ISTR in any we only used libc and I don't think we need to be as strict, but flagging it anyway.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ack.

num_enum = { version = "0.6", default-features = false }
object = { version = "0.31", default-features = false }
parking_lot = { version = "0.12.0", default-features = false }
Expand Down
19 changes: 13 additions & 6 deletions bpf/aya-log-ebpf/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
#![no_std]
#![warn(clippy::cast_lossless, clippy::cast_sign_loss)]

use aya_bpf::{
macros::map,
maps::{PerCpuArray, PerfEventByteArray},
};
#[cfg(target_arch = "bpf")]
use aya_bpf::macros::map;
use aya_bpf::maps::{PerCpuArray, PerfEventByteArray};
pub use aya_log_common::{write_record_header, Level, WriteToBuf, LOG_BUF_CAPACITY};
pub use aya_log_ebpf_macros::{debug, error, info, log, trace, warn};

Expand All @@ -15,11 +14,19 @@ pub struct LogBuf {
}

#[doc(hidden)]
#[map]
// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't
// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so
// because the integration-test crate depends on this crate transitively. See comment in
// test/integration-test/Cargo.toml.
#[cfg_attr(target_arch = "bpf", map)]
pub static mut AYA_LOG_BUF: PerCpuArray<LogBuf> = PerCpuArray::with_max_entries(1, 0);

#[doc(hidden)]
#[map]
// This cfg_attr prevents compilation failures on macOS where the generated section name doesn't
// meet mach-o's requirements. We wouldn't ordinarily build this crate for macOS, but we do so
// because the integration-test crate depends on this crate transitively. See comment in
// test/integration-test/Cargo.toml.
#[cfg_attr(target_arch = "bpf", map)]
pub static mut AYA_LOGS: PerfEventByteArray = PerfEventByteArray::new(0);

#[doc(hidden)]
Expand Down
10 changes: 10 additions & 0 deletions init/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[package]
name = "init"
version = "0.1.0"
authors = ["Tamir Duberstein <tamird@gmail.com>"]
edition = "2021"
publish = false

[dependencies]
anyhow = { workspace = true, features = ["std"] }
nix = { workspace = true, features = ["fs", "mount", "reboot"] }
166 changes: 166 additions & 0 deletions init/src/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
//! init is the first process started by the kernel.
//!
//! This implementation creates the minimal mounts required to run BPF programs, runs all binaries
//! in /bin, prints a final message ("init: success|failure"), and powers off the machine.

use anyhow::Context as _;

#[derive(Debug)]
struct Errors(Vec<anyhow::Error>);

impl std::fmt::Display for Errors {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let Self(errors) = self;
for (i, error) in errors.iter().enumerate() {
if i != 0 {
writeln!(f)?;
}
write!(f, "{:?}", error)?;
}
Ok(())
}
}

impl std::error::Error for Errors {}

fn run() -> anyhow::Result<()> {
const RXRXRX: nix::sys::stat::Mode = nix::sys::stat::Mode::empty()
.union(nix::sys::stat::Mode::S_IRUSR)
.union(nix::sys::stat::Mode::S_IXUSR)
.union(nix::sys::stat::Mode::S_IRGRP)
.union(nix::sys::stat::Mode::S_IXGRP)
.union(nix::sys::stat::Mode::S_IROTH)
.union(nix::sys::stat::Mode::S_IXOTH);

struct Mount {
source: &'static str,
target: &'static str,
fstype: &'static str,
flags: nix::mount::MsFlags,
data: Option<&'static str>,
target_mode: Option<nix::sys::stat::Mode>,
}

for Mount {
source,
target,
fstype,
flags,
data,
target_mode,
} in [
Mount {
source: "proc",
target: "/proc",
fstype: "proc",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: Some(RXRXRX),
},
Mount {
source: "sysfs",
target: "/sys",
fstype: "sysfs",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: Some(RXRXRX),
},
Mount {
source: "debugfs",
target: "/sys/kernel/debug",
fstype: "debugfs",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: None,
},
Mount {
source: "bpffs",
target: "/sys/fs/bpf",
fstype: "bpf",
flags: nix::mount::MsFlags::empty(),
data: None,
target_mode: None,
},
] {
match target_mode {
None => {
// Must exist.
let nix::sys::stat::FileStat { st_mode, .. } = nix::sys::stat::stat(target)
.with_context(|| format!("stat({target}) failed"))?;
dave-tucker marked this conversation as resolved.
Show resolved Hide resolved
let s_flag = nix::sys::stat::SFlag::from_bits_truncate(st_mode);

if !s_flag.contains(nix::sys::stat::SFlag::S_IFDIR) {
anyhow::bail!("{target} is not a directory");
}
}
Some(target_mode) => {
// Must not exist.
nix::unistd::mkdir(target, target_mode)
.with_context(|| format!("mkdir({target}) failed"))?;
}
}
nix::mount::mount(Some(source), target, Some(fstype), flags, data).with_context(|| {
format!("mount({source}, {target}, {fstype}, {flags:?}, {data:?}) failed")
})?;
}

// By contract we run everything in /bin and assume they're rust test binaries.
//
// If the user requested command line arguments, they're named init.arg={}.

// Read kernel parameters from /proc/cmdline. They're space separated on a single line.
let cmdline = std::fs::read_to_string("/proc/cmdline")
.with_context(|| "read_to_string(/proc/cmdline) failed")?;
let args = cmdline
.split_whitespace()
.filter_map(|parameter| {
parameter
.strip_prefix("init.arg=")
.map(std::ffi::OsString::from)
})
.collect::<Vec<_>>();

// Iterate files in /bin.
let read_dir = std::fs::read_dir("/bin").context("read_dir(/bin) failed")?;
dave-tucker marked this conversation as resolved.
Show resolved Hide resolved
let errors = read_dir
.filter_map(|entry| {
match (|| {
let entry = entry.context("read_dir(/bin) failed")?;
let path = entry.path();
let status = std::process::Command::new(&path)
.args(&args)
.status()
.with_context(|| format!("failed to execute {}", path.display()))?;

if status.code() == Some(0) {
Ok(())
} else {
Err(anyhow::anyhow!("{} failed: {status:?}", path.display()))
}
})() {
Ok(()) => None,
Err(err) => Some(err),
}
})
.collect::<Vec<_>>();
if errors.is_empty() {
Ok(())
} else {
Err(Errors(errors).into())
}
}

fn main() {
match run() {
Ok(()) => {
println!("init: success");
}
Err(err) => {
println!("{err:?}");
println!("init: failure");
dave-tucker marked this conversation as resolved.
Show resolved Hide resolved
}
}
let how = nix::sys::reboot::RebootMode::RB_POWER_OFF;
let _: std::convert::Infallible = nix::sys::reboot::reboot(how)
.unwrap_or_else(|err| panic!("reboot({how:?}) failed: {err:?}"));
}
Loading
Loading