From 213f01fa18eca8a03c8793887975cd6158fbe735 Mon Sep 17 00:00:00 2001 From: Todd Neal Date: Mon, 24 Jun 2024 15:27:21 +0000 Subject: [PATCH] add ephemeral-storage commands --- .../api/apiclient/src/ephemeral_storage.rs | 87 ++++ sources/api/apiclient/src/lib.rs | 1 + sources/api/apiclient/src/main.rs | 236 +++++++++- .../apiserver/src/server/ephemeral_storage.rs | 414 ++++++++++++++++++ sources/api/apiserver/src/server/error.rs | 16 + sources/api/apiserver/src/server/mod.rs | 94 +++- sources/api/openapi.yaml | 126 +++++- sources/models/src/ephemeral_storage.rs | 32 ++ sources/models/src/lib.rs | 3 + 9 files changed, 995 insertions(+), 14 deletions(-) create mode 100644 sources/api/apiclient/src/ephemeral_storage.rs create mode 100644 sources/api/apiserver/src/server/ephemeral_storage.rs create mode 100644 sources/models/src/ephemeral_storage.rs diff --git a/sources/api/apiclient/src/ephemeral_storage.rs b/sources/api/apiclient/src/ephemeral_storage.rs new file mode 100644 index 000000000..5b596f944 --- /dev/null +++ b/sources/api/apiclient/src/ephemeral_storage.rs @@ -0,0 +1,87 @@ +use model::ephemeral_storage::{Bind, Filesystem, Init}; +use snafu::ResultExt; +use std::path::Path; + +/// Requests ephemeral storage initialization through the API +pub async fn initialize

( + socket_path: P, + filesystem: Option, + disks: Option>, +) -> Result<()> +where + P: AsRef, +{ + let uri = "/actions/ephemeral-storage/init"; + let opts = + serde_json::to_string(&Init { filesystem, disks }).context(error::JsonSerializeSnafu {})?; + let method = "POST"; + let (_status, _body) = crate::raw_request(&socket_path, &uri, method, Some(opts)) + .await + .context(error::RequestSnafu { uri, method })?; + Ok(()) +} + +/// Requests binding of directories to configured ephemeral storage through the API +pub async fn bind

(socket_path: P, targets: Vec) -> Result<()> +where + P: AsRef, +{ + let uri = "/actions/ephemeral-storage/bind"; + let opts = serde_json::to_string(&Bind { targets }).context(error::JsonSerializeSnafu {})?; + let method = "POST"; + let (_status, _body) = crate::raw_request(&socket_path, &uri, method, Some(opts)) + .await + .context(error::RequestSnafu { uri, method })?; + Ok(()) +} + +/// Lists the ephemeral disks available for configuration +pub async fn list_disks

(socket_path: P, format: Option) -> Result +where + P: AsRef, +{ + list(socket_path, "list-disks", format).await +} + +/// Lists the ephemeral disks available for configuration +pub async fn list_dirs

(socket_path: P, format: Option) -> Result +where + P: AsRef, +{ + list(socket_path, "list-dirs", format).await +} +async fn list

(socket_path: P, item: &str, format: Option) -> Result +where + P: AsRef, +{ + let mut query = Vec::new(); + if let Some(query_format) = format { + query.push(format!("format={}", query_format)); + } + + let uri = format!("/actions/ephemeral-storage/{}?{}", item, query.join("&")); + let method = "GET"; + let (_status, body) = crate::raw_request(&socket_path, &uri, method, None) + .await + .context(error::RequestSnafu { uri, method })?; + Ok(body) +} +mod error { + use snafu::Snafu; + + #[derive(Debug, Snafu)] + #[snafu(visibility(pub(super)))] + pub enum Error { + #[snafu(display("Failed {} request to '{}': {}", method, uri, source))] + Request { + method: String, + uri: String, + #[snafu(source(from(crate::Error, Box::new)))] + source: Box, + }, + #[snafu(display("Failed to serialize parameters"))] + JsonSerialize { source: serde_json::Error }, + } +} +pub use error::Error; +pub type Result = std::result::Result; diff --git a/sources/api/apiclient/src/lib.rs b/sources/api/apiclient/src/lib.rs index 8fcbf9697..c322c2f7c 100644 --- a/sources/api/apiclient/src/lib.rs +++ b/sources/api/apiclient/src/lib.rs @@ -20,6 +20,7 @@ use snafu::{ensure, ResultExt}; use std::{fmt, fmt::Display, path::Path}; pub mod apply; +pub mod ephemeral_storage; pub mod exec; pub mod get; pub mod reboot; diff --git a/sources/api/apiclient/src/main.rs b/sources/api/apiclient/src/main.rs index 44b30db0e..b7579c56f 100644 --- a/sources/api/apiclient/src/main.rs +++ b/sources/api/apiclient/src/main.rs @@ -6,8 +6,9 @@ // library calls based on the given flags, etc.) The library modules contain the code for talking // to the API, which is intended to be reusable by other crates. -use apiclient::{apply, exec, get, reboot, report, set, update, SettingsInput}; +use apiclient::{apply, ephemeral_storage, exec, get, reboot, report, set, update, SettingsInput}; use log::{info, log_enabled, trace, warn}; +use model::ephemeral_storage::Filesystem; use serde::{Deserialize, Serialize}; use simplelog::{ ColorChoice, ConfigBuilder as LogConfigBuilder, LevelFilter, TermLogger, TerminalMode, @@ -15,8 +16,10 @@ use simplelog::{ use snafu::ResultExt; use std::env; use std::ffi::OsString; +use std::iter::Peekable; use std::process; use std::str::FromStr; +use std::vec::IntoIter; use unindent::unindent; const DEFAULT_METHOD: &str = "GET"; @@ -48,6 +51,7 @@ enum Subcommand { Set(SetArgs), Update(UpdateSubcommand), Report(ReportSubcommand), + EphemeralStorage(EphemeralStorageSubcommand), } /// Stores user-supplied arguments for the 'apply' subcommand. @@ -140,6 +144,33 @@ struct UpdateApplyArgs { #[derive(Debug)] struct UpdateCancelArgs {} +/// Stores the 'ephemeral-storage' subcommand specified by the user. +#[derive(Debug)] +enum EphemeralStorageSubcommand { + Init(EphemeralStorageInitArgs), + Bind(EphemeralStorageBindArgs), + ListDisks(EphemeralStorageFormatArgs), + ListDirs(EphemeralStorageFormatArgs), +} + +/// Stores user-supplied arguments for the 'ephemeral-storage init' subcommand. +#[derive(Debug)] +struct EphemeralStorageInitArgs { + disks: Option>, + filesystem: Option, +} + +/// Stores user-supplied arguments for the 'ephemeral-storage bind' subcommand. +#[derive(Debug)] +struct EphemeralStorageBindArgs { + targets: Vec, +} +/// Stores user-supplied arguments for the 'ephemeral-storage list-disks/list-dirs' subcommand. +#[derive(Debug)] +struct EphemeralStorageFormatArgs { + format: Option, +} + /// Informs the user about proper usage of the program and exits. fn usage() -> ! { let msg = &format!( @@ -166,6 +197,12 @@ fn usage() -> ! { report cis Retrieve a Bottlerocket CIS benchmark compliance report. report cis-k8s Retrieve a Kubernetes CIS benchmark compliance report. report fips Retrieve a FIPS Security Policy compliance report. + ephemeral-storage init Initialize ephemeral storage + ephemeral-storage bind Bind directories to previously initialized ephemeral storage. + ephemeral-storage list-disks + List the discovered ephemeral disks that can be initialized. + ephemeral-storage list-dirs + List the directories that can be bound to ephemeral storage. raw options: -u, --uri URI Required; URI to request from the server, e.g. /tx @@ -223,7 +260,28 @@ fn usage() -> ! { report cis-k8s options: -f, --format Format of the CIS report (text or json). Default format is text. - -l, --level CIS compliance level to report on (1 or 2). Default is 1."#, + -l, --level CIS compliance level to report on (1 or 2). Default is 1. + + ephemeral-storage init options: + -t, --filesystem Filesystem to initialize the array as (ext4 or xfs). Default is + xfs. If a single disk is provided, it is mounted directly without + constructing an array. If no ephemeral disks are found, this + operation does nothing. + --disks DISK [DISK ...] Local disks to configure for storage. Default is all ephemeral + disks. + + ephemeral-storage bind options: + --dirs DIR [DIR ...] Directories to bind to configured ephemeral storage + (e.g. /var/lib/containerd). If no ephemeral disks are found + this operation does nothing. + + ephemeral-storage list-disks options: + -f, --format Format of the disk listing (text or json). Default format is text. + + ephemeral-storage list-dirs options: + -f, --format Format of the directory listing (text or json). Default format is text. + + "#, socket = constants::API_SOCKET, method = DEFAULT_METHOD, ); @@ -272,6 +330,7 @@ fn parse_args(args: env::Args) -> (Args, Subcommand) { // Subcommands "raw" | "apply" | "exec" | "get" | "reboot" | "report" | "set" | "update" + | "ephemeral-storage" if subcommand.is_none() && !arg.starts_with('-') => { subcommand = Some(arg) @@ -292,6 +351,7 @@ fn parse_args(args: env::Args) -> (Args, Subcommand) { Some("report") => (global_args, parse_report_args(subcommand_args)), Some("set") => (global_args, parse_set_args(subcommand_args)), Some("update") => (global_args, parse_update_args(subcommand_args)), + Some("ephemeral-storage") => (global_args, parse_ephemeral_storage_args(subcommand_args)), _ => usage_msg("Missing or unknown subcommand"), } } @@ -668,6 +728,140 @@ fn parse_fips_arguments(args: Vec) -> FipsReportArgs { FipsReportArgs { format } } +/// Parse the desired subcommand of 'ephemeral-storage' +fn parse_ephemeral_storage_args(args: Vec) -> Subcommand { + let mut subcommand = None; + let mut subcommand_args = Vec::new(); + + for arg in args.into_iter() { + match arg.as_ref() { + // Subcommands + "init" | "bind" | "list-disks" | "list-dirs" + if subcommand.is_none() && !arg.starts_with('-') => + { + subcommand = Some(arg) + } + + // Other arguments are passed to the subcommand parser + _ => subcommand_args.push(arg), + } + } + + let cmd = match subcommand.as_deref() { + Some("init") => parse_ephemeral_storage_init_args(subcommand_args), + Some("bind") => parse_ephemeral_storage_bind_args(subcommand_args), + Some("list-disks") => EphemeralStorageSubcommand::ListDisks( + parse_ephemeral_storage_list_format_args(subcommand_args), + ), + Some("list-dirs") => EphemeralStorageSubcommand::ListDirs( + parse_ephemeral_storage_list_format_args(subcommand_args), + ), + _ => usage_msg("Missing or unknown subcommand for 'ephemeral-storage'"), + }; + Subcommand::EphemeralStorage(cmd) +} + +/// Parses arguments for the 'init' ephemeral-storage subcommand. +fn parse_ephemeral_storage_init_args(args: Vec) -> EphemeralStorageSubcommand { + let mut disks: Option> = None; + let mut filesystem = None; + let mut iter = args.into_iter().peekable(); + while let Some(arg) = iter.next() { + match arg.as_ref() { + "-t" | "--filesystem" => { + match iter + .next() + .unwrap_or_else(|| usage_msg("Did not give argument to -t | --filesystem")) + .as_str() + { + "ext4" => filesystem = Some(Filesystem::Ext4), + "xfs" => filesystem = Some(Filesystem::Xfs), + _ => usage_msg("Unsupported filesystem type"), + } + } + "--disks" => { + let mut names = collect_non_args(&mut iter); + if names.is_empty() { + usage_msg("Did not give argument to --disks") + } + if let Some(existing) = &mut disks { + existing.append(&mut names); + } else { + disks = Some(names); + } + } + x => usage_msg(format!("Unknown argument '{}'", x)), + } + } + EphemeralStorageSubcommand::Init(EphemeralStorageInitArgs { disks, filesystem }) +} + +/// Parses arguments for the 'bind' ephemeral-storage subcommand. +fn parse_ephemeral_storage_bind_args(args: Vec) -> EphemeralStorageSubcommand { + if args.is_empty() { + usage_msg("Did not give arguments to bind") + } + let mut targets = Vec::new(); + let mut iter = args.into_iter().peekable(); + while let Some(arg) = iter.next() { + match arg.as_ref() { + "--dirs" => { + targets.append(&mut collect_non_args(&mut iter)); + if targets.is_empty() { + usage_msg("Did not give argument to --dirs") + } + } + x => usage_msg(format!("Unknown argument '{}'", x)), + } + } + EphemeralStorageSubcommand::Bind(EphemeralStorageBindArgs { targets }) +} + +/// Parses arguments for the 'list-disks' and 'list-dirs' ephemeral-storage subcommand. +fn parse_ephemeral_storage_list_format_args(args: Vec) -> EphemeralStorageFormatArgs { + let mut format = None; + + let mut iter = args.into_iter(); + while let Some(arg) = iter.next() { + match arg.as_ref() { + "-f" | "--format" => { + format = Some( + iter.next() + .unwrap_or_else(|| usage_msg("Did not give argument to -f | --format")), + ) + } + x => usage_msg(format!("Unknown argument '{}'", x)), + } + } + EphemeralStorageFormatArgs { format } +} + +/// collects non-argument parameters (those not starting with a '-') up until the next +/// argument is seen +fn collect_non_args(iter: &mut Peekable>) -> Vec { + let mut result = Vec::new(); + loop { + // look at the following argument and stop accepting disk names + // once we reach the end of arguments, or find the beginning of + // the next argument + match iter.peek() { + None => { + break; + } + Some(peeked) => { + if peeked.is_empty() || peeked.starts_with('-') { + break; + } + } + } + let next = iter + .next() + .unwrap_or_else(|| usage_msg("Expected non-empty argument")); + result.push(next); + } + result +} + // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= // Helpers @@ -859,6 +1053,39 @@ async fn run() -> Result<()> { } } }, + + Subcommand::EphemeralStorage(subcommand) => match subcommand { + EphemeralStorageSubcommand::Init(cfg_args) => { + ephemeral_storage::initialize( + &args.socket_path, + cfg_args.filesystem, + cfg_args.disks, + ) + .await + .context(error::EphemeralStorageSnafu)?; + } + EphemeralStorageSubcommand::Bind(bind_args) => { + ephemeral_storage::bind(&args.socket_path, bind_args.targets) + .await + .context(error::EphemeralStorageSnafu)?; + } + EphemeralStorageSubcommand::ListDisks(bind_args) => { + let body = ephemeral_storage::list_disks(&args.socket_path, bind_args.format) + .await + .context(error::EphemeralStorageSnafu)?; + if !body.is_empty() { + print!("{}", body); + } + } + EphemeralStorageSubcommand::ListDirs(bind_args) => { + let body = ephemeral_storage::list_dirs(&args.socket_path, bind_args.format) + .await + .context(error::EphemeralStorageSnafu)?; + if !body.is_empty() { + print!("{}", body); + } + } + }, } Ok(()) @@ -876,7 +1103,7 @@ async fn main() { } mod error { - use apiclient::{apply, exec, get, reboot, report, set, update}; + use apiclient::{apply, ephemeral_storage, exec, get, reboot, report, set, update}; use snafu::Snafu; #[derive(Debug, Snafu)] @@ -922,6 +1149,9 @@ mod error { #[snafu(display("Failed to check for updates: {}", source))] UpdateCheck { source: update::Error }, + + #[snafu(display("Failed to initialize ephemeral storage: {}", source))] + EphemeralStorage { source: ephemeral_storage::Error }, } } type Result = std::result::Result; diff --git a/sources/api/apiserver/src/server/ephemeral_storage.rs b/sources/api/apiserver/src/server/ephemeral_storage.rs new file mode 100644 index 000000000..740f887c9 --- /dev/null +++ b/sources/api/apiserver/src/server/ephemeral_storage.rs @@ -0,0 +1,414 @@ +//! The 'ephemeral_storage' module supports configuring and using local instance storage. + +use model::ephemeral_storage::Filesystem; + +use snafu::{ensure, ResultExt}; +use std::collections::HashSet; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::Path; +use std::process::Command; + +static MOUNT: &str = "/usr/bin/mount"; +static MDADM: &str = "/usr/sbin/mdadm"; +static BLKID: &str = "/usr/sbin/blkid"; +static MKFSXFS: &str = "/usr/sbin/mkfs.xfs"; +static MKFSEXT4: &str = "/usr/sbin/mkfs.ext4"; +static FINDMNT: &str = "/usr/bin/findmnt"; + +/// Name of the array (if created) and filesystem label. Selected to be 12 characters so it +/// fits within both the xfs and ext4 volume label limit. +static EPHEMERAL: &str = ".ephemeral"; + +/// initialize prepares the ephemeral storage for formatting and formats it. For multiple disks +/// preparation is the creation of a RAID0 array, for a single disk this is a no-op. The array or disk +/// is then formatted with the specified filesystem (default=xfs) if not formatted already. +pub fn initialize(fs: Option, disks: Option>) -> Result<()> { + let known_disks = ephemeral_devices()?; + let known_disks_hash = HashSet::<_>::from_iter(known_disks.iter()); + + let disks = match disks { + Some(disks) => { + // we have disks provided, so match them against the list of valid disks + for disk in &disks { + ensure!( + known_disks_hash.contains(disk), + error::InvalidParameterSnafu { + parameter: "disks", + reason: format!("unknown disk {:?}", disk), + } + ) + } + disks + } + None => { + // if there are no disks specified, and none are available we treat the init as a + // no-op to allow "ephemeral-storage init"/"ephemeral-storage bind" to work on instances + // with and without ephemeral storage + if known_disks.is_empty() { + info!("no ephemeral disks found, skipping ephemeral storage initialization"); + return Ok(()); + } + // no disks specified, so use the default + known_disks + } + }; + + ensure!( + !disks.is_empty(), + error::InvalidParameterSnafu { + parameter: "disks", + reason: "no local ephemeral disks specified", + } + ); + + info!("initializing ephemeral storage disks={:?}", disks); + // with a single disk, there is no need to create the array + let device_name = match disks.len() { + 1 => disks.first().expect("non-empty").clone(), + _ => { + let scan_output = mdadm_scan()?; + // no previously configured array found, so construct a new one + if scan_output.is_empty() { + info!("creating array named {:?} from {:?}", EPHEMERAL, disks); + mdadm_create(EPHEMERAL, disks.iter().map(|x| x.as_str()).collect())?; + } + // can't lookup the array until it's created + resolve_array_by_id()? + } + }; + + let fs = fs.unwrap_or(Filesystem::Xfs); + if !is_formatted(&device_name, &fs)? { + info!("formatting {:?} as {}", device_name, fs); + format_device(&device_name, &fs)?; + } else { + info!( + "{:?} is already formatted as {}, skipping format", + device_name, fs + ); + } + + Ok(()) +} + +/// binds the specified directories to the pre-configured array, creating those directories if +/// they do not exist. +pub fn bind(variant: &str, dirs: Vec) -> Result<()> { + // handle the no local instance storage case + if ephemeral_devices()?.is_empty() { + info!("no ephemeral disks found, skipping ephemeral storage binding"); + return Ok(()); + } + + let device_name = resolve_device_by_label()?; + let mount_point = format!("/mnt/{}", EPHEMERAL); + let mount_point = Path::new(&mount_point); + let allowed_dirs = allowed_bind_dirs(variant); + for dir in &dirs { + ensure!( + allowed_dirs.contains(dir.as_str()), + error::InvalidParameterSnafu { + parameter: dir, + reason: "specified bind directory not in allow list", + } + ) + } + std::fs::create_dir_all(mount_point).context(error::MkdirSnafu {})?; + + info!("mounting {:?} as {:?}", device_name, mount_point); + let output = Command::new(MOUNT) + .args([ + OsString::from(device_name.clone()), + OsString::from(mount_point.as_os_str()), + ]) + .output() + .context(error::ExecutionFailureSnafu { command: MOUNT })?; + + ensure!( + output.status.success(), + error::MountArrayFailureSnafu { + what: device_name, + dest: mount_point.to_string_lossy().to_string(), + output + } + ); + + for dir in &dirs { + // construct a directory name (E.g. /var/lib/kubelet => ._var_lib_kubelet) that will be + // unique between the binding targets + let mut directory_name = dir.replace('/', "_"); + directory_name.insert(0, '.'); + let mount_destination = mount_point.join(&directory_name); + + // we may run before the directories we are binding exist, so create them + std::fs::create_dir_all(dir).context(error::MkdirSnafu {})?; + std::fs::create_dir_all(&mount_destination).context(error::MkdirSnafu {})?; + + if is_mounted(dir)? { + info!("skipping bind mount of {:?}, already mounted", dir); + continue; + } + // call the equivalent of + // mount --rbind /mnt/.ephemeral/._var_lib_kubelet /var/lib/kubelet + let source_dir = OsString::from(&dir); + info!("binding {:?} to {:?}", source_dir, mount_destination); + + let output = Command::new(MOUNT) + .args([ + OsStr::new("--rbind"), + mount_destination.as_ref(), + &source_dir, + ]) + .output() + .context(error::ExecutionFailureSnafu { command: MOUNT })?; + + ensure!( + output.status.success(), + error::BindDirectoryFailureSnafu { + dir: String::from_utf8_lossy(source_dir.as_encoded_bytes()), + output, + } + ); + } + + for dir in dirs { + let source_dir = OsString::from(&dir); + info!("sharing mounts for {:?}", source_dir); + // mount --make-rshared /var/lib/kubelet + let output = Command::new(MOUNT) + .args([OsStr::new("--make-rshared"), &source_dir]) + .output() + .context(error::ExecutionFailureSnafu { command: MOUNT })?; + + ensure!( + output.status.success(), + error::ShareMountsFailureSnafu { + dir: String::from_utf8_lossy(source_dir.as_encoded_bytes()), + output + } + ); + } + + Ok(()) +} + +/// is_bound returns true if the specified path is already listed as a mount +fn is_mounted(path: &String) -> Result { + let status = Command::new(FINDMNT) + .arg(OsString::from(path)) + .status() + .context(error::FindMntFailureSnafu {})?; + Ok(status.success()) +} + +/// resolve_device_by_label resolves the by-label link for the raid array or single disk to the device name +fn resolve_device_by_label() -> Result { + canonical_name(format!("/dev/disk/by-label/{}", EPHEMERAL)) +} + +/// resolve_array_by_name resolves the by-id link for the raid array +fn resolve_array_by_id() -> Result { + canonical_name(format!("/dev/disk/by-id/md-name-{}", EPHEMERAL)) +} + +/// canonical_name will create the canonical, absolute form of a path with all intermediate +/// components normalized and symbolic links resolved +fn canonical_name(name: String) -> Result { + Ok(std::fs::canonicalize(OsString::from(name)) + .context(error::CanonicalizeFailureSnafu {})? + .to_string_lossy() + .to_string()) +} + +/// creates the array with the given name from the specified disks +fn mdadm_create>(name: T, disks: Vec) -> Result<()> { + let mut device_name = OsString::from("/dev/md/"); + device_name.push(name.as_ref()); + + let mut cmd = Command::new(MDADM); + cmd.arg("--create"); + cmd.arg("--force"); + cmd.arg("--verbose"); + cmd.arg(device_name); + cmd.arg("--level=0"); + // By default, mdadm uses a 512KB chunk size. mkfs.xfs attempts to match some of its settings to + // the array size for maximum throughput, but the max log stripe size for xfs is 256KB. We limit + // the chunk size to 256KB here so that XFS can set the same value and avoid the fallback to + // a 32 KB log stripe size. + cmd.arg("--chunk=256"); + cmd.arg("--name"); + cmd.arg(OsString::from(name.as_ref())); + cmd.arg("--raid-devices"); + cmd.arg(OsString::from(disks.len().to_string())); + for disk in disks { + cmd.arg(OsString::from(disk.as_ref())); + } + let output = cmd + .output() + .context(error::ExecutionFailureSnafu { command: MDADM })?; + ensure!( + output.status.success(), + error::CreateArrayFailureSnafu { output } + ); + Ok(()) +} + +/// ephemeral_devices returns the full path name to the block devices in /dev/disk/ephemeral +pub fn ephemeral_devices() -> Result> { + const EPHEMERAL_PATH: &str = "/dev/disk/ephemeral"; + let mut filenames = Vec::new(); + // for instances without ephemeral storage, we don't error and just return an empty vector so + // it can be handled gracefully + if fs::metadata(EPHEMERAL_PATH).is_err() { + return Ok(filenames); + } + + let entries = std::fs::read_dir(EPHEMERAL_PATH).context(error::DiscoverEphemeralSnafu { + path: String::from(EPHEMERAL_PATH), + })?; + for entry in entries { + let entry = entry.context(error::DiscoverEphemeralSnafu { + path: String::from(EPHEMERAL_PATH), + })?; + filenames.push(entry.path().into_os_string().to_string_lossy().to_string()); + } + Ok(filenames) +} + +/// allowed_bind_dirs returns a set of the directories that can be bound to ephemeral storage, which +/// varies based on the variant +pub fn allowed_bind_dirs(variant: &str) -> HashSet<&'static str> { + let mut allowed = HashSet::from(["/var/lib/containerd", "/var/lib/host-containerd"]); + if variant.contains("k8s") { + allowed.insert("/var/lib/kubelet"); + allowed.insert("/var/log/pods"); + } + if variant.contains("ecs") { + allowed.insert("/var/lib/docker"); + allowed.insert("/var/log/ecs"); + } + allowed +} + +/// scans the raid array to identify if it has been created already +fn mdadm_scan() -> Result> { + let output = Command::new(MDADM) + .args([OsStr::new("--detail"), OsStr::new("--scan")]) + .output() + .context(error::ExecutionFailureSnafu { command: MDADM })?; + ensure!( + output.status.success(), + error::ScanArrayFailureSnafu { output } + ); + Ok(output.stdout) +} + +/// is_formatted returns true if the array is already formatted with the specified filesystem +pub fn is_formatted>(device: S, format: &Filesystem) -> Result { + let mut fmt_arg = OsString::from("TYPE="); + fmt_arg.push(OsString::from(format.to_string())); + + let blkid = Command::new(BLKID) + .args([ + OsStr::new("--match-token"), + fmt_arg.as_ref(), + device.as_ref(), + ]) + .status() + .context(error::DetermineFormatFailureSnafu {})?; + + Ok(blkid.success()) +} + +/// formats the specified device with the given filesystem format +pub fn format_device>(device: S, format: &Filesystem) -> Result<()> { + let binary = match format { + Filesystem::Xfs => MKFSXFS, + Filesystem::Ext4 => MKFSEXT4, + }; + + let mut mkfs = Command::new(binary); + mkfs.arg(device.as_ref()); + // labeled, XFS has a max of 12 characters, EXT4 allows 16 + mkfs.arg("-L"); + mkfs.arg(EPHEMERAL); + + let output = mkfs + .output() + .context(error::ExecutionFailureSnafu { command: binary })?; + + ensure!( + output.status.success(), + error::FormatFilesystemFailureSnafu { output } + ); + Ok(()) +} + +pub mod error { + use snafu::Snafu; + + #[derive(Debug, Snafu)] + #[snafu(visibility(pub(super)))] + pub enum Error { + #[snafu(display("Failed to execute '{:?}': {}", command, source))] + ExecutionFailure { + command: &'static str, + source: std::io::Error, + }, + + #[snafu(display("Failed to discover ephemeral disks from {}: {}", path, source))] + DiscoverEphemeral { + source: std::io::Error, + path: String, + }, + + #[snafu(display("Failed to mount {} to {}: {}", what, dest, String::from_utf8_lossy(output.stderr.as_slice())))] + MountArrayFailure { + what: String, + dest: String, + output: std::process::Output, + }, + + #[snafu(display("Failed to create disk symlink {}", source))] + DiskSymlinkFailure { source: std::io::Error }, + + #[snafu(display("Failed to bind directory {}: {}", dir, String::from_utf8_lossy(output.stderr.as_slice())))] + BindDirectoryFailure { + dir: String, + output: std::process::Output, + }, + + #[snafu(display("Failed to share mounts for directory {} : {}", dir, String::from_utf8_lossy(output.stderr.as_slice())))] + ShareMountsFailure { + dir: String, + output: std::process::Output, + }, + + #[snafu(display("Failed to create array : {}", String::from_utf8_lossy(output.stderr.as_slice())))] + CreateArrayFailure { output: std::process::Output }, + + #[snafu(display("Failed to scan array : {}", String::from_utf8_lossy(output.stderr.as_slice())))] + ScanArrayFailure { output: std::process::Output }, + + #[snafu(display("Failed to determine filesystem format {}", source))] + DetermineFormatFailure { source: std::io::Error }, + + #[snafu(display("Failed to determine mount status {}", source))] + FindMntFailure { source: std::io::Error }, + + #[snafu(display("Failed to format filesystem : {}", String::from_utf8_lossy(output.stderr.as_slice())))] + FormatFilesystemFailure { output: std::process::Output }, + + #[snafu(display("Invalid Parameter '{}', {}", parameter, reason))] + InvalidParameter { parameter: String, reason: String }, + + #[snafu(display("Failed to create directory, {}", source))] + Mkdir { source: std::io::Error }, + + #[snafu(display("Failed to canonicalize path, {}", source))] + CanonicalizeFailure { source: std::io::Error }, + } +} + +pub type Result = std::result::Result; diff --git a/sources/api/apiserver/src/server/error.rs b/sources/api/apiserver/src/server/error.rs index 0c66af2ed..cd7819398 100644 --- a/sources/api/apiserver/src/server/error.rs +++ b/sources/api/apiserver/src/server/error.rs @@ -1,3 +1,4 @@ +use crate::server::ephemeral_storage; use actix_web::{HttpResponseBuilder, ResponseError}; use datastore::{self, deserialization, serialization}; use nix::unistd::Gid; @@ -109,6 +110,21 @@ pub enum Error { source: serde_json::Error, }, + #[snafu(display("Unable to initialize ephemeral storage: {}", source))] + EphemeralInitialize { + source: ephemeral_storage::error::Error, + }, + + #[snafu(display("Unable to bind ephemeral storage: {}", source))] + EphemeralBind { + source: ephemeral_storage::error::Error, + }, + + #[snafu(display("Unable to list ephemeral disks: {}", source))] + EphemeralListDisks { + source: ephemeral_storage::error::Error, + }, + #[snafu(display("Unable to make {} key '{}': {}", key_type, name, source))] NewKey { key_type: String, diff --git a/sources/api/apiserver/src/server/mod.rs b/sources/api/apiserver/src/server/mod.rs index fa6da4c6d..1f08ac686 100644 --- a/sources/api/apiserver/src/server/mod.rs +++ b/sources/api/apiserver/src/server/mod.rs @@ -2,6 +2,7 @@ //! server::controller module. mod controller; +mod ephemeral_storage; mod error; mod exec; @@ -15,6 +16,7 @@ use error::Result; use fs2::FileExt; use http::StatusCode; use log::info; +use model::ephemeral_storage::{Bind, Init}; use model::{ConfigurationFiles, Model, Report, Services, Settings}; use nix::unistd::{chown, Gid}; use serde::{Deserialize, Serialize}; @@ -124,7 +126,23 @@ where .route("/refresh-updates", web::post().to(refresh_updates)) .route("/prepare-update", web::post().to(prepare_update)) .route("/activate-update", web::post().to(activate_update)) - .route("/deactivate-update", web::post().to(deactivate_update)), + .route("/deactivate-update", web::post().to(deactivate_update)) + .route( + "/ephemeral-storage/init", + web::post().to(initialize_ephemeral_storage), + ) + .route( + "/ephemeral-storage/bind", + web::post().to(bind_ephemeral_storage), + ) + .route( + "/ephemeral-storage/list-disks", + web::get().to(list_ephemeral_storage_disks), + ) + .route( + "/ephemeral-storage/list-dirs", + web::get().to(list_ephemeral_storage_dirs), + ), ) .service(web::scope("/updates").route("/status", web::get().to(get_update_status))) .service(web::resource("/exec").route(web::get().to(exec::ws_exec))) @@ -648,6 +666,74 @@ async fn get_fips_report(query: web::Query>) -> Result) -> Result { + ephemeral_storage::initialize(cfg.0.filesystem, cfg.0.disks) + .context(error::EphemeralInitializeSnafu {})?; + Ok(HttpResponse::NoContent().finish()) // 204 +} +/// Bind directories to ephemeral storage (mount array, bind, and unmount) +async fn bind_ephemeral_storage(cfg: web::Json) -> Result { + let os_info = controller::get_os_info()?; + ephemeral_storage::bind(&os_info.variant_id, cfg.0.targets) + .context(error::EphemeralBindSnafu {})?; + Ok(HttpResponse::NoContent().finish()) // 204 +} + +/// Lists the known ephemeral disks that can be configured. +async fn list_ephemeral_storage_disks( + req: HttpRequest, + query: web::Query>, +) -> Result { + let disks = + ephemeral_storage::ephemeral_devices().context(error::EphemeralListDisksSnafu {})?; + + let mut text_response = String::new(); + for disk in &disks { + text_response.push_str(disk); + text_response.push('\n'); + } + list_ephemeral_response(req, query, disks, text_response).await +} + +/// Lists the known ephemeral disks that can be configured. +async fn list_ephemeral_storage_dirs( + req: HttpRequest, + query: web::Query>, +) -> Result { + let os_info = controller::get_os_info()?; + + let allowed = ephemeral_storage::allowed_bind_dirs(&os_info.variant_id); + let mut text_response = String::new(); + for dir in &allowed { + text_response.push_str(dir); + text_response.push('\n'); + } + + let allowed: Vec = allowed.iter().map(|x| String::from(*x)).collect(); + list_ephemeral_response(req, query, allowed, text_response).await +} + +// Responds to a list request with the text or JSON resposne depending on the query format. +async fn list_ephemeral_response( + req: HttpRequest, + query: web::Query>, + items: Vec, + text_response: String, +) -> Result { + match query + .get("format") + .unwrap_or(&String::from("text")) + .as_str() + { + "json" => Ok(EphemeralListResponse(items).respond_to(&req)), + "text" => Ok(HttpResponse::Ok() + .content_type("application/text") + .body(text_response)), + _ => Ok(HttpResponse::BadRequest().body("unsupported format")), + } +} + // =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= // Helpers for handler methods called by the router @@ -782,6 +868,9 @@ impl ResponseError for error::Error { Deserialization { .. } => StatusCode::INTERNAL_SERVER_ERROR, DataStoreSerialization { .. } => StatusCode::INTERNAL_SERVER_ERROR, CommandSerialization { .. } => StatusCode::INTERNAL_SERVER_ERROR, + EphemeralBind { .. } => StatusCode::INTERNAL_SERVER_ERROR, + EphemeralInitialize { .. } => StatusCode::INTERNAL_SERVER_ERROR, + EphemeralListDisks { .. } => StatusCode::INTERNAL_SERVER_ERROR, InvalidMetadata { .. } => StatusCode::INTERNAL_SERVER_ERROR, ConfigApplierFork { .. } => StatusCode::INTERNAL_SERVER_ERROR, ConfigApplierStart { .. } => StatusCode::INTERNAL_SERVER_ERROR, @@ -895,3 +984,6 @@ impl_responder_for!(TransactionListResponse, self, self.0); struct ReportListResponse(Vec); impl_responder_for!(ReportListResponse, self, self.0); + +struct EphemeralListResponse(Vec); +impl_responder_for!(EphemeralListResponse, self, self.0); diff --git a/sources/api/openapi.yaml b/sources/api/openapi.yaml index 67c0b4306..0b806fd9d 100644 --- a/sources/api/openapi.yaml +++ b/sources/api/openapi.yaml @@ -198,7 +198,18 @@ components: $ref: '#/components/schemas/ConfigurationFiles' os: $ref: '#/components/schemas/BottlerocketRelease' - + EphemeralStorageInit: + type: object + properties: + filesystem: + type: string + disks: + type: array + EphemeralStorageBind: + type: object + properties: + targets: + type: array paths: /: get: @@ -217,7 +228,7 @@ paths: content: application/json: schema: - $ref: "Model" + $ref: "#/components/schemas/Model" 500: description: "Server error" @@ -249,7 +260,7 @@ paths: content: application/json: schema: - $ref: "Settings" + $ref: "#/components/schemas/Settings" 500: description: "Server error" patch: @@ -267,7 +278,7 @@ paths: content: application/json: schema: - $ref: "Settings" + $ref: "#/components/schemas/Settings" responses: 204: description: "Settings successfully staged for update" @@ -291,7 +302,7 @@ paths: content: application/json: schema: - $ref: "SettingsKeyPair" + $ref: "#/components/schemas/SettingsKeyPair" responses: 204: description: "Settings successfully staged for update" @@ -316,7 +327,7 @@ paths: content: application/json: schema: - $ref: "Settings" + $ref: #/components/schemas/Settings" 500: description: "Server error" delete: @@ -530,7 +541,7 @@ paths: content: application/json: schema: - $ref: "Services" + $ref: "#/components/schemas/Services" 500: description: "Server error" @@ -562,7 +573,7 @@ paths: content: application/json: schema: - $ref: "ConfigurationFiles" + $ref: "#/components/schemas/ConfigurationFiles" 500: description: "Server error" @@ -646,7 +657,7 @@ paths: content: application/json: schema: - $ref: "UpdateStatus" + $ref: "#/components/schemas/UpdateStatus" 500: description: "Server error" 423: @@ -674,7 +685,7 @@ paths: schema: type: array items: - $ref: "Report" + $ref: "#/components/schemas/Report" 500: description: "Server error" @@ -735,3 +746,98 @@ paths: description: "Unprocessable request" 500: description: "Server error" + /ephemeral-storage/init: + post: + summary: "Initialize ephemeral storage" + operationId: "init" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EphemeralStorageInit" + responses: + 200: + description: "Successful request" + content: + application/json: + schema: + type: string + 400: + description: "Bad request input" + 422: + description: "Unprocessable request" + 500: + description: "Server error" + /ephemeral-storage/bind: + post: + summary: "Bind directories to previously initialized ephemeral storage" + operationId: "bind" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EphemeralStorageBind" + responses: + 200: + description: "Successful request" + content: + application/json: + schema: + type: string + 400: + description: "Bad request input" + 422: + description: "Unprocessable request" + 500: + description: "Server error" + /ephemeral-storage/list-disks: + get: + summary: "List the discovered ephemeral disks that can be initialized" + operationId: "list-disks" + parameters: + - in: query + name: format + description: "Format of the disk listing (text or json). Default format is text." + schema: + type: string + required: false + responses: + 200: + description: "Successful request" + content: + application/json: + schema: + type: string + 400: + description: "Bad request input" + 422: + description: "Unprocessable request" + 500: + description: "Server error" + /ephemeral-storage/list-dirs: + get: + summary: "List the directories that can be bound to ephemeral storage" + operationId: "list-dirs" + parameters: + - in: query + name: format + description: "Format of the directory listing (text or json). Default format is text." + schema: + type: string + required: false + responses: + 200: + description: "Successful request" + content: + application/json: + schema: + type: string + 400: + description: "Bad request input" + 422: + description: "Unprocessable request" + 500: + description: "Server error" + diff --git a/sources/models/src/ephemeral_storage.rs b/sources/models/src/ephemeral_storage.rs new file mode 100644 index 000000000..2fa87a402 --- /dev/null +++ b/sources/models/src/ephemeral_storage.rs @@ -0,0 +1,32 @@ +//! The 'ephemeral_storage' module holds types used to communicate between client and server for +//! 'apiclient ephemeral-storage'. +use serde::{Deserialize, Serialize}; +use std::fmt::{Display, Formatter}; + +/// Supported filesystems for ephemeral storage +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Filesystem { + Xfs, + Ext4, +} +impl Display for Filesystem { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Filesystem::Xfs => f.write_str("xfs"), + Filesystem::Ext4 => f.write_str("ext4"), + } + } +} + +/// Initialize ephemeral storage +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Init { + pub filesystem: Option, + pub disks: Option>, +} + +/// Bind directories to configured ephemeral storage +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Bind { + pub targets: Vec, +} diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs index e3015d3f9..fa76b8c46 100644 --- a/sources/models/src/lib.rs +++ b/sources/models/src/lib.rs @@ -21,6 +21,9 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and // Types used to communicate between client and server for 'apiclient exec'. pub mod exec; +// Types used to communicate between client and server for 'apiclient ephemeral-storage'. +pub mod ephemeral_storage; + use bottlerocket_release::BottlerocketRelease; use bottlerocket_settings_models::model_derive::model; use bottlerocket_settings_plugin::BottlerocketSettings;