diff --git a/sources/api/apiclient/src/ephemeral_storage.rs b/sources/api/apiclient/src/ephemeral_storage.rs
new file mode 100644
index 000000000..5b596f944
--- /dev/null
+++ b/sources/api/apiclient/src/ephemeral_storage.rs
@@ -0,0 +1,87 @@
+use model::ephemeral_storage::{Bind, Filesystem, Init};
+use snafu::ResultExt;
+use std::path::Path;
+
+/// Requests ephemeral storage initialization through the API
+pub async fn initialize
(
+ socket_path: P,
+ filesystem: Option,
+ disks: Option>,
+) -> Result<()>
+where
+ P: AsRef,
+{
+ let uri = "/actions/ephemeral-storage/init";
+ let opts =
+ serde_json::to_string(&Init { filesystem, disks }).context(error::JsonSerializeSnafu {})?;
+ let method = "POST";
+ let (_status, _body) = crate::raw_request(&socket_path, &uri, method, Some(opts))
+ .await
+ .context(error::RequestSnafu { uri, method })?;
+ Ok(())
+}
+
+/// Requests binding of directories to configured ephemeral storage through the API
+pub async fn bind(socket_path: P, targets: Vec) -> Result<()>
+where
+ P: AsRef,
+{
+ let uri = "/actions/ephemeral-storage/bind";
+ let opts = serde_json::to_string(&Bind { targets }).context(error::JsonSerializeSnafu {})?;
+ let method = "POST";
+ let (_status, _body) = crate::raw_request(&socket_path, &uri, method, Some(opts))
+ .await
+ .context(error::RequestSnafu { uri, method })?;
+ Ok(())
+}
+
+/// Lists the ephemeral disks available for configuration
+pub async fn list_disks(socket_path: P, format: Option) -> Result
+where
+ P: AsRef,
+{
+ list(socket_path, "list-disks", format).await
+}
+
+/// Lists the ephemeral disks available for configuration
+pub async fn list_dirs(socket_path: P, format: Option) -> Result
+where
+ P: AsRef,
+{
+ list(socket_path, "list-dirs", format).await
+}
+async fn list(socket_path: P, item: &str, format: Option) -> Result
+where
+ P: AsRef,
+{
+ let mut query = Vec::new();
+ if let Some(query_format) = format {
+ query.push(format!("format={}", query_format));
+ }
+
+ let uri = format!("/actions/ephemeral-storage/{}?{}", item, query.join("&"));
+ let method = "GET";
+ let (_status, body) = crate::raw_request(&socket_path, &uri, method, None)
+ .await
+ .context(error::RequestSnafu { uri, method })?;
+ Ok(body)
+}
+mod error {
+ use snafu::Snafu;
+
+ #[derive(Debug, Snafu)]
+ #[snafu(visibility(pub(super)))]
+ pub enum Error {
+ #[snafu(display("Failed {} request to '{}': {}", method, uri, source))]
+ Request {
+ method: String,
+ uri: String,
+ #[snafu(source(from(crate::Error, Box::new)))]
+ source: Box,
+ },
+ #[snafu(display("Failed to serialize parameters"))]
+ JsonSerialize { source: serde_json::Error },
+ }
+}
+pub use error::Error;
+pub type Result = std::result::Result;
diff --git a/sources/api/apiclient/src/lib.rs b/sources/api/apiclient/src/lib.rs
index 8fcbf9697..c322c2f7c 100644
--- a/sources/api/apiclient/src/lib.rs
+++ b/sources/api/apiclient/src/lib.rs
@@ -20,6 +20,7 @@ use snafu::{ensure, ResultExt};
use std::{fmt, fmt::Display, path::Path};
pub mod apply;
+pub mod ephemeral_storage;
pub mod exec;
pub mod get;
pub mod reboot;
diff --git a/sources/api/apiclient/src/main.rs b/sources/api/apiclient/src/main.rs
index 44b30db0e..b7579c56f 100644
--- a/sources/api/apiclient/src/main.rs
+++ b/sources/api/apiclient/src/main.rs
@@ -6,8 +6,9 @@
// library calls based on the given flags, etc.) The library modules contain the code for talking
// to the API, which is intended to be reusable by other crates.
-use apiclient::{apply, exec, get, reboot, report, set, update, SettingsInput};
+use apiclient::{apply, ephemeral_storage, exec, get, reboot, report, set, update, SettingsInput};
use log::{info, log_enabled, trace, warn};
+use model::ephemeral_storage::Filesystem;
use serde::{Deserialize, Serialize};
use simplelog::{
ColorChoice, ConfigBuilder as LogConfigBuilder, LevelFilter, TermLogger, TerminalMode,
@@ -15,8 +16,10 @@ use simplelog::{
use snafu::ResultExt;
use std::env;
use std::ffi::OsString;
+use std::iter::Peekable;
use std::process;
use std::str::FromStr;
+use std::vec::IntoIter;
use unindent::unindent;
const DEFAULT_METHOD: &str = "GET";
@@ -48,6 +51,7 @@ enum Subcommand {
Set(SetArgs),
Update(UpdateSubcommand),
Report(ReportSubcommand),
+ EphemeralStorage(EphemeralStorageSubcommand),
}
/// Stores user-supplied arguments for the 'apply' subcommand.
@@ -140,6 +144,33 @@ struct UpdateApplyArgs {
#[derive(Debug)]
struct UpdateCancelArgs {}
+/// Stores the 'ephemeral-storage' subcommand specified by the user.
+#[derive(Debug)]
+enum EphemeralStorageSubcommand {
+ Init(EphemeralStorageInitArgs),
+ Bind(EphemeralStorageBindArgs),
+ ListDisks(EphemeralStorageFormatArgs),
+ ListDirs(EphemeralStorageFormatArgs),
+}
+
+/// Stores user-supplied arguments for the 'ephemeral-storage init' subcommand.
+#[derive(Debug)]
+struct EphemeralStorageInitArgs {
+ disks: Option>,
+ filesystem: Option,
+}
+
+/// Stores user-supplied arguments for the 'ephemeral-storage bind' subcommand.
+#[derive(Debug)]
+struct EphemeralStorageBindArgs {
+ targets: Vec,
+}
+/// Stores user-supplied arguments for the 'ephemeral-storage list-disks/list-dirs' subcommand.
+#[derive(Debug)]
+struct EphemeralStorageFormatArgs {
+ format: Option,
+}
+
/// Informs the user about proper usage of the program and exits.
fn usage() -> ! {
let msg = &format!(
@@ -166,6 +197,12 @@ fn usage() -> ! {
report cis Retrieve a Bottlerocket CIS benchmark compliance report.
report cis-k8s Retrieve a Kubernetes CIS benchmark compliance report.
report fips Retrieve a FIPS Security Policy compliance report.
+ ephemeral-storage init Initialize ephemeral storage
+ ephemeral-storage bind Bind directories to previously initialized ephemeral storage.
+ ephemeral-storage list-disks
+ List the discovered ephemeral disks that can be initialized.
+ ephemeral-storage list-dirs
+ List the directories that can be bound to ephemeral storage.
raw options:
-u, --uri URI Required; URI to request from the server, e.g. /tx
@@ -223,7 +260,28 @@ fn usage() -> ! {
report cis-k8s options:
-f, --format Format of the CIS report (text or json). Default format is text.
- -l, --level CIS compliance level to report on (1 or 2). Default is 1."#,
+ -l, --level CIS compliance level to report on (1 or 2). Default is 1.
+
+ ephemeral-storage init options:
+ -t, --filesystem Filesystem to initialize the array as (ext4 or xfs). Default is
+ xfs. If a single disk is provided, it is mounted directly without
+ constructing an array. If no ephemeral disks are found, this
+ operation does nothing.
+ --disks DISK [DISK ...] Local disks to configure for storage. Default is all ephemeral
+ disks.
+
+ ephemeral-storage bind options:
+ --dirs DIR [DIR ...] Directories to bind to configured ephemeral storage
+ (e.g. /var/lib/containerd). If no ephemeral disks are found
+ this operation does nothing.
+
+ ephemeral-storage list-disks options:
+ -f, --format Format of the disk listing (text or json). Default format is text.
+
+ ephemeral-storage list-dirs options:
+ -f, --format Format of the directory listing (text or json). Default format is text.
+
+ "#,
socket = constants::API_SOCKET,
method = DEFAULT_METHOD,
);
@@ -272,6 +330,7 @@ fn parse_args(args: env::Args) -> (Args, Subcommand) {
// Subcommands
"raw" | "apply" | "exec" | "get" | "reboot" | "report" | "set" | "update"
+ | "ephemeral-storage"
if subcommand.is_none() && !arg.starts_with('-') =>
{
subcommand = Some(arg)
@@ -292,6 +351,7 @@ fn parse_args(args: env::Args) -> (Args, Subcommand) {
Some("report") => (global_args, parse_report_args(subcommand_args)),
Some("set") => (global_args, parse_set_args(subcommand_args)),
Some("update") => (global_args, parse_update_args(subcommand_args)),
+ Some("ephemeral-storage") => (global_args, parse_ephemeral_storage_args(subcommand_args)),
_ => usage_msg("Missing or unknown subcommand"),
}
}
@@ -668,6 +728,140 @@ fn parse_fips_arguments(args: Vec) -> FipsReportArgs {
FipsReportArgs { format }
}
+/// Parse the desired subcommand of 'ephemeral-storage'
+fn parse_ephemeral_storage_args(args: Vec) -> Subcommand {
+ let mut subcommand = None;
+ let mut subcommand_args = Vec::new();
+
+ for arg in args.into_iter() {
+ match arg.as_ref() {
+ // Subcommands
+ "init" | "bind" | "list-disks" | "list-dirs"
+ if subcommand.is_none() && !arg.starts_with('-') =>
+ {
+ subcommand = Some(arg)
+ }
+
+ // Other arguments are passed to the subcommand parser
+ _ => subcommand_args.push(arg),
+ }
+ }
+
+ let cmd = match subcommand.as_deref() {
+ Some("init") => parse_ephemeral_storage_init_args(subcommand_args),
+ Some("bind") => parse_ephemeral_storage_bind_args(subcommand_args),
+ Some("list-disks") => EphemeralStorageSubcommand::ListDisks(
+ parse_ephemeral_storage_list_format_args(subcommand_args),
+ ),
+ Some("list-dirs") => EphemeralStorageSubcommand::ListDirs(
+ parse_ephemeral_storage_list_format_args(subcommand_args),
+ ),
+ _ => usage_msg("Missing or unknown subcommand for 'ephemeral-storage'"),
+ };
+ Subcommand::EphemeralStorage(cmd)
+}
+
+/// Parses arguments for the 'init' ephemeral-storage subcommand.
+fn parse_ephemeral_storage_init_args(args: Vec) -> EphemeralStorageSubcommand {
+ let mut disks: Option> = None;
+ let mut filesystem = None;
+ let mut iter = args.into_iter().peekable();
+ while let Some(arg) = iter.next() {
+ match arg.as_ref() {
+ "-t" | "--filesystem" => {
+ match iter
+ .next()
+ .unwrap_or_else(|| usage_msg("Did not give argument to -t | --filesystem"))
+ .as_str()
+ {
+ "ext4" => filesystem = Some(Filesystem::Ext4),
+ "xfs" => filesystem = Some(Filesystem::Xfs),
+ _ => usage_msg("Unsupported filesystem type"),
+ }
+ }
+ "--disks" => {
+ let mut names = collect_non_args(&mut iter);
+ if names.is_empty() {
+ usage_msg("Did not give argument to --disks")
+ }
+ if let Some(existing) = &mut disks {
+ existing.append(&mut names);
+ } else {
+ disks = Some(names);
+ }
+ }
+ x => usage_msg(format!("Unknown argument '{}'", x)),
+ }
+ }
+ EphemeralStorageSubcommand::Init(EphemeralStorageInitArgs { disks, filesystem })
+}
+
+/// Parses arguments for the 'bind' ephemeral-storage subcommand.
+fn parse_ephemeral_storage_bind_args(args: Vec) -> EphemeralStorageSubcommand {
+ if args.is_empty() {
+ usage_msg("Did not give arguments to bind")
+ }
+ let mut targets = Vec::new();
+ let mut iter = args.into_iter().peekable();
+ while let Some(arg) = iter.next() {
+ match arg.as_ref() {
+ "--dirs" => {
+ targets.append(&mut collect_non_args(&mut iter));
+ if targets.is_empty() {
+ usage_msg("Did not give argument to --dirs")
+ }
+ }
+ x => usage_msg(format!("Unknown argument '{}'", x)),
+ }
+ }
+ EphemeralStorageSubcommand::Bind(EphemeralStorageBindArgs { targets })
+}
+
+/// Parses arguments for the 'list-disks' and 'list-dirs' ephemeral-storage subcommand.
+fn parse_ephemeral_storage_list_format_args(args: Vec) -> EphemeralStorageFormatArgs {
+ let mut format = None;
+
+ let mut iter = args.into_iter();
+ while let Some(arg) = iter.next() {
+ match arg.as_ref() {
+ "-f" | "--format" => {
+ format = Some(
+ iter.next()
+ .unwrap_or_else(|| usage_msg("Did not give argument to -f | --format")),
+ )
+ }
+ x => usage_msg(format!("Unknown argument '{}'", x)),
+ }
+ }
+ EphemeralStorageFormatArgs { format }
+}
+
+/// collects non-argument parameters (those not starting with a '-') up until the next
+/// argument is seen
+fn collect_non_args(iter: &mut Peekable>) -> Vec {
+ let mut result = Vec::new();
+ loop {
+ // look at the following argument and stop accepting disk names
+ // once we reach the end of arguments, or find the beginning of
+ // the next argument
+ match iter.peek() {
+ None => {
+ break;
+ }
+ Some(peeked) => {
+ if peeked.is_empty() || peeked.starts_with('-') {
+ break;
+ }
+ }
+ }
+ let next = iter
+ .next()
+ .unwrap_or_else(|| usage_msg("Expected non-empty argument"));
+ result.push(next);
+ }
+ result
+}
+
// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
// Helpers
@@ -859,6 +1053,39 @@ async fn run() -> Result<()> {
}
}
},
+
+ Subcommand::EphemeralStorage(subcommand) => match subcommand {
+ EphemeralStorageSubcommand::Init(cfg_args) => {
+ ephemeral_storage::initialize(
+ &args.socket_path,
+ cfg_args.filesystem,
+ cfg_args.disks,
+ )
+ .await
+ .context(error::EphemeralStorageSnafu)?;
+ }
+ EphemeralStorageSubcommand::Bind(bind_args) => {
+ ephemeral_storage::bind(&args.socket_path, bind_args.targets)
+ .await
+ .context(error::EphemeralStorageSnafu)?;
+ }
+ EphemeralStorageSubcommand::ListDisks(bind_args) => {
+ let body = ephemeral_storage::list_disks(&args.socket_path, bind_args.format)
+ .await
+ .context(error::EphemeralStorageSnafu)?;
+ if !body.is_empty() {
+ print!("{}", body);
+ }
+ }
+ EphemeralStorageSubcommand::ListDirs(bind_args) => {
+ let body = ephemeral_storage::list_dirs(&args.socket_path, bind_args.format)
+ .await
+ .context(error::EphemeralStorageSnafu)?;
+ if !body.is_empty() {
+ print!("{}", body);
+ }
+ }
+ },
}
Ok(())
@@ -876,7 +1103,7 @@ async fn main() {
}
mod error {
- use apiclient::{apply, exec, get, reboot, report, set, update};
+ use apiclient::{apply, ephemeral_storage, exec, get, reboot, report, set, update};
use snafu::Snafu;
#[derive(Debug, Snafu)]
@@ -922,6 +1149,9 @@ mod error {
#[snafu(display("Failed to check for updates: {}", source))]
UpdateCheck { source: update::Error },
+
+ #[snafu(display("Failed to initialize ephemeral storage: {}", source))]
+ EphemeralStorage { source: ephemeral_storage::Error },
}
}
type Result = std::result::Result;
diff --git a/sources/api/apiserver/src/server/ephemeral_storage.rs b/sources/api/apiserver/src/server/ephemeral_storage.rs
new file mode 100644
index 000000000..740f887c9
--- /dev/null
+++ b/sources/api/apiserver/src/server/ephemeral_storage.rs
@@ -0,0 +1,414 @@
+//! The 'ephemeral_storage' module supports configuring and using local instance storage.
+
+use model::ephemeral_storage::Filesystem;
+
+use snafu::{ensure, ResultExt};
+use std::collections::HashSet;
+use std::ffi::{OsStr, OsString};
+use std::fs;
+use std::path::Path;
+use std::process::Command;
+
+static MOUNT: &str = "/usr/bin/mount";
+static MDADM: &str = "/usr/sbin/mdadm";
+static BLKID: &str = "/usr/sbin/blkid";
+static MKFSXFS: &str = "/usr/sbin/mkfs.xfs";
+static MKFSEXT4: &str = "/usr/sbin/mkfs.ext4";
+static FINDMNT: &str = "/usr/bin/findmnt";
+
+/// Name of the array (if created) and filesystem label. Selected to be 12 characters so it
+/// fits within both the xfs and ext4 volume label limit.
+static EPHEMERAL: &str = ".ephemeral";
+
+/// initialize prepares the ephemeral storage for formatting and formats it. For multiple disks
+/// preparation is the creation of a RAID0 array, for a single disk this is a no-op. The array or disk
+/// is then formatted with the specified filesystem (default=xfs) if not formatted already.
+pub fn initialize(fs: Option, disks: Option>) -> Result<()> {
+ let known_disks = ephemeral_devices()?;
+ let known_disks_hash = HashSet::<_>::from_iter(known_disks.iter());
+
+ let disks = match disks {
+ Some(disks) => {
+ // we have disks provided, so match them against the list of valid disks
+ for disk in &disks {
+ ensure!(
+ known_disks_hash.contains(disk),
+ error::InvalidParameterSnafu {
+ parameter: "disks",
+ reason: format!("unknown disk {:?}", disk),
+ }
+ )
+ }
+ disks
+ }
+ None => {
+ // if there are no disks specified, and none are available we treat the init as a
+ // no-op to allow "ephemeral-storage init"/"ephemeral-storage bind" to work on instances
+ // with and without ephemeral storage
+ if known_disks.is_empty() {
+ info!("no ephemeral disks found, skipping ephemeral storage initialization");
+ return Ok(());
+ }
+ // no disks specified, so use the default
+ known_disks
+ }
+ };
+
+ ensure!(
+ !disks.is_empty(),
+ error::InvalidParameterSnafu {
+ parameter: "disks",
+ reason: "no local ephemeral disks specified",
+ }
+ );
+
+ info!("initializing ephemeral storage disks={:?}", disks);
+ // with a single disk, there is no need to create the array
+ let device_name = match disks.len() {
+ 1 => disks.first().expect("non-empty").clone(),
+ _ => {
+ let scan_output = mdadm_scan()?;
+ // no previously configured array found, so construct a new one
+ if scan_output.is_empty() {
+ info!("creating array named {:?} from {:?}", EPHEMERAL, disks);
+ mdadm_create(EPHEMERAL, disks.iter().map(|x| x.as_str()).collect())?;
+ }
+ // can't lookup the array until it's created
+ resolve_array_by_id()?
+ }
+ };
+
+ let fs = fs.unwrap_or(Filesystem::Xfs);
+ if !is_formatted(&device_name, &fs)? {
+ info!("formatting {:?} as {}", device_name, fs);
+ format_device(&device_name, &fs)?;
+ } else {
+ info!(
+ "{:?} is already formatted as {}, skipping format",
+ device_name, fs
+ );
+ }
+
+ Ok(())
+}
+
+/// binds the specified directories to the pre-configured array, creating those directories if
+/// they do not exist.
+pub fn bind(variant: &str, dirs: Vec) -> Result<()> {
+ // handle the no local instance storage case
+ if ephemeral_devices()?.is_empty() {
+ info!("no ephemeral disks found, skipping ephemeral storage binding");
+ return Ok(());
+ }
+
+ let device_name = resolve_device_by_label()?;
+ let mount_point = format!("/mnt/{}", EPHEMERAL);
+ let mount_point = Path::new(&mount_point);
+ let allowed_dirs = allowed_bind_dirs(variant);
+ for dir in &dirs {
+ ensure!(
+ allowed_dirs.contains(dir.as_str()),
+ error::InvalidParameterSnafu {
+ parameter: dir,
+ reason: "specified bind directory not in allow list",
+ }
+ )
+ }
+ std::fs::create_dir_all(mount_point).context(error::MkdirSnafu {})?;
+
+ info!("mounting {:?} as {:?}", device_name, mount_point);
+ let output = Command::new(MOUNT)
+ .args([
+ OsString::from(device_name.clone()),
+ OsString::from(mount_point.as_os_str()),
+ ])
+ .output()
+ .context(error::ExecutionFailureSnafu { command: MOUNT })?;
+
+ ensure!(
+ output.status.success(),
+ error::MountArrayFailureSnafu {
+ what: device_name,
+ dest: mount_point.to_string_lossy().to_string(),
+ output
+ }
+ );
+
+ for dir in &dirs {
+ // construct a directory name (E.g. /var/lib/kubelet => ._var_lib_kubelet) that will be
+ // unique between the binding targets
+ let mut directory_name = dir.replace('/', "_");
+ directory_name.insert(0, '.');
+ let mount_destination = mount_point.join(&directory_name);
+
+ // we may run before the directories we are binding exist, so create them
+ std::fs::create_dir_all(dir).context(error::MkdirSnafu {})?;
+ std::fs::create_dir_all(&mount_destination).context(error::MkdirSnafu {})?;
+
+ if is_mounted(dir)? {
+ info!("skipping bind mount of {:?}, already mounted", dir);
+ continue;
+ }
+ // call the equivalent of
+ // mount --rbind /mnt/.ephemeral/._var_lib_kubelet /var/lib/kubelet
+ let source_dir = OsString::from(&dir);
+ info!("binding {:?} to {:?}", source_dir, mount_destination);
+
+ let output = Command::new(MOUNT)
+ .args([
+ OsStr::new("--rbind"),
+ mount_destination.as_ref(),
+ &source_dir,
+ ])
+ .output()
+ .context(error::ExecutionFailureSnafu { command: MOUNT })?;
+
+ ensure!(
+ output.status.success(),
+ error::BindDirectoryFailureSnafu {
+ dir: String::from_utf8_lossy(source_dir.as_encoded_bytes()),
+ output,
+ }
+ );
+ }
+
+ for dir in dirs {
+ let source_dir = OsString::from(&dir);
+ info!("sharing mounts for {:?}", source_dir);
+ // mount --make-rshared /var/lib/kubelet
+ let output = Command::new(MOUNT)
+ .args([OsStr::new("--make-rshared"), &source_dir])
+ .output()
+ .context(error::ExecutionFailureSnafu { command: MOUNT })?;
+
+ ensure!(
+ output.status.success(),
+ error::ShareMountsFailureSnafu {
+ dir: String::from_utf8_lossy(source_dir.as_encoded_bytes()),
+ output
+ }
+ );
+ }
+
+ Ok(())
+}
+
+/// is_bound returns true if the specified path is already listed as a mount
+fn is_mounted(path: &String) -> Result {
+ let status = Command::new(FINDMNT)
+ .arg(OsString::from(path))
+ .status()
+ .context(error::FindMntFailureSnafu {})?;
+ Ok(status.success())
+}
+
+/// resolve_device_by_label resolves the by-label link for the raid array or single disk to the device name
+fn resolve_device_by_label() -> Result {
+ canonical_name(format!("/dev/disk/by-label/{}", EPHEMERAL))
+}
+
+/// resolve_array_by_name resolves the by-id link for the raid array
+fn resolve_array_by_id() -> Result {
+ canonical_name(format!("/dev/disk/by-id/md-name-{}", EPHEMERAL))
+}
+
+/// canonical_name will create the canonical, absolute form of a path with all intermediate
+/// components normalized and symbolic links resolved
+fn canonical_name(name: String) -> Result {
+ Ok(std::fs::canonicalize(OsString::from(name))
+ .context(error::CanonicalizeFailureSnafu {})?
+ .to_string_lossy()
+ .to_string())
+}
+
+/// creates the array with the given name from the specified disks
+fn mdadm_create>(name: T, disks: Vec) -> Result<()> {
+ let mut device_name = OsString::from("/dev/md/");
+ device_name.push(name.as_ref());
+
+ let mut cmd = Command::new(MDADM);
+ cmd.arg("--create");
+ cmd.arg("--force");
+ cmd.arg("--verbose");
+ cmd.arg(device_name);
+ cmd.arg("--level=0");
+ // By default, mdadm uses a 512KB chunk size. mkfs.xfs attempts to match some of its settings to
+ // the array size for maximum throughput, but the max log stripe size for xfs is 256KB. We limit
+ // the chunk size to 256KB here so that XFS can set the same value and avoid the fallback to
+ // a 32 KB log stripe size.
+ cmd.arg("--chunk=256");
+ cmd.arg("--name");
+ cmd.arg(OsString::from(name.as_ref()));
+ cmd.arg("--raid-devices");
+ cmd.arg(OsString::from(disks.len().to_string()));
+ for disk in disks {
+ cmd.arg(OsString::from(disk.as_ref()));
+ }
+ let output = cmd
+ .output()
+ .context(error::ExecutionFailureSnafu { command: MDADM })?;
+ ensure!(
+ output.status.success(),
+ error::CreateArrayFailureSnafu { output }
+ );
+ Ok(())
+}
+
+/// ephemeral_devices returns the full path name to the block devices in /dev/disk/ephemeral
+pub fn ephemeral_devices() -> Result> {
+ const EPHEMERAL_PATH: &str = "/dev/disk/ephemeral";
+ let mut filenames = Vec::new();
+ // for instances without ephemeral storage, we don't error and just return an empty vector so
+ // it can be handled gracefully
+ if fs::metadata(EPHEMERAL_PATH).is_err() {
+ return Ok(filenames);
+ }
+
+ let entries = std::fs::read_dir(EPHEMERAL_PATH).context(error::DiscoverEphemeralSnafu {
+ path: String::from(EPHEMERAL_PATH),
+ })?;
+ for entry in entries {
+ let entry = entry.context(error::DiscoverEphemeralSnafu {
+ path: String::from(EPHEMERAL_PATH),
+ })?;
+ filenames.push(entry.path().into_os_string().to_string_lossy().to_string());
+ }
+ Ok(filenames)
+}
+
+/// allowed_bind_dirs returns a set of the directories that can be bound to ephemeral storage, which
+/// varies based on the variant
+pub fn allowed_bind_dirs(variant: &str) -> HashSet<&'static str> {
+ let mut allowed = HashSet::from(["/var/lib/containerd", "/var/lib/host-containerd"]);
+ if variant.contains("k8s") {
+ allowed.insert("/var/lib/kubelet");
+ allowed.insert("/var/log/pods");
+ }
+ if variant.contains("ecs") {
+ allowed.insert("/var/lib/docker");
+ allowed.insert("/var/log/ecs");
+ }
+ allowed
+}
+
+/// scans the raid array to identify if it has been created already
+fn mdadm_scan() -> Result> {
+ let output = Command::new(MDADM)
+ .args([OsStr::new("--detail"), OsStr::new("--scan")])
+ .output()
+ .context(error::ExecutionFailureSnafu { command: MDADM })?;
+ ensure!(
+ output.status.success(),
+ error::ScanArrayFailureSnafu { output }
+ );
+ Ok(output.stdout)
+}
+
+/// is_formatted returns true if the array is already formatted with the specified filesystem
+pub fn is_formatted>(device: S, format: &Filesystem) -> Result {
+ let mut fmt_arg = OsString::from("TYPE=");
+ fmt_arg.push(OsString::from(format.to_string()));
+
+ let blkid = Command::new(BLKID)
+ .args([
+ OsStr::new("--match-token"),
+ fmt_arg.as_ref(),
+ device.as_ref(),
+ ])
+ .status()
+ .context(error::DetermineFormatFailureSnafu {})?;
+
+ Ok(blkid.success())
+}
+
+/// formats the specified device with the given filesystem format
+pub fn format_device>(device: S, format: &Filesystem) -> Result<()> {
+ let binary = match format {
+ Filesystem::Xfs => MKFSXFS,
+ Filesystem::Ext4 => MKFSEXT4,
+ };
+
+ let mut mkfs = Command::new(binary);
+ mkfs.arg(device.as_ref());
+ // labeled, XFS has a max of 12 characters, EXT4 allows 16
+ mkfs.arg("-L");
+ mkfs.arg(EPHEMERAL);
+
+ let output = mkfs
+ .output()
+ .context(error::ExecutionFailureSnafu { command: binary })?;
+
+ ensure!(
+ output.status.success(),
+ error::FormatFilesystemFailureSnafu { output }
+ );
+ Ok(())
+}
+
+pub mod error {
+ use snafu::Snafu;
+
+ #[derive(Debug, Snafu)]
+ #[snafu(visibility(pub(super)))]
+ pub enum Error {
+ #[snafu(display("Failed to execute '{:?}': {}", command, source))]
+ ExecutionFailure {
+ command: &'static str,
+ source: std::io::Error,
+ },
+
+ #[snafu(display("Failed to discover ephemeral disks from {}: {}", path, source))]
+ DiscoverEphemeral {
+ source: std::io::Error,
+ path: String,
+ },
+
+ #[snafu(display("Failed to mount {} to {}: {}", what, dest, String::from_utf8_lossy(output.stderr.as_slice())))]
+ MountArrayFailure {
+ what: String,
+ dest: String,
+ output: std::process::Output,
+ },
+
+ #[snafu(display("Failed to create disk symlink {}", source))]
+ DiskSymlinkFailure { source: std::io::Error },
+
+ #[snafu(display("Failed to bind directory {}: {}", dir, String::from_utf8_lossy(output.stderr.as_slice())))]
+ BindDirectoryFailure {
+ dir: String,
+ output: std::process::Output,
+ },
+
+ #[snafu(display("Failed to share mounts for directory {} : {}", dir, String::from_utf8_lossy(output.stderr.as_slice())))]
+ ShareMountsFailure {
+ dir: String,
+ output: std::process::Output,
+ },
+
+ #[snafu(display("Failed to create array : {}", String::from_utf8_lossy(output.stderr.as_slice())))]
+ CreateArrayFailure { output: std::process::Output },
+
+ #[snafu(display("Failed to scan array : {}", String::from_utf8_lossy(output.stderr.as_slice())))]
+ ScanArrayFailure { output: std::process::Output },
+
+ #[snafu(display("Failed to determine filesystem format {}", source))]
+ DetermineFormatFailure { source: std::io::Error },
+
+ #[snafu(display("Failed to determine mount status {}", source))]
+ FindMntFailure { source: std::io::Error },
+
+ #[snafu(display("Failed to format filesystem : {}", String::from_utf8_lossy(output.stderr.as_slice())))]
+ FormatFilesystemFailure { output: std::process::Output },
+
+ #[snafu(display("Invalid Parameter '{}', {}", parameter, reason))]
+ InvalidParameter { parameter: String, reason: String },
+
+ #[snafu(display("Failed to create directory, {}", source))]
+ Mkdir { source: std::io::Error },
+
+ #[snafu(display("Failed to canonicalize path, {}", source))]
+ CanonicalizeFailure { source: std::io::Error },
+ }
+}
+
+pub type Result = std::result::Result;
diff --git a/sources/api/apiserver/src/server/error.rs b/sources/api/apiserver/src/server/error.rs
index 0c66af2ed..cd7819398 100644
--- a/sources/api/apiserver/src/server/error.rs
+++ b/sources/api/apiserver/src/server/error.rs
@@ -1,3 +1,4 @@
+use crate::server::ephemeral_storage;
use actix_web::{HttpResponseBuilder, ResponseError};
use datastore::{self, deserialization, serialization};
use nix::unistd::Gid;
@@ -109,6 +110,21 @@ pub enum Error {
source: serde_json::Error,
},
+ #[snafu(display("Unable to initialize ephemeral storage: {}", source))]
+ EphemeralInitialize {
+ source: ephemeral_storage::error::Error,
+ },
+
+ #[snafu(display("Unable to bind ephemeral storage: {}", source))]
+ EphemeralBind {
+ source: ephemeral_storage::error::Error,
+ },
+
+ #[snafu(display("Unable to list ephemeral disks: {}", source))]
+ EphemeralListDisks {
+ source: ephemeral_storage::error::Error,
+ },
+
#[snafu(display("Unable to make {} key '{}': {}", key_type, name, source))]
NewKey {
key_type: String,
diff --git a/sources/api/apiserver/src/server/mod.rs b/sources/api/apiserver/src/server/mod.rs
index fa6da4c6d..1f08ac686 100644
--- a/sources/api/apiserver/src/server/mod.rs
+++ b/sources/api/apiserver/src/server/mod.rs
@@ -2,6 +2,7 @@
//! server::controller module.
mod controller;
+mod ephemeral_storage;
mod error;
mod exec;
@@ -15,6 +16,7 @@ use error::Result;
use fs2::FileExt;
use http::StatusCode;
use log::info;
+use model::ephemeral_storage::{Bind, Init};
use model::{ConfigurationFiles, Model, Report, Services, Settings};
use nix::unistd::{chown, Gid};
use serde::{Deserialize, Serialize};
@@ -124,7 +126,23 @@ where
.route("/refresh-updates", web::post().to(refresh_updates))
.route("/prepare-update", web::post().to(prepare_update))
.route("/activate-update", web::post().to(activate_update))
- .route("/deactivate-update", web::post().to(deactivate_update)),
+ .route("/deactivate-update", web::post().to(deactivate_update))
+ .route(
+ "/ephemeral-storage/init",
+ web::post().to(initialize_ephemeral_storage),
+ )
+ .route(
+ "/ephemeral-storage/bind",
+ web::post().to(bind_ephemeral_storage),
+ )
+ .route(
+ "/ephemeral-storage/list-disks",
+ web::get().to(list_ephemeral_storage_disks),
+ )
+ .route(
+ "/ephemeral-storage/list-dirs",
+ web::get().to(list_ephemeral_storage_dirs),
+ ),
)
.service(web::scope("/updates").route("/status", web::get().to(get_update_status)))
.service(web::resource("/exec").route(web::get().to(exec::ws_exec)))
@@ -648,6 +666,74 @@ async fn get_fips_report(query: web::Query>) -> Result) -> Result {
+ ephemeral_storage::initialize(cfg.0.filesystem, cfg.0.disks)
+ .context(error::EphemeralInitializeSnafu {})?;
+ Ok(HttpResponse::NoContent().finish()) // 204
+}
+/// Bind directories to ephemeral storage (mount array, bind, and unmount)
+async fn bind_ephemeral_storage(cfg: web::Json) -> Result {
+ let os_info = controller::get_os_info()?;
+ ephemeral_storage::bind(&os_info.variant_id, cfg.0.targets)
+ .context(error::EphemeralBindSnafu {})?;
+ Ok(HttpResponse::NoContent().finish()) // 204
+}
+
+/// Lists the known ephemeral disks that can be configured.
+async fn list_ephemeral_storage_disks(
+ req: HttpRequest,
+ query: web::Query>,
+) -> Result {
+ let disks =
+ ephemeral_storage::ephemeral_devices().context(error::EphemeralListDisksSnafu {})?;
+
+ let mut text_response = String::new();
+ for disk in &disks {
+ text_response.push_str(disk);
+ text_response.push('\n');
+ }
+ list_ephemeral_response(req, query, disks, text_response).await
+}
+
+/// Lists the known ephemeral disks that can be configured.
+async fn list_ephemeral_storage_dirs(
+ req: HttpRequest,
+ query: web::Query>,
+) -> Result {
+ let os_info = controller::get_os_info()?;
+
+ let allowed = ephemeral_storage::allowed_bind_dirs(&os_info.variant_id);
+ let mut text_response = String::new();
+ for dir in &allowed {
+ text_response.push_str(dir);
+ text_response.push('\n');
+ }
+
+ let allowed: Vec = allowed.iter().map(|x| String::from(*x)).collect();
+ list_ephemeral_response(req, query, allowed, text_response).await
+}
+
+// Responds to a list request with the text or JSON resposne depending on the query format.
+async fn list_ephemeral_response(
+ req: HttpRequest,
+ query: web::Query>,
+ items: Vec,
+ text_response: String,
+) -> Result {
+ match query
+ .get("format")
+ .unwrap_or(&String::from("text"))
+ .as_str()
+ {
+ "json" => Ok(EphemeralListResponse(items).respond_to(&req)),
+ "text" => Ok(HttpResponse::Ok()
+ .content_type("application/text")
+ .body(text_response)),
+ _ => Ok(HttpResponse::BadRequest().body("unsupported format")),
+ }
+}
+
// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=
// Helpers for handler methods called by the router
@@ -782,6 +868,9 @@ impl ResponseError for error::Error {
Deserialization { .. } => StatusCode::INTERNAL_SERVER_ERROR,
DataStoreSerialization { .. } => StatusCode::INTERNAL_SERVER_ERROR,
CommandSerialization { .. } => StatusCode::INTERNAL_SERVER_ERROR,
+ EphemeralBind { .. } => StatusCode::INTERNAL_SERVER_ERROR,
+ EphemeralInitialize { .. } => StatusCode::INTERNAL_SERVER_ERROR,
+ EphemeralListDisks { .. } => StatusCode::INTERNAL_SERVER_ERROR,
InvalidMetadata { .. } => StatusCode::INTERNAL_SERVER_ERROR,
ConfigApplierFork { .. } => StatusCode::INTERNAL_SERVER_ERROR,
ConfigApplierStart { .. } => StatusCode::INTERNAL_SERVER_ERROR,
@@ -895,3 +984,6 @@ impl_responder_for!(TransactionListResponse, self, self.0);
struct ReportListResponse(Vec);
impl_responder_for!(ReportListResponse, self, self.0);
+
+struct EphemeralListResponse(Vec);
+impl_responder_for!(EphemeralListResponse, self, self.0);
diff --git a/sources/api/openapi.yaml b/sources/api/openapi.yaml
index 67c0b4306..0b806fd9d 100644
--- a/sources/api/openapi.yaml
+++ b/sources/api/openapi.yaml
@@ -198,7 +198,18 @@ components:
$ref: '#/components/schemas/ConfigurationFiles'
os:
$ref: '#/components/schemas/BottlerocketRelease'
-
+ EphemeralStorageInit:
+ type: object
+ properties:
+ filesystem:
+ type: string
+ disks:
+ type: array
+ EphemeralStorageBind:
+ type: object
+ properties:
+ targets:
+ type: array
paths:
/:
get:
@@ -217,7 +228,7 @@ paths:
content:
application/json:
schema:
- $ref: "Model"
+ $ref: "#/components/schemas/Model"
500:
description: "Server error"
@@ -249,7 +260,7 @@ paths:
content:
application/json:
schema:
- $ref: "Settings"
+ $ref: "#/components/schemas/Settings"
500:
description: "Server error"
patch:
@@ -267,7 +278,7 @@ paths:
content:
application/json:
schema:
- $ref: "Settings"
+ $ref: "#/components/schemas/Settings"
responses:
204:
description: "Settings successfully staged for update"
@@ -291,7 +302,7 @@ paths:
content:
application/json:
schema:
- $ref: "SettingsKeyPair"
+ $ref: "#/components/schemas/SettingsKeyPair"
responses:
204:
description: "Settings successfully staged for update"
@@ -316,7 +327,7 @@ paths:
content:
application/json:
schema:
- $ref: "Settings"
+ $ref: #/components/schemas/Settings"
500:
description: "Server error"
delete:
@@ -530,7 +541,7 @@ paths:
content:
application/json:
schema:
- $ref: "Services"
+ $ref: "#/components/schemas/Services"
500:
description: "Server error"
@@ -562,7 +573,7 @@ paths:
content:
application/json:
schema:
- $ref: "ConfigurationFiles"
+ $ref: "#/components/schemas/ConfigurationFiles"
500:
description: "Server error"
@@ -646,7 +657,7 @@ paths:
content:
application/json:
schema:
- $ref: "UpdateStatus"
+ $ref: "#/components/schemas/UpdateStatus"
500:
description: "Server error"
423:
@@ -674,7 +685,7 @@ paths:
schema:
type: array
items:
- $ref: "Report"
+ $ref: "#/components/schemas/Report"
500:
description: "Server error"
@@ -735,3 +746,98 @@ paths:
description: "Unprocessable request"
500:
description: "Server error"
+ /ephemeral-storage/init:
+ post:
+ summary: "Initialize ephemeral storage"
+ operationId: "init"
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/EphemeralStorageInit"
+ responses:
+ 200:
+ description: "Successful request"
+ content:
+ application/json:
+ schema:
+ type: string
+ 400:
+ description: "Bad request input"
+ 422:
+ description: "Unprocessable request"
+ 500:
+ description: "Server error"
+ /ephemeral-storage/bind:
+ post:
+ summary: "Bind directories to previously initialized ephemeral storage"
+ operationId: "bind"
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/EphemeralStorageBind"
+ responses:
+ 200:
+ description: "Successful request"
+ content:
+ application/json:
+ schema:
+ type: string
+ 400:
+ description: "Bad request input"
+ 422:
+ description: "Unprocessable request"
+ 500:
+ description: "Server error"
+ /ephemeral-storage/list-disks:
+ get:
+ summary: "List the discovered ephemeral disks that can be initialized"
+ operationId: "list-disks"
+ parameters:
+ - in: query
+ name: format
+ description: "Format of the disk listing (text or json). Default format is text."
+ schema:
+ type: string
+ required: false
+ responses:
+ 200:
+ description: "Successful request"
+ content:
+ application/json:
+ schema:
+ type: string
+ 400:
+ description: "Bad request input"
+ 422:
+ description: "Unprocessable request"
+ 500:
+ description: "Server error"
+ /ephemeral-storage/list-dirs:
+ get:
+ summary: "List the directories that can be bound to ephemeral storage"
+ operationId: "list-dirs"
+ parameters:
+ - in: query
+ name: format
+ description: "Format of the directory listing (text or json). Default format is text."
+ schema:
+ type: string
+ required: false
+ responses:
+ 200:
+ description: "Successful request"
+ content:
+ application/json:
+ schema:
+ type: string
+ 400:
+ description: "Bad request input"
+ 422:
+ description: "Unprocessable request"
+ 500:
+ description: "Server error"
+
diff --git a/sources/models/src/ephemeral_storage.rs b/sources/models/src/ephemeral_storage.rs
new file mode 100644
index 000000000..2fa87a402
--- /dev/null
+++ b/sources/models/src/ephemeral_storage.rs
@@ -0,0 +1,32 @@
+//! The 'ephemeral_storage' module holds types used to communicate between client and server for
+//! 'apiclient ephemeral-storage'.
+use serde::{Deserialize, Serialize};
+use std::fmt::{Display, Formatter};
+
+/// Supported filesystems for ephemeral storage
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub enum Filesystem {
+ Xfs,
+ Ext4,
+}
+impl Display for Filesystem {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ match self {
+ Filesystem::Xfs => f.write_str("xfs"),
+ Filesystem::Ext4 => f.write_str("ext4"),
+ }
+ }
+}
+
+/// Initialize ephemeral storage
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Init {
+ pub filesystem: Option,
+ pub disks: Option>,
+}
+
+/// Bind directories to configured ephemeral storage
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Bind {
+ pub targets: Vec,
+}
diff --git a/sources/models/src/lib.rs b/sources/models/src/lib.rs
index e3015d3f9..fa76b8c46 100644
--- a/sources/models/src/lib.rs
+++ b/sources/models/src/lib.rs
@@ -21,6 +21,9 @@ The `#[model]` attribute on Settings and its sub-structs reduces duplication and
// Types used to communicate between client and server for 'apiclient exec'.
pub mod exec;
+// Types used to communicate between client and server for 'apiclient ephemeral-storage'.
+pub mod ephemeral_storage;
+
use bottlerocket_release::BottlerocketRelease;
use bottlerocket_settings_models::model_derive::model;
use bottlerocket_settings_plugin::BottlerocketSettings;