Skip to content

Commit

Permalink
fix: error logging and code refactoring
Browse files Browse the repository at this point in the history 
Signed-off-by: Sayan Paul <saypaul@redhat.com>
  • Loading branch information
@say-paul
say-paul committed Nov 1, 2023
1 parent 0064726 commit 60e4fc1
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 116 deletions.
78 changes: 29 additions & 49 deletions src/handler/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,90 +16,78 @@ pub fn handle_reboot(force: bool) -> Result<()> {
None => bail!("boot_counter is not set"),
}
}
log::info!("restarting system");
let status = Command::new("systemctl").arg("reboot").status()?;
if status.success() {
return Ok(());
}
bail!("systemd returned error");
log::info!("restarting the system");
Command::new("systemctl").arg("reboot").spawn()?;
Ok(())
}

/// rollback to previous ostree deployment if boot counter is less than 0
pub fn handle_rollback() -> Result<()> {
if let Some(t) = get_boot_counter() {
if t <= 0 {
log::info!("Greenboot will now attempt to rollback");
let status = Command::new("rpm-ostree").arg("rollback").status()?;
if status.success() {
return Ok(());
}
bail!(status.to_string());
Command::new("rpm-ostree").arg("rollback").spawn()?;
return Ok(());
} else {
bail!("Rollback not initiated, boot_counter {t}");
}
}
log::info!("Rollback not initiated as boot_counter is either unset or not equal to 0");
Ok(())
bail!("Rollback not initiated, boot_counter unset");
}

/// sets grub variable boot_counter if not set
pub fn set_boot_counter(reboot_count: i32) -> Result<()> {
if let Some(current_counter) = get_boot_counter() {
log::info!("boot_counter={current_counter}");
return Ok(());
} else if set_grub_var("boot_counter", reboot_count) {
log::info!("Set boot_counter={reboot_count}");
return Ok(());
}
bail!("Failed to set GRUB variable: boot_counter");
log::info!("setting boot counter");
set_grub_var("boot_counter", reboot_count)?;
Ok(())
}

/// resets grub variable boot_counter
pub fn unset_boot_counter() -> Result<()> {
let status = Command::new("grub2-editenv")
Command::new("grub2-editenv")
.arg("-")
.arg("unset")
.arg("boot_counter")
.status()?;
if status.success() {
return Ok(());
}
bail!("grub returned an error")
Ok(())
}

/// sets grub variable boot_success
pub fn handle_boot_success(success: bool) -> Result<()> {
pub fn handle_boot_status(success: bool) -> Result<()> {
if success {
if !set_grub_var("boot_success", 1) {
bail!("unable to mark boot as success, grub returned an error")
}
match unset_boot_counter() {
Ok(_) => return Ok(()),
Err(e) => bail!("unable to remove boot_counter, {e}"),
}
} else if !set_grub_var("boot_success", 0) {
bail!("unable to mark boot as failure, grub returned an error")
set_grub_var("boot_success", 1)?;
unset_boot_counter()?;
} else {
set_grub_var("boot_success", 0)?;
}
Ok(())
}

/// writes greenboot status to motd.d/boot-status
pub fn handle_motd(state: &str) -> Result<()> {
let motd = format!("Greenboot {state}.");

std::fs::write("/etc/motd.d/boot-status", motd.as_bytes())?;
std::fs::write(
"/etc/motd.d/boot-status",
format!("Greenboot {state}.").as_bytes(),
)?;
Ok(())
}

/// fetches boot_counter value, none if not set
pub fn get_boot_counter() -> Option<i32> {
let grub_vars = Command::new("grub2-editenv").arg("-").arg("list").output();
if grub_vars.is_err() {
log::error!("{}", grub_vars.unwrap_err());
return None;
}
let grub_vars = grub_vars.unwrap();
let grub_vars = match str::from_utf8(&grub_vars.stdout[..]) {
Ok(vars) => vars.lines(),
Err(e) => {
log::error!("Unable to fetch grub variables, {e}");
log::error!("{e},unable to fetch grub variables");
return None;
}
};
Expand All @@ -116,7 +104,7 @@ pub fn get_boot_counter() -> Option<i32> {
match v.parse::<i32>() {
Ok(count) => return Some(count),
Err(_) => {
log::error!("boot_counter not a valid integer");
log::error!("boot_counter {v} , should be a valid integer");
return None;
}
}
Expand All @@ -125,19 +113,11 @@ pub fn get_boot_counter() -> Option<i32> {
}

/// helper function to set any grub variable
fn set_grub_var(key: &str, val: i32) -> bool {
match Command::new("grub2-editenv")
fn set_grub_var(key: &str, val: i32) -> Result<()> {
Command::new("grub2-editenv")
.arg("-")
.arg("set")
.arg(format!("{key}={val}"))
.status()
{
Ok(status) => {
if status.success() {
return true;
}
false
}
Err(_) => false,
}
.spawn()?;
Ok(())
}
131 changes: 64 additions & 67 deletions src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,25 @@ impl GreenbootConfig {
Ok(c) => {
config.max_reboot = match c.get_int("GREENBOOT_MAX_BOOT_ATTEMPTS") {
Ok(c) => c.try_into().unwrap_or_else(|e| {
log::warn!("config format error:{e}, using default value");
log::warn!(
"config error:{e}, using default value: {}",
config.max_reboot
);
config.max_reboot
}),
Err(e) => {
log::warn!("error reading config:{e}, using default value");
log::warn!(
"error reading config:{e}, using default value: {}",
config.max_reboot
);
config.max_reboot
}
}
}
Err(e) => log::warn!("config file error:{e}, using default value"),
Err(e) => log::warn!(
"config file error:{e}, using default value: {}",
config.max_reboot
),
}
config
}
Expand Down Expand Up @@ -107,17 +116,14 @@ fn run_diagnostics() -> Result<(), Error> {
continue;
}
path_exists = true;
let greenboot_required_path = format!("{greenboot_required_path}*.sh");
for entry in glob(&greenboot_required_path)?.flatten() {
log::info!("running required check {}", entry.to_string_lossy());
let output = Command::new("bash").arg("-C").arg(entry.clone()).output()?;
if !output.status.success() {
log::warn!(
"required script {} failed! \n{} \n{}",
entry.to_string_lossy(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
match run_scripts("required", &greenboot_required_path) {
Ok(script_status) => {
if !script_status {
script_failure = true;
}
}
Err(e) => {
log::warn!("required script runner error: {}", e.to_string());
script_failure = true;
}
}
Expand All @@ -128,20 +134,9 @@ fn run_diagnostics() -> Result<(), Error> {
}

for path in GREENBOOT_INSTALL_PATHS {
let greenboot_wanted_path = format!("{path}/check/wanted.d/*.sh");
for entry in glob(&greenboot_wanted_path)?.flatten() {
log::info!("running wanted check {}", entry.to_string_lossy());
let output = Command::new("bash").arg("-C").arg(entry.clone()).output()?;
if !output.status.success() {
log::warn!(
"wanted script {} failed! \n{} \n{}",
entry.to_string_lossy(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
} else {
log::warn!("wanted script {} success!", entry.to_string_lossy());
}
let greenboot_wanted_path = format!("{path}/check/wanted.d/");
if let Some(e) = run_scripts("wanted", &greenboot_wanted_path).err() {
log::error!("wanted script runner error:{}", e.to_string());
}
}

Expand All @@ -154,18 +149,9 @@ fn run_diagnostics() -> Result<(), Error> {
/// runs the scripts in red.d when health-check fails
fn run_red() -> Result<(), Error> {
for path in GREENBOOT_INSTALL_PATHS {
let red_path = format!("{path}/red.d/*.*");
for entry in glob(&red_path)?.flatten() {
log::info!("running red check {}", entry.to_string_lossy());
let output = Command::new("bash").arg("-C").arg(entry.clone()).output()?;
if !output.status.success() {
log::warn!(
"red script {} failed! \n{} \n{}",
entry.to_string_lossy(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
let red_path = format!("{path}/red.d/");
if let Some(e) = run_scripts("red", &red_path).err() {
log::error!("red script runner error: {}", e.to_string());
}
}
Ok(())
Expand All @@ -174,50 +160,38 @@ fn run_red() -> Result<(), Error> {
/// runs the scripts green.d when health-check passes
fn run_green() -> Result<(), Error> {
for path in GREENBOOT_INSTALL_PATHS {
let green_path = format!("{path}/green.d/*.*");
for entry in glob(&green_path)?.flatten() {
log::info!("running green check {}", entry.to_string_lossy());
let output = Command::new("bash").arg("-C").arg(entry.clone()).output()?;
if !output.status.success() {
log::warn!(
"green script {} failed! \n{} \n{}",
entry.to_string_lossy(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
}
let green_path = format!("{path}/green.d/");
if let Some(e) = run_scripts("green", &green_path).err() {
log::error!("green script runner error: {}", e.to_string());
}
}
Ok(())
}

/// triggers the diagnostics followed by the action on the outcome
/// this also handle setting the grub variables and system restart
/// this also handles setting the grub variables and system restart
fn health_check() -> Result<()> {
let config = GreenbootConfig::get_config();
log::info!("{config:?}");
log::debug!("{config:?}");
handle_motd("healthcheck is in progress")?;
let run_status = run_diagnostics();
match run_status {
match run_diagnostics() {
Ok(()) => {
log::info!("greenboot health-check passed.");
run_green().unwrap_or_else(|e| {
log::error!("cannot run green script due to: {}", e.to_string())
});
run_green()
.unwrap_or_else(|e| log::error!("cannot run green script: {}", e.to_string()));
handle_motd("healthcheck passed - status is GREEN")
.unwrap_or_else(|e| log::error!("cannot set motd due to : {}", e.to_string()));
handle_boot_success(true)?;
.unwrap_or_else(|e| log::error!("cannot set motd: {}", e.to_string()));
handle_boot_status(true)?;
Ok(())
}
Err(e) => {
log::error!("Greenboot health-check failed!");
handle_motd("healthcheck failed - status is RED")
.unwrap_or_else(|e| log::error!("cannot set motd due to : {}", e.to_string()));
run_red()
.unwrap_or_else(|e| log::error!("cannot run red script due to: {}", e.to_string()));
handle_boot_success(false)?;
.unwrap_or_else(|e| log::error!("cannot set motd: {}", e.to_string()));
run_red().unwrap_or_else(|e| log::error!("cannot run red script: {}", e.to_string()));
handle_boot_status(false)?;
set_boot_counter(config.max_reboot)
.unwrap_or_else(|e| log::error!("cannot set boot_counter as: {}", e.to_string()));
.unwrap_or_else(|e| log::error!("cannot set boot_counter: {}", e.to_string()));
handle_reboot(false)
.unwrap_or_else(|e| log::error!("cannot reboot as: {}", e.to_string()));
Err(e)
Expand All @@ -234,9 +208,32 @@ fn trigger_rollback() -> Result<()> {
handle_reboot(true)
}
Err(e) => {
bail!("Rollback not initiated as {}", e);
bail!("{e}, Rollback is not initiated");
}
}
}

/// takes in a path and runs all the .sh files within the path
/// returns false if any script fails
fn run_scripts(name: &str, path: &str) -> Result<bool> {
let mut status = true;
let scripts = format!("{path}*.sh");
for entry in glob(&scripts)?.flatten() {
log::info!("running {name} check {}", entry.to_string_lossy());
let output = Command::new("bash").arg("-C").arg(entry.clone()).output()?;
if !output.status.success() {
status = false;
log::warn!(
"{name} script {} failed! \n{} \n{}",
entry.to_string_lossy(),
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
} else {
log::warn!("{name} script {} success!", entry.to_string_lossy());
}
}
Ok(status)
}

fn main() -> Result<()> {
Expand Down

0 comments on commit 60e4fc1

Please sign in to comment.