From 650fde595a38aaf4652f4df23fd1b9e970790830 Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Wed, 26 Nov 2025 13:26:31 -0500 Subject: [PATCH 1/5] Include /var/adm/messages in archived zone logs System messages on illumos are logged to `/var/adm/messages`. In the event of a hardware or kernel issue, an archive of these logs can be for understanding the nature of the failure. Currently we archive all Oxide service logs from the global zone and any zones we create, but not `/var/adm/messages`. Update the archiver to include the latter as well. --- .../config-reconciler/src/dump_setup.rs | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/sled-agent/config-reconciler/src/dump_setup.rs b/sled-agent/config-reconciler/src/dump_setup.rs index 03846454b86..26c980c4076 100644 --- a/sled-agent/config-reconciler/src/dump_setup.rs +++ b/sled-agent/config-reconciler/src/dump_setup.rs @@ -1101,15 +1101,22 @@ impl DumpSetupWorker { let oxz_zones = self.zone_invoker.get_zones().await?; for zone in oxz_zones { let logdir = if zone.global() { - PathBuf::from("/var/svc/log") + zone.path.join("var/svc/log") } else { zone.path().join("root/var/svc/log") }; let zone_name = zone.name(); self.archive_logs_from_zone_path( - debug_dir, logdir, zone_name, false, + debug_dir, logdir, "*.log", zone_name, false, ) .await?; + if zone.global() { + let adm_logdir = zone.path.join("var/adm"); + self.archive_logs_from_zone_path( + debug_dir, adm_logdir, "messages", zone_name, false, + ) + .await?; + } } Ok(()) } @@ -1129,6 +1136,7 @@ impl DumpSetupWorker { .archive_logs_from_zone_path( debug_dir, logdir.into(), + "*.log", zone_name, true, ) @@ -1149,13 +1157,14 @@ impl DumpSetupWorker { &self, debug_dir: &DebugDataset, logdir: PathBuf, + log_name_pattern: &str, zone_name: &str, include_live: bool, ) -> Result<(), ArchiveLogsError> { let mut rotated_log_files = Vec::new(); if include_live { let pattern = logdir - .join("*.log*") + .join(format!("{log_name_pattern}*")) .to_str() .ok_or_else(|| ArchiveLogsError::Utf8(zone_name.to_string()))? .to_string(); @@ -1166,7 +1175,7 @@ impl DumpSetupWorker { // any for n in 1..9 { let pattern = logdir - .join(format!("*.log.{}", "[0-9]".repeat(n))) + .join(format!("{log_name_pattern}.{}", "[0-9]".repeat(n))) .to_str() .ok_or_else(|| { ArchiveLogsError::Utf8(zone_name.to_string()) @@ -1794,8 +1803,13 @@ mod tests { let core_dir = tempdir.path().join(CRASH_DATASET); let debug_dir = tempdir.path().join(DUMP_DATASET); let zone_logs = tempdir.path().join("root/var/svc/log"); + let adm_logs = tempdir.path().join("var/adm"); let tempdir_path = tempdir.path().as_str().to_string(); + let global_zone = Zone::from_str(&format!( + "0:global:running:{tempdir_path}::ipkg:shared" + )) + .unwrap(); let zone = Zone::from_str(&format!( "1:myzone:running:{tempdir_path}::ipkg:shared" )) @@ -1831,7 +1845,7 @@ mod tests { .into_iter() .collect(), }), - Box::new(FakeZone { zones: vec![zone.clone()] }), + Box::new(FakeZone { zones: vec![global_zone, zone.clone()] }), logctx.log.clone(), tokio::sync::mpsc::channel(1).1, ); @@ -1839,6 +1853,7 @@ mod tests { tokio::fs::create_dir_all(&core_dir).await.unwrap(); tokio::fs::create_dir_all(&debug_dir).await.unwrap(); tokio::fs::create_dir_all(&zone_logs).await.unwrap(); + tokio::fs::create_dir_all(&adm_logs).await.unwrap(); const LOG_NAME: &'static str = "foo.log.0"; tokio::fs::File::create(zone_logs.join(LOG_NAME)) .await @@ -1847,6 +1862,14 @@ mod tests { .await .expect("writing fake log"); + const ADM_LOG_NAME: &'static str = "messages.0"; + tokio::fs::File::create(adm_logs.join(ADM_LOG_NAME)) + .await + .expect("creating fake adm log") + .write_all(b"admin stuff") + .await + .expect("writing fake adm log"); + const CORE_NAME: &str = "core.myzone.myexe.123.1690540950"; tokio::fs::File::create(core_dir.join(CORE_NAME)) .await @@ -1878,6 +1901,12 @@ mod tests { debug_dir.join(zone.name()).join(LOG_NAME.replace(".0", ".*")); assert_eq!(glob::glob(log_glob.as_str()).unwrap().count(), 1); assert!(!zone_logs.join(LOG_NAME).is_file()); + + let adm_glob = + debug_dir.join("global").join(ADM_LOG_NAME.replace(".0", ".*")); + assert_eq!(glob::glob(adm_glob.as_str()).unwrap().count(), 1); + assert!(!adm_logs.join(ADM_LOG_NAME).is_file()); + assert!(debug_dir.join(CORE_NAME).is_file()); assert!(!core_dir.join(CORE_NAME).is_file()); logctx.cleanup_successful(); From b386fe201b00a0a4ae6bd69e1063b1b9b4ab0454 Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Tue, 2 Dec 2025 11:33:24 -0500 Subject: [PATCH 2/5] Create separate directories for zones --- sled-agent/config-reconciler/src/dump_setup.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/sled-agent/config-reconciler/src/dump_setup.rs b/sled-agent/config-reconciler/src/dump_setup.rs index 26c980c4076..879eac90a43 100644 --- a/sled-agent/config-reconciler/src/dump_setup.rs +++ b/sled-agent/config-reconciler/src/dump_setup.rs @@ -1802,16 +1802,18 @@ mod tests { let tempdir = Utf8TempDir::new().unwrap(); let core_dir = tempdir.path().join(CRASH_DATASET); let debug_dir = tempdir.path().join(DUMP_DATASET); - let zone_logs = tempdir.path().join("root/var/svc/log"); - let adm_logs = tempdir.path().join("var/adm"); + let zone_path = tempdir.path().join("myzone"); + let zone_logs = zone_path.join("root/var/svc/log"); + let global_path = tempdir.path().join("global"); + let adm_logs = global_path.join("var/adm"); let tempdir_path = tempdir.path().as_str().to_string(); let global_zone = Zone::from_str(&format!( - "0:global:running:{tempdir_path}::ipkg:shared" + "0:global:running:{global_path}::ipkg:shared" )) .unwrap(); let zone = Zone::from_str(&format!( - "1:myzone:running:{tempdir_path}::ipkg:shared" + "1:myzone:running:{zone_path}::ipkg:shared" )) .unwrap(); @@ -1850,6 +1852,8 @@ mod tests { tokio::sync::mpsc::channel(1).1, ); + tokio::fs::create_dir_all(&zone_path).await.unwrap(); + tokio::fs::create_dir_all(&global_path).await.unwrap(); tokio::fs::create_dir_all(&core_dir).await.unwrap(); tokio::fs::create_dir_all(&debug_dir).await.unwrap(); tokio::fs::create_dir_all(&zone_logs).await.unwrap(); From ddc56e2df09bbb680c49505196f45cae3ed1fae9 Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Tue, 2 Dec 2025 11:23:41 -0500 Subject: [PATCH 3/5] Capture /var/adm/messages for all zones --- sled-agent/config-reconciler/src/dump_setup.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sled-agent/config-reconciler/src/dump_setup.rs b/sled-agent/config-reconciler/src/dump_setup.rs index 879eac90a43..4bd94656537 100644 --- a/sled-agent/config-reconciler/src/dump_setup.rs +++ b/sled-agent/config-reconciler/src/dump_setup.rs @@ -1110,13 +1110,12 @@ impl DumpSetupWorker { debug_dir, logdir, "*.log", zone_name, false, ) .await?; - if zone.global() { - let adm_logdir = zone.path.join("var/adm"); - self.archive_logs_from_zone_path( - debug_dir, adm_logdir, "messages", zone_name, false, - ) - .await?; - } + + let adm_logdir = zone.path.join("var/adm"); + self.archive_logs_from_zone_path( + debug_dir, adm_logdir, "messages", zone_name, false, + ) + .await?; } Ok(()) } From 6e5ba44ff9f7374dc4244c3de6c09665756e0638 Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Tue, 2 Dec 2025 12:34:00 -0500 Subject: [PATCH 4/5] Comment on archive_logs_from_zone_path params --- sled-agent/config-reconciler/src/dump_setup.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sled-agent/config-reconciler/src/dump_setup.rs b/sled-agent/config-reconciler/src/dump_setup.rs index 4bd94656537..c22871da771 100644 --- a/sled-agent/config-reconciler/src/dump_setup.rs +++ b/sled-agent/config-reconciler/src/dump_setup.rs @@ -1152,6 +1152,11 @@ impl DumpSetupWorker { rv } + // Archives log files found in `logdir` for zone `zone_name` to the destination debug dataset. + // + // `log_name_pattern` should be a glob pattern that matches against file names, e.g., `*.log`, `mylog`. + // If `include_live` is `true`, this will archive all logs, matching on `{log_name_pattern}*`. + // If it is `false`, only rotated logs will be archived, matching on `{log_name_pattern}.[0-9]`. async fn archive_logs_from_zone_path( &self, debug_dir: &DebugDataset, From 1a457af0e044720a5df87a94f141cefe61c3d086 Mon Sep 17 00:00:00 2001 From: Will Chandler Date: Thu, 4 Dec 2025 10:33:12 -0500 Subject: [PATCH 5/5] Set zone root properly for non-GZ --- sled-agent/config-reconciler/src/dump_setup.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sled-agent/config-reconciler/src/dump_setup.rs b/sled-agent/config-reconciler/src/dump_setup.rs index c22871da771..b3a6ab1bbe0 100644 --- a/sled-agent/config-reconciler/src/dump_setup.rs +++ b/sled-agent/config-reconciler/src/dump_setup.rs @@ -1099,19 +1099,21 @@ impl DumpSetupWorker { .as_ref() .ok_or(ArchiveLogsError::NoDebugDirYet)?; let oxz_zones = self.zone_invoker.get_zones().await?; + for zone in oxz_zones { - let logdir = if zone.global() { - zone.path.join("var/svc/log") + let zone_root = if zone.global() { + zone.path().to_owned() } else { - zone.path().join("root/var/svc/log") + zone.path().join("root") }; + let logdir = zone_root.join("var/svc/log"); let zone_name = zone.name(); self.archive_logs_from_zone_path( debug_dir, logdir, "*.log", zone_name, false, ) .await?; - let adm_logdir = zone.path.join("var/adm"); + let adm_logdir = zone_root.join("var/adm"); self.archive_logs_from_zone_path( debug_dir, adm_logdir, "messages", zone_name, false, )