Add support for recording ethtool stats (#8204)

Summary:
The PR is broken into modular commits, so it might be easier to review the PR commit-by-commit.

Commit: [Add crate for reading NIC stats using ethtool](965795c)
  - Adds the library support to read and parse `ethtool` stats using `ioctl`
  - Parses some common stats into fields while others are stored as raw stats

Commit: [Add model to represent ethtool stats](66dbb26)
  - Adds the model struct and parsing logic to convert `ethtool` stats into a sub-model in existing `network.interface` model

Commit: [Add render configs for ethtool fields](9401fed)
  - Updates and adds the required commands for dumping new fields and model
  - Adds the parsing for rendering the new fields and model

Commit: [Add ability for printing raw_stats in OpenMetrics format](7d6349e)
  - Modifies the printing logic of `OpenMetrics` format to handle a map type

Pull Request resolved: #8204

Reviewed By: brianc118

Differential Revision: D50714625

Pulled By: dschatzberg

fbshipit-source-id: 156f3054944d86fe489bd9ca5bfb3023d1e02c26

Cargo.toml

@@ -7,6 +7,7 @@ members = [
   "below/common",
   "below/config",
   "below/dump",
+  "below/ethtool",
   "below/gpu_stats",
   "below/model",
   "below/procfs",

below/config/src/lib.rs

@@ -45,6 +45,7 @@ pub struct BelowConfig {
     pub enable_btrfs_stats: bool,
     pub btrfs_samples: u64,
     pub btrfs_min_pct: f64,
+    pub enable_ethtool_stats: bool,
 }
 
 impl Default for BelowConfig {
@@ -59,6 +60,7 @@ impl Default for BelowConfig {
             enable_btrfs_stats: false,
             btrfs_samples: btrfs::DEFAULT_SAMPLES,
             btrfs_min_pct: btrfs::DEFAULT_MIN_PCT,
+            enable_ethtool_stats: false,
         }
     }
 }

below/dump/src/command.rs

@@ -25,6 +25,7 @@ use model::SingleCgroupModelFieldId;
 use model::SingleDiskModelFieldId;
 use model::SingleNetModelFieldId;
 use model::SingleProcessModelFieldId;
+use model::SingleQueueModelFieldId;
 use model::SystemModelFieldId;
 use once_cell::sync::Lazy;
 use regex::Regex;
@@ -664,11 +665,13 @@ pub enum IfaceAggField {
     Rate,
     Rx,
     Tx,
+    Ethtool,
 }
 
 impl AggField<SingleNetModelFieldId> for IfaceAggField {
-    fn expand(&self, _detail: bool) -> Vec<SingleNetModelFieldId> {
+    fn expand(&self, detail: bool) -> Vec<SingleNetModelFieldId> {
         use model::SingleNetModelFieldId::*;
+
         match self {
             Self::Rate => vec![
                 RxBytesPerSec,
@@ -703,6 +706,13 @@ impl AggField<SingleNetModelFieldId> for IfaceAggField {
                 TxPackets,
                 TxWindowErrors,
             ],
+            Self::Ethtool => {
+                let mut fields = vec![TxTimeoutPerSec];
+                if detail {
+                    fields.push(RawStats);
+                }
+                fields
+            }
         }
     }
 }
@@ -717,6 +727,7 @@ pub static DEFAULT_IFACE_FIELDS: &[IfaceOptionField] = &[
     DumpOptionField::Agg(IfaceAggField::Rate),
     DumpOptionField::Agg(IfaceAggField::Rx),
     DumpOptionField::Agg(IfaceAggField::Tx),
+    DumpOptionField::Agg(IfaceAggField::Ethtool),
     DumpOptionField::Unit(DumpField::Common(CommonField::Timestamp)),
 ];
 
@@ -739,7 +750,9 @@ static IFACE_LONG_ABOUT: Lazy<String> = Lazy::new(|| {
 
 * tx: includes [{agg_tx_fields}].
 
-* --detail: no effect.
+* ethtool: includes [{agg_ethtool_fields}].
+
+* --detail: includes `raw_stats` field.
 
 * --default: includes [{default_fields}].
 
@@ -763,6 +776,7 @@ $ below dump iface -b "08:30:00" -e "08:30:30" -s interface -F eth* -O json
         agg_rate_fields = join(IfaceAggField::Rate.expand(false)),
         agg_rx_fields = join(IfaceAggField::Rx.expand(false)),
         agg_tx_fields = join(IfaceAggField::Tx.expand(false)),
+        agg_ethtool_fields = join(IfaceAggField::Ethtool.expand(false)),
         default_fields = join(DEFAULT_IFACE_FIELDS.to_owned()),
     )
 });
@@ -941,6 +955,83 @@ $ below dump transport -b "08:30:00" -e "08:30:30" -f tcp udp -O json
     )
 });
 
+/// Represents the ethtool queue sub-model of the network model.
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    below_derive::EnumFromStr,
+    below_derive::EnumToString
+)]
+pub enum EthtoolQueueAggField {
+    Queue,
+}
+
+impl AggField<SingleQueueModelFieldId> for EthtoolQueueAggField {
+    fn expand(&self, detail: bool) -> Vec<SingleQueueModelFieldId> {
+        use model::SingleQueueModelFieldId::*;
+        let mut fields = vec![
+            Interface,
+            QueueId,
+            RxBytesPerSec,
+            TxBytesPerSec,
+            RxCountPerSec,
+            TxCountPerSec,
+            TxMissedTx,
+            TxUnmaskInterrupt,
+        ];
+        if detail {
+            fields.push(RawStats);
+        }
+        match self {
+            Self::Queue => fields,
+        }
+    }
+}
+
+pub type EthtoolQueueOptionField = DumpOptionField<SingleQueueModelFieldId, EthtoolQueueAggField>;
+
+pub static DEFAULT_ETHTOOL_QUEUE_FIELDS: &[EthtoolQueueOptionField] = &[
+    DumpOptionField::Unit(DumpField::Common(CommonField::Datetime)),
+    DumpOptionField::Agg(EthtoolQueueAggField::Queue),
+    DumpOptionField::Unit(DumpField::Common(CommonField::Timestamp)),
+];
+
+const ETHTOOL_QUEUE_ABOUT: &str = "Dump network interface queue stats";
+
+/// Generated about message for Ethtool Queue dump so supported fields are up-to-date.
+static ETHTOOL_QUEUE_LONG_ABOUT: Lazy<String> = Lazy::new(|| {
+    format!(
+        r#"{about}
+
+********************** Available fields **********************
+
+{common_fields}, and expanded fields below.
+
+********************** Aggregated fields **********************
+
+* queue: includes [{agg_queue_fields}].
+
+* --detail: includes `raw_stats` field.
+
+* --default: includes [{default_fields}].
+
+* --everything: includes everything (equivalent to --default --detail).
+
+********************** Example Commands **********************
+
+Example:
+
+$ below dump ethtool-queue -b "08:30:00" -e "08:30:30" -O json
+
+"#,
+        about = ETHTOOL_QUEUE_ABOUT,
+        common_fields = join(enum_iterator::all::<CommonField>()),
+        agg_queue_fields = join(EthtoolQueueAggField::Queue.expand(false)),
+        default_fields = join(DEFAULT_ETHTOOL_QUEUE_FIELDS.to_owned()),
+    )
+});
+
 make_option! (OutputFormat {
     "raw": Raw,
     "csv": Csv,
@@ -1113,4 +1204,15 @@ pub enum DumpCommand {
         #[clap(long, short, conflicts_with("fields"))]
         pattern: Option<String>,
     },
+    #[clap(about = ETHTOOL_QUEUE_ABOUT, long_about = ETHTOOL_QUEUE_LONG_ABOUT.as_str())]
+    EthtoolQueue {
+        /// Select which fields to display and in what order.
+        #[clap(short, long, num_args = 1..)]
+        fields: Option<Vec<EthtoolQueueOptionField>>,
+        #[clap(flatten)]
+        opts: GeneralOpt,
+        /// Saved pattern in the dumprc file under [ethtool] section.
+        #[clap(long, short, conflicts_with("fields"))]
+        pattern: Option<String>,
+    },
 }

below/dump/src/ethtool.rs

@@ -0,0 +1,110 @@
+use super::*;
+
+pub struct EthtoolQueue {
+    opts: GeneralOpt,
+    fields: Vec<EthtoolQueueField>,
+}
+
+impl EthtoolQueue {
+    pub fn new(opts: &GeneralOpt, fields: Vec<EthtoolQueueField>) -> Self {
+        Self {
+            opts: opts.to_owned(),
+            fields,
+        }
+    }
+}
+
+impl Dumper for EthtoolQueue {
+    fn dump_model(
+        &self,
+        ctx: &CommonFieldContext,
+        model: &model::Model,
+        output: &mut dyn Write,
+        round: &mut usize,
+        comma_flag: bool,
+    ) -> Result<IterExecResult> {
+        let mut queues = Vec::new();
+        for nic in model.network.interfaces.values() {
+            for queue in &nic.queues {
+                queues.push(queue);
+            }
+        }
+
+        // Return if we filtered everything.
+        if queues.is_empty() {
+            return Ok(IterExecResult::Skip);
+        }
+
+        let mut json_output = json!([]);
+
+        queues
+            .into_iter()
+            .map(|queue| {
+                match self.opts.output_format {
+                    Some(OutputFormat::Raw) | None => write!(
+                        output,
+                        "{}",
+                        print::dump_raw(
+                            &self.fields,
+                            ctx,
+                            queue,
+                            *round,
+                            self.opts.repeat_title,
+                            self.opts.disable_title,
+                            self.opts.raw
+                        )
+                    )?,
+                    Some(OutputFormat::Csv) => write!(
+                        output,
+                        "{}",
+                        print::dump_csv(
+                            &self.fields,
+                            ctx,
+                            queue,
+                            *round,
+                            self.opts.disable_title,
+                            self.opts.raw
+                        )
+                    )?,
+                    Some(OutputFormat::Tsv) => write!(
+                        output,
+                        "{}",
+                        print::dump_tsv(
+                            &self.fields,
+                            ctx,
+                            queue,
+                            *round,
+                            self.opts.disable_title,
+                            self.opts.raw
+                        )
+                    )?,
+                    Some(OutputFormat::KeyVal) => write!(
+                        output,
+                        "{}",
+                        print::dump_kv(&self.fields, ctx, queue, self.opts.raw)
+                    )?,
+                    Some(OutputFormat::Json) => {
+                        let par = print::dump_json(&self.fields, ctx, queue, self.opts.raw);
+                        json_output.as_array_mut().unwrap().push(par);
+                    }
+                    Some(OutputFormat::OpenMetrics) => write!(
+                        output,
+                        "{}",
+                        print::dump_openmetrics(&self.fields, ctx, queue)
+                    )?,
+                }
+                *round += 1;
+                Ok(())
+            })
+            .collect::<Result<Vec<_>>>()?;
+
+        match (self.opts.output_format, comma_flag) {
+            (Some(OutputFormat::Json), true) => write!(output, ",{}", json_output)?,
+            (Some(OutputFormat::Json), false) => write!(output, "{}", json_output)?,
+            (Some(OutputFormat::OpenMetrics), _) => (),
+            _ => write!(output, "\n")?,
+        };
+
+        Ok(IterExecResult::Success)
+    }
+}

below/dump/src/lib.rs

@@ -46,6 +46,7 @@ pub mod btrfs;
 pub mod cgroup;
 pub mod command;
 pub mod disk;
+pub mod ethtool;
 pub mod iface;
 pub mod network;
 pub mod print;
@@ -115,6 +116,7 @@ pub type NetworkField = DumpField<model::NetworkModelFieldId>;
 pub type IfaceField = DumpField<model::SingleNetModelFieldId>;
 // Essentially the same as NetworkField
 pub type TransportField = DumpField<model::NetworkModelFieldId>;
+pub type EthtoolQueueField = DumpField<model::SingleQueueModelFieldId>;
 
 fn get_advance(
     logger: slog::Logger,
@@ -520,5 +522,42 @@ pub fn run(
                 errs,
             )
         }
+        DumpCommand::EthtoolQueue {
+            fields,
+            opts,
+            pattern,
+        } => {
+            let (time_begin, time_end, advance) =
+                get_advance(logger, dir, host, port, snapshot, &opts)?;
+            let default = opts.everything || opts.default;
+            let detail = opts.everything || opts.detail;
+            let fields = if let Some(pattern_key) = pattern {
+                parse_pattern(filename, pattern_key, "ethtool_queue")
+            } else {
+                fields
+            };
+            let fields = expand_fields(
+                match fields.as_ref() {
+                    Some(fields) if !default => fields,
+                    _ => command::DEFAULT_ETHTOOL_QUEUE_FIELDS,
+                },
+                detail,
+            );
+            let ethtool = ethtool::EthtoolQueue::new(&opts, fields);
+            let mut output: Box<dyn Write> = match opts.output.as_ref() {
+                Some(file_path) => Box::new(File::create(file_path)?),
+                None => Box::new(io::stdout()),
+            };
+            dump_timeseries(
+                advance,
+                time_begin,
+                time_end,
+                &ethtool,
+                output.as_mut(),
+                opts.output_format,
+                opts.br,
+                errs,
+            )
+        }
     }
 }