traffic_ctl: Add support to monitor metrics. (#9423)

The tool now accepts passing -i(time interval) and -c(count) to track one or
more metrics.

Author: brbzull0

doc/appendices/command-line/traffic_ctl.en.rst

@@ -330,6 +330,36 @@ traffic_ctl metric
 
    Error output available if ``--format pretty`` is specified.
 
+.. program:: traffic_ctl metric
+.. option:: monitor [-i, -c] METRIC [METRIC...]
+
+   Display the current value of the specified metric(s) using an interval time
+   and a count value. Use ``-i`` to set the interval time between requests, and
+   ``-c`` to set the number of requests the program will send in total per metric.
+   Note that the metric will display `+` or `-` depending on the value of the last
+   metric and the current being shown, if current is greater, then  `+` will be
+   added beside the metric value, `-` if the last value is less than current,
+   and no symbol is the same.
+
+   Example:
+
+   .. code-block:: bash
+
+      $ traffic_ctl  metric monitor proxy.process.eventloop.time.min.10s -i 2 -c 10
+      proxy.process.eventloop.time.min.10s: 4025085
+      proxy.process.eventloop.time.min.10s: 4025085
+      proxy.process.eventloop.time.min.10s: 4025085
+      proxy.process.eventloop.time.min.10s: 4025085
+      proxy.process.eventloop.time.min.10s: 4011194 -
+      proxy.process.eventloop.time.min.10s: 4011194
+      proxy.process.eventloop.time.min.10s: 4011194
+      proxy.process.eventloop.time.min.10s: 4011194
+      proxy.process.eventloop.time.min.10s: 4011194
+      proxy.process.eventloop.time.min.10s: 4018669 +
+      --- metric monitor statistics ---
+      ┌ proxy.process.eventloop.time.min.10s
+      └─ min/avg/max = 4011194/4017498/4025085
+
 .. _traffic-control-command-server:
 
 traffic_ctl server

src/traffic_ctl/CtrlCommands.cc

@@ -20,6 +20,8 @@
  */
 #include <fstream>
 #include <unordered_map>
+#include <chrono>
+#include <thread>
 
 #include "CtrlCommands.h"
 #include "jsonrpc/CtrlRPCRequests.h"
@@ -241,6 +243,9 @@ MetricCommand::MetricCommand(ts::Arguments *args) : RecordCommand(args)
   } else if (args->get(ZERO_STR)) {
     _printer      = std::make_unique<GenericPrinter>(printOpts);
     _invoked_func = [&]() { metric_zero(); };
+  } else if (args->get(MONITOR_STR)) {
+    _printer      = std::make_unique<MetricRecordPrinter>(printOpts);
+    _invoked_func = [&]() { metric_monitor(); };
   }
 }
 
@@ -277,6 +282,86 @@ MetricCommand::metric_zero()
 
   [[maybe_unused]] auto const &response = invoke_rpc(request);
 }
+
+void
+MetricCommand::metric_monitor()
+{
+  ts::ArgumentData const &arg = get_parsed_arguments()->get(MONITOR_STR);
+  std::string err_text;
+
+  //
+  // Note: if any of the string->number fails, the exception will be caught by the invoke function from the ArgParser.
+  //
+  const int32_t count    = std::stoi(get_parsed_arguments()->get("count").value());
+  const int32_t interval = std::stoi(get_parsed_arguments()->get("interval").value());
+  if (count <= 0 || interval <= 0) {
+    throw std::runtime_error(
+      ts::bwprint(err_text, "monitor: invalid input, count({}), interval({}) should be > than '0'", count, interval));
+  }
+
+  // keep track of each metric
+  struct ctx {
+    float min{std::numeric_limits<float>::max()};
+    float max{std::numeric_limits<float>::lowest()};
+    float sum{0.0f};
+    float last{0.0f};
+  };
+
+  // Keep track of the requested metric(s), we support more than one at the same time.
+  std::unordered_map<std::string, ctx> summary;
+
+  for (int idx = 0;; idx++) {
+    // Request will hold all metrics in a single message.
+    shared::rpc::JSONRPCResponse const &resp = record_fetch(arg, shared::rpc::NOT_REGEX, RecordQueryType::METRIC);
+
+    if (resp.is_error()) { // something went wrong in the server, report it.
+      _printer->write_output(resp);
+      return;
+    }
+
+    auto const &response = resp.result.as<shared::rpc::RecordLookUpResponse>();
+    if (response.errorList.size() && response.recordList.size() == 0) {
+      // nothing to be done or report, use '-f rpc'  for details.
+      break;
+    }
+
+    for (auto &&rec : response.recordList) { // requested metric(s)
+      auto &s = summary[rec.name];           // We will update it.
+      // Note: To avoid
+      const float val = std::stof(rec.currentValue);
+
+      s.sum += val;
+      s.max = std::max<float>(s.max, val);
+      s.min = std::min<float>(s.min, val);
+      std::string symbol;
+      if (idx > 0) {
+        if (val > s.last) {
+          symbol = "+";
+        } else if (val < s.last) {
+          symbol = "-";
+        }
+      }
+      s.last = val;
+      _printer->write_output(ts::bwprint(err_text, "{}: {} {}", rec.name, rec.currentValue, symbol));
+    }
+    if (idx == count - 1) {
+      break;
+    }
+    std::this_thread::sleep_for(std::chrono::seconds(interval));
+  }
+  if (summary.size() == 0) {
+    // nothing to report.
+    return;
+  }
+
+  _printer->write_output("--- metric monitor statistics ---");
+
+  for (auto &&item : summary) {
+    ctx const &s  = item.second;
+    const int avg = s.sum / count;
+    _printer->write_output(ts::bwprint(err_text, "┌ {}\n└─ min/avg/max = {:.5}/{}/{:.5}", item.first, s.min, avg, s.max));
+  }
+}
 //------------------------------------------------------------------------------------------------------------------------------------
 // TODO, let call the super const
 HostCommand::HostCommand(ts::Arguments *args) : CtrlCommand(args)

src/traffic_ctl/CtrlCommands.h

@@ -144,12 +144,14 @@ class MetricCommand : public RecordCommand
 {
   static inline const std::string CLEAR_STR{"clear"};
   static inline const std::string ZERO_STR{"zero"};
+  static inline const std::string MONITOR_STR{"monitor"};
 
   void metric_get();
   void metric_match();
   void metric_describe();
   void metric_clear();
   void metric_zero();
+  void metric_monitor();
 
 public:
   MetricCommand(ts::Arguments *args);

src/traffic_ctl/traffic_ctl.cc

@@ -123,8 +123,11 @@ main(int argc, const char **argv)
                              [&]() { command->execute(); }); // not implemented
   metric_command.add_command("match", "Get metrics matching a regular expression", "", MORE_THAN_ZERO_ARG_N,
                              [&]() { command->execute(); });
-  metric_command.add_command("monitor", "Display the value of a metric over time", "", MORE_THAN_ZERO_ARG_N,
-                             [&]() { CtrlUnimplementedCommand("monitor"); }); // not implemented
+  metric_command
+    .add_command("monitor", "Display the value of a metric(s) over time", "", MORE_THAN_ZERO_ARG_N, [&]() { command->execute(); })
+    .add_example_usage("traffic_ctl metric monitor METRIC -i 3 -c 10")
+    .add_option("--count", "-c", "Stop after requesting count metrics.", "", 1, "10")
+    .add_option("--interval", "-i", "Wait interval seconds between sending each metric request. Minimum value is 1s.", "", 1, "5");
   metric_command.add_command("zero", "Clear one or more metric values", "", MORE_THAN_ONE_ARG_N, [&]() { command->execute(); });
 
   // plugin command