From 02bacd67db6e787b5016ac4a7cb64b4e14fce22c Mon Sep 17 00:00:00 2001
From: Steven McDonald <steven.mcdonald@usabilla.com>
Date: Wed, 7 Aug 2019 10:22:06 +0200
Subject: [PATCH] Export the oldest and newest timekeys in each buffer

A metric listing all timekeys in each buffer was added to fluentd
in <https://github.com/fluent/fluentd/pull/2343>. This exports the
oldest and newest values in that list as Prometheus metrics. See
discussion in
<https://github.com/fluent/fluent-plugin-prometheus/issues/89> for
context and rationale.

Signed-off-by: Steven McDonald <steven.mcdonald@usabilla.com>
---
 lib/fluent/plugin/in_prometheus_monitor.rb       | 16 +++++++++++++++-
 .../plugin/in_prometheus_output_monitor.rb       | 12 ++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/lib/fluent/plugin/in_prometheus_monitor.rb b/lib/fluent/plugin/in_prometheus_monitor.rb
index 63a9570..7ea54ac 100644
--- a/lib/fluent/plugin/in_prometheus_monitor.rb
+++ b/lib/fluent/plugin/in_prometheus_monitor.rb
@@ -45,6 +45,12 @@ def configure(conf)
     def start
       super
 
+      @buffer_newest_timekey = @registry.gauge(
+        :fluentd_status_buffer_newest_timekey,
+        'Newest timekey in buffer.')
+      @buffer_oldest_timekey = @registry.gauge(
+        :fluentd_status_buffer_oldest_timekey,
+        'Oldest timekey in buffer.')
       buffer_queue_length = @registry.gauge(
         :fluentd_status_buffer_queue_length,
         'Current buffer queue length.')
@@ -65,11 +71,19 @@ def start
 
     def update_monitor_info
       @monitor_agent.plugins_info_all.each do |info|
+        label = labels(info)
+
         @monitor_info.each do |name, metric|
           if info[name]
-            metric.set(labels(info), info[name])
+            metric.set(label, info[name])
           end
         end
+
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @buffer_newest_timekey.set(label, timekeys.max)
+          @buffer_oldest_timekey.set(label, timekeys.min)
+        end
       end
     end
 
diff --git a/lib/fluent/plugin/in_prometheus_output_monitor.rb b/lib/fluent/plugin/in_prometheus_output_monitor.rb
index 15f932f..1659f69 100644
--- a/lib/fluent/plugin/in_prometheus_output_monitor.rb
+++ b/lib/fluent/plugin/in_prometheus_output_monitor.rb
@@ -60,6 +60,12 @@ def start
       super
 
       @metrics = {
+        buffer_newest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_newest_timekey,
+          'Newest timekey in buffer.'),
+        buffer_oldest_timekey: @registry.gauge(
+          :fluentd_output_status_buffer_oldest_timekey,
+          'Oldest timekey in buffer.'),
         buffer_queue_length: @registry.gauge(
           :fluentd_output_status_buffer_queue_length,
           'Current buffer queue length.'),
@@ -123,6 +129,12 @@ def update_monitor_info
           end
         end
 
+        timekeys = info["buffer_timekeys"]
+        if timekeys && !timekeys.empty?
+          @metrics[:buffer_newest_timekey].set(label, timekeys.max)
+          @metrics[:buffer_oldest_timekey].set(label, timekeys.min)
+        end
+
         if info['instance_variables']
           instance_vars_info.each do |name, metric|
             if info['instance_variables'][name]