Check warn/crit against a linear projection of retrieved data series …

…into the future

README.md

@@ -23,6 +23,7 @@ check_graphite accepts the following options:
 * `-c`: critical threshold for the metric
 * `-t`: timeout after which the metric should be considered unknown
 * `--ignore-missing`: return `OK` when the metric doesn't exist yet e.g. errors have not occurred
+* `--projection`: Warn on a value linearly extrapolated into the future, defaults to "2days"
 
 ## How it works
 
@@ -31,3 +32,5 @@ points collected, it then checks the value against supplied thresholds. Threshol
 in the format given in [The Nagios Developer Guidelines](http://nagiosplug.sourceforge.net/developer-guidelines.html#THRESHOLDFORMAT).
 
 NaN values are not taken into account in the average
+
+When running with --projection, note that you probably want to set a bigger --from window so you get a reasonable projection. Both --from and --projection accepts arguments in Graphite format.

check_graphite.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
 
   # specify any dependencies here; for example:
   # s.add_development_dependency "rspec"
+  s.add_runtime_dependency "linear-regression"
   s.add_runtime_dependency "nagios_check"
 
   s.add_development_dependency "rake"

lib/attime.rb

@@ -0,0 +1,61 @@
+# This helper is trying to replicate the relative parts of
+# https://github.com/graphite-project/graphite-web/blob/master/webapp/graphite/render/attime.py
+
+module CheckGraphite
+  DAY = 86400
+  UNITS = {
+    "s" => "second",
+    "se" => "second",
+    "sec" => "second",
+    "second" => 1,
+    "seconds" => "second",
+
+    "min" => "minute",
+    "minute" => 60,
+    "minutes" => "minute",
+
+    "ho" => "hour",
+    "hour" => 3600,
+    "hours" => "hour",
+
+    "d" => "day",
+    "da" => "day",
+    "day" => DAY,
+    "days" => "day",
+
+    "week" => 7 * DAY,
+    "weeks" => "week",
+
+    "m" => nil, # min or mon?
+    "mon" => "month",
+    "month" => 30 * DAY,
+    "months" => "month",
+
+    "y" => "year",
+    "ye" => "year",
+    "year" => 365 * DAY,
+    "years" => "year",
+  }
+  EXPR = /([+-])?([0-9]*)(.*)/
+
+  def self.attime(text, time = Time.now)
+    match = EXPR.match(text)
+    raise "Unparseable time period #{text}" unless match
+    sign, scalar, unit = match[1..3]
+    raise "Missing scalar in time #{text}" if scalar == ""
+    raise "Missing unit in #{text}" if unit == ""
+    time + Integer(scalar) * lookup(unit) * (sign == "-" ? -1 : 1)
+  end
+
+  private
+
+  def self.lookup(unit)
+    x = UNITS[unit.downcase]
+    raise "Bad unit #{unit}" unless x
+    if x.is_a? String
+      lookup(x)
+    else
+      x
+    end
+  end
+end

lib/check_graphite.rb

@@ -2,11 +2,13 @@
 require "json"
 require "net/https"
 require "check_graphite/version"
+require 'check_graphite/projection'
 
 module CheckGraphite
 
   class Command
     include NagiosCheck
+    include Projection
 
     on "--endpoint ENDPOINT", "-H ENDPOINT", :mandatory
     on "--metric METRIC", "-M METRIC", :mandatory

lib/check_graphite/projection.rb

@@ -0,0 +1,39 @@
+require 'attime'
+require 'bigdecimal'
+require 'nagios_check'
+require 'linear-regression'
+
+module CheckGraphite
+  module Projection
+    def self.included(base)
+      base.on '--projection FUTURE_TIMEFRAME', :default => '2days' do |timeframe|
+        options.send('processor=', method(:projected_value))
+        options.send('timeframe=', timeframe)
+      end
+    end
+
+    def projected_value(datapoints)
+      ys, xs = datapoints.transpose
+      lr = Regression::Linear.new(xs, ys)
+      future = CheckGraphite.attime(options.timeframe, xs[-1])
+      value = lr.predict(future.to_i)
+      p = Regression::CorrelationCoefficient.new(xs, ys).pearson
+      store_value options.name, value
+      # Unfortunately, nagios_check converts both primary and
+      # secondary values to float. Hence manual assignment instead:
+      @values['p-value'] = format_float(p.abs)
+      store_message "#{options.name}=#{format_float(value)} in #{options.timeframe}"
+      return value
+    end
+
+    private
+
+     def format_float(v)
+       if v.nan?
+         'undefined'
+       else
+         BigDecimal.new(v, 3).to_s('F')
+       end
+     end
+  end
+end

spec/attime_spec.rb

@@ -0,0 +1,31 @@
+require 'attime'
+
+describe '#attime' do
+  subject do |example|
+    delta = example.metadata[:description_args][0]
+    CheckGraphite.attime(delta)
+  end
+
+  def timedelta(n)
+    Time.now + n
+  end
+
+  it("10days") { should be_within(0.9).of(timedelta(10 * 86400)) }
+  it("0days") { should be_within(0.9).of(timedelta(0)) }
+  it("-10days") { should be_within(0.9).of(timedelta(-10 * 86400)) }
+  it("5seconds") { should be_within(0.9).of(timedelta(5)) }
+  it("5minutes") { should be_within(0.9).of(timedelta(5 * 60)) }
+  it("5hours") { should be_within(0.9).of(timedelta(5 * 3600)) }
+  it("5weeks") { should be_within(0.9).of(timedelta(86400 * 7 * 5)) }
+  it("1month") { should be_within(0.9).of(timedelta(30 * 86400)) }
+  it("2months") { should be_within(0.9).of(timedelta(60 * 86400)) }
+  it("12months") { should be_within(0.9).of(timedelta(360 * 86400)) }
+  it("1year") { should be_within(0.9).of(timedelta(365 * 86400)) }
+  it("2years") { should be_within(0.9).of(timedelta(730 * 86400)) }
+
+  it(1) { expect { subject }.to raise_error(/string/i) }
+  it("Something") { expect { subject }.to raise_error(RuntimeError) }
+  it("1m") { expect { subject }.to raise_error(/bad unit/i) }
+  it("10") { expect { subject }.to raise_error(/unit/) }
+  it("month") { expect { subject }.to raise_error(/scalar/) }
+end

spec/projection_spec.rb

@@ -0,0 +1,80 @@
+require 'check_graphite/projection'
+require 'nagios_check'
+
+describe describe CheckGraphite::Projection do
+  let :check do
+    Class.new do
+      attr_reader :values
+      include NagiosCheck
+      include CheckGraphite::Projection
+    end.new
+  end
+
+  describe 'primary value' do
+    subject do
+      check.prepare
+      check.send(:parse_options, ['--projection', projection])
+      check.options.processor.call(datapoints)
+      check.values.first[1]
+    end
+
+    context 'given a linearly decreasing series and a projection of 2 sec' do
+      let(:projection) { '2sec' }
+      let(:datapoints) { [[10,0], [9,1], [8,2]] }
+
+      it { should be_within(0.01).of(6) }
+    end
+
+    context 'given a constant series and a projection of 2 sec' do
+      let(:projection) { '2sec' }
+      let(:datapoints) { [[10,0], [10,1], [10,2]] }
+
+      it { should be_within(0.01).of(10) }
+    end
+  end
+
+  describe 'p-value' do
+    subject do
+      check.prepare
+      check.send(:parse_options, ['--projection', projection])
+      check.options.processor.call(datapoints)
+      check.values['p-value']
+    end
+
+    context 'given a constant series (where Pearson is undefined)' do
+      let(:projection) { '2sec' }
+      let(:datapoints) { [[10,0], [10,1], [10,2]] }
+      it { should eq('undefined') }
+    end
+
+    context 'given a linearly decreasing series' do
+      let(:projection) { '2sec' }
+      let(:datapoints) { [[10,0], [9,1], [8,2]] }
+
+      it { should eq("1.0") }
+    end
+  end
+
+  describe 'integration test' do
+    before do
+      FakeWeb.register_uri(
+        :get, "http://your.graphite.host/render?target=collectd.somebox.load.load.midterm&from=-30seconds&format=json",
+        :body => '[{"target": "collectd.somebox.load.load.midterm", "datapoints": [[1.0, 1339512060], [2.0, 1339512120], [6.0, 1339512180], [7.0, 1339512240]]}]',
+        :content_type => "application/json"
+      )
+    end
+
+    it 'should output value, projection interval and p-value' do
+      stub_const("ARGV", %w{
+        -H http://your.graphite.host/render
+        -M collectd.somebox.load.load.midterm
+        -c 0:10
+        --projection 5min
+        --name ze-name
+      })
+      c = CheckGraphite::Command.new
+      STDOUT.should_receive(:puts).with(match(/ze-name.*in 5min.*p-value=0.9/))
+      lambda { c.run }.should raise_error SystemExit
+    end
+  end
+end