Merge pull request #14332 from blomquisg/splitting_gap_collection

Split metric collections into smaller intervals

app/models/metric/ci_mixin/capture.rb

@@ -17,6 +17,20 @@ def queue_name_for_metrics_collection
  ems.metrics_collector_queue_name
  end
 
+ def split_capture_intervals(interval_name, start_time, end_time, threshold = 1.day)
+ raise _("Start time must be earlier than End time") if start_time > end_time
+ # Create an array of ordered pairs from start_time and end_time so that each ordered pair is contained
+ # within the threshold. Then, reverse it so the newest ordered pair is first:
+ # start_time = 2017/01/01 12:00:00, end_time = 2017/01/04 12:00:00
+ # [[interval_name, 2017-01-03 12:00:00 UTC, 2017-01-04 12:00:00 UTC],
+ # [interval_name, 2017-01-02 12:00:00 UTC, 2017-01-03 12:00:00 UTC],
+ # [interval_name, 2017-01-01 12:00:00 UTC, 2017-01-02 12:00:00 UTC]]
+ (start_time.utc..end_time.utc).step_value(threshold).each_cons(2).collect do |s_time, e_time|
+ [interval_name, s_time, e_time]
+ end.reverse
+ end
+ private :split_capture_intervals
+
  def perf_capture_queue(interval_name, options = {})
  start_time = options[:start_time]
  end_time = options[:end_time]
@@ -32,27 +46,20 @@ def perf_capture_queue(interval_name, options = {})
 
  log_target = "#{self.class.name} name: [#{name}], id: [#{id}]"
 
- # Determine what items we should be queuing up
- items = []
  cb = nil
  if interval_name == 'historical'
  start_time = Metric::Capture.historical_start_time if start_time.nil?
  end_time = Time.now.utc if end_time.nil?
-
- start_hour = start_time
- while start_hour != end_time
- end_hour = start_hour + 1.day
- end_hour = end_time if end_hour > end_time
- items.unshift([interval_name, start_hour, end_hour])
- start_hour = end_hour
- end
  else
- items << [interval_name]
- items[0] << start_time << end_time unless start_time.nil?
-
+ start_time = last_perf_capture_on unless start_time
+ end_time = Time.now.utc unless end_time
  cb = {:class_name => self.class.name, :instance_id => id, :method_name => :perf_capture_callback, :args => [[task_id]]} if task_id
  end
 
+ # Determine what items we should be queuing up
+ items = [interval_name]
+ items = split_capture_intervals(interval_name, start_time, end_time) if start_time
+
  # Queue up the actual items
  queue_item = {
  :class_name => self.class.name,

spec/models/metric/ci_mixin/capture_spec.rb

@@ -36,6 +36,48 @@ def expected_stats_period_end
  parse_datetime('2013-08-28T12:41:40Z')
  end
 
+ context "#perf_capture_queue" do
+ def test_perf_capture_queue(time_since_last_perf_capture, total_queue_items, verify_queue_items_count)
+ # There are usually some lingering queue items from creating the provider above. Notably `stop_event_monitor`
+ MiqQueue.delete_all
+ Timecop.freeze do
+ start_time = (Time.now.utc - time_since_last_perf_capture)
+ @vm.last_perf_capture_on = start_time
+ @vm.perf_capture_queue("realtime")
+ expect(MiqQueue.count).to eq total_queue_items
+
+ # make sure the queue items are in the correct order
+ queue_items = MiqQueue.order(:id).limit(verify_queue_items_count)
+ days_ago = (time_since_last_perf_capture.to_i / 1.day.to_i).days
+ partial_days = time_since_last_perf_capture - days_ago
+ interval_start_time = (start_time + days_ago).utc
+ interval_end_time = (interval_start_time + partial_days).utc
+ queue_items.each do |q_item|
+ q_start_time, q_end_time = q_item.args
+ expect(q_start_time).to be_same_time_as interval_start_time
+ expect(q_end_time).to be_same_time_as interval_end_time
+ interval_end_time = interval_start_time
+ # if the collection threshold is ever parameterized, then this 1.day will have to change
+ interval_start_time -= 1.day
+ end
+ end
+ end
+
+ it "splits up long perf_capture durations for old last_perf_capture_on" do
+ # test when last perf capture was many days ago
+ # total queue items == 11
+ # verify last 3 queue items
+ test_perf_capture_queue(10.days + 5.hours + 23.minutes, 11, 3)
+ end
+
+ it "does not get confused when dealing with a single day" do
+ # test when perf capture is just a few hours ago
+ # total queue items == 1
+ # verify last 1 queue item
+ test_perf_capture_queue(0.days + 2.hours + 5.minutes, 1, 1)
+ end
+ end
+
  context "2 collection periods total, end of 1. period has incomplete stat" do
  ###################################################################################################################
  # DESCRIPTION FOR: net_usage_rate_average