Update selection metric rollups in consumption history

- rewrite remove_duplicate_timestamps to SQL
- rewrite also "group by tenant" feature
- reduce returned object from BIG base_rollup query

app/models/chargeback/consumption_history.rb

@@ -14,15 +14,18 @@ def self.for_report(cb_class, options, region)
 
  next unless options.include_metrics?
 
- records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
+ records = MetricRollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
  records = cb_class.where_clause(records, options, region)
- records = Metric::Helper.remove_duplicate_timestamps(records)
+ records = uniq_timestamp_record_map(records, options.group_by_tenant?)
+
  next if records.empty?
  _log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")
 
  # we are building hash with grouped calculated values
  # values are grouped by resource_id and timestamp (query_start_time...query_end_time)
- options.group_with(records).each_value do |metric_rollup_records|
+
+ records.each_value do |rollup_record_ids|
+ metric_rollup_records = base_rollup.where(:id => rollup_record_ids).to_a
  consumption = ConsumptionWithRollups.new(metric_rollup_records, query_start_time, query_end_time)
  yield(consumption) unless consumption.consumed_hours_in_interval.zero?
  end
@@ -43,5 +46,29 @@ def self.base_rollup_scope
  end
 
  private_class_method :base_rollup_scope
+
+ def self.uniq_timestamp_record_map(report_scope, group_by_tenant = false)
+ main_select = MetricRollup.select(:id, :resource_id).arel.ast.to_sql
+ .gsub("SELECT", "DISTINCT ON (resource_type, resource_id, timestamp)")
+ .gsub(/ FROM.*$/, '')
+
+ query = report_scope.select(main_select)
+ .order(:resource_type, :resource_id, :timestamp)
+ .order("created_on DESC")
+
+ rows = ActiveRecord::Base.connection.select_rows(query.to_sql)
+
+ if group_by_tenant
+ vms = Hash[Vm.where(:id => rows.map(&:second)).pluck(:id, :tenant_id)]
+ end
+
+ rows.each_with_object({}) do |(id, resource_id), result|
+ resource_id = vms[resource_id] if group_by_tenant
+ result[resource_id] ||= []
+ result[resource_id] << id
+ end
+ end
+
+ private_class_method :uniq_timestamp_record_map
  end
 end

app/models/chargeback/report_options.rb

@@ -121,10 +121,6 @@ def tenant_for(consumption)
  consumption.resource.tenant
  end
 
- def group_with(records)
- group_by_tenant? ? records.group_by { |x| x.resource.tenant.id } : records.group_by(&:resource_id)
- end
-
  def classification_for(consumption)
  tag = consumption.tag_names.find { |x| x.starts_with?(groupby_tag) } # 'department/*'
  tag = tag.split('/').second unless tag.blank? # 'department/finance' -> 'finance'