Merge pull request #17538 from lpichler/replace_remove_duplicate_time…

…stamp_by_sql_version

Replace remove duplicate timestamp by sql version for chargeback

app/models/chargeback/consumption_history.rb

@@ -14,15 +14,18 @@ def self.for_report(cb_class, options, region)
 
         next unless options.include_metrics?
 
-        records = base_rollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
+        records = MetricRollup.where(:timestamp => query_start_time...query_end_time, :capture_interval_name => 'hourly')
         records = cb_class.where_clause(records, options, region)
-        records = Metric::Helper.remove_duplicate_timestamps(records)
+        records = uniq_timestamp_record_map(records, options.group_by_tenant?)
+
         next if records.empty?
         _log.info("Found #{records.length} records for time range #{[query_start_time, query_end_time].inspect}")
 
         # we are building hash with grouped calculated values
         # values are grouped by resource_id and timestamp (query_start_time...query_end_time)
-        options.group_with(records).each_value do |metric_rollup_records|
+
+        records.each_value do |rollup_record_ids|
+          metric_rollup_records = base_rollup.where(:id => rollup_record_ids).to_a
           consumption = ConsumptionWithRollups.new(metric_rollup_records, query_start_time, query_end_time)
           yield(consumption) unless consumption.consumed_hours_in_interval.zero?
         end
@@ -43,5 +46,29 @@ def self.base_rollup_scope
     end
 
     private_class_method :base_rollup_scope
+
+    def self.uniq_timestamp_record_map(report_scope, group_by_tenant = false)
+      main_select = MetricRollup.select(:id, :resource_id).arel.ast.to_sql
+                                .gsub("SELECT", "DISTINCT ON (resource_type, resource_id, timestamp)")
+                                .gsub(/ FROM.*$/, '')
+
+      query = report_scope.select(main_select)
+                          .order(:resource_type, :resource_id, :timestamp)
+                          .order("created_on DESC")
+
+      rows = ActiveRecord::Base.connection.select_rows(query.to_sql)
+
+      if group_by_tenant
+        vms = Hash[Vm.where(:id => rows.map(&:second)).pluck(:id, :tenant_id)]
+      end
+
+      rows.each_with_object({}) do |(id, resource_id), result|
+        resource_id = vms[resource_id] if group_by_tenant
+        result[resource_id] ||= []
+        result[resource_id] << id
+      end
+    end
+
+    private_class_method :uniq_timestamp_record_map
   end
 end

app/models/chargeback/report_options.rb

@@ -121,10 +121,6 @@ def tenant_for(consumption)
       consumption.resource.tenant
     end
 
-    def group_with(records)
-      group_by_tenant? ? records.group_by { |x| x.resource.tenant.id } : records.group_by(&:resource_id)
-    end
-
     def classification_for(consumption)
       tag = consumption.tag_names.find { |x| x.starts_with?(groupby_tag) } # 'department/*'
       tag = tag.split('/').second unless tag.blank? # 'department/finance' -> 'finance'