Add indexes for cloud entities

db/migrate/20180123130030_clean_up_duplicates_in_cloud_tables.rb

@@ -0,0 +1,90 @@
+class CleanUpDuplicatesInCloudTables < ActiveRecord::Migration[5.0]
+  class AvailabilityZone < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudService < ApplicationRecord; end
+  class CloudTenant < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudObjectStoreContainer < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudObjectStoreObject < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudNetwork < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudVolume < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudVolumeBackup < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudVolumeSnapshot < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class CloudResourceQuota < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class Flavor < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class ResourceGroup < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class HostAggregate < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class OrchestrationTemplate < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class OrchestrationStack < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+  class Vm < ApplicationRecord
+    self.inheritance_column = :_type_disabled
+  end
+
+  # class OrchestrationTemplateCatalog < ApplicationRecord; end # FIXME
+  # class AuthKeyPair < ApplicationRecord; end # FIXME
+
+  def cleanup_duplicate_data_delete_all(model, unique_index_columns)
+    # 'ORDER BY id DESC' keeps the highest `id`. `ORDER BY id ASC` (more common case) would keep the lowest `id`.
+    query = <<-SQL
+      DELETE FROM #{model.table_name}
+        WHERE id IN (SELECT id
+                      FROM (SELECT id, ROW_NUMBER() OVER (partition BY #{unique_index_columns.join(",")} ORDER BY id DESC) AS rnum
+                             FROM #{model.table_name}) t
+                      WHERE t.rnum > 1);
+    SQL
+    connection.execute(query)
+  end
+
+  UNIQUE_INDEXES_FOR_MODELS = {
+    # Just having :ems_id & :ems_ref
+    CloudService              => %i(ems_id ems_ref),
+    ResourceGroup             => %i(ems_id ems_ref),
+    CloudTenant               => %i(ems_id ems_ref),
+    Flavor                    => %i(ems_id ems_ref),
+    AvailabilityZone          => %i(ems_id ems_ref),
+    HostAggregate             => %i(ems_id ems_ref),
+    OrchestrationTemplate     => %i(ems_id ems_ref),
+    OrchestrationStack        => %i(ems_id ems_ref),
+    CloudVolume               => %i(ems_id ems_ref),
+    CloudVolumeBackup         => %i(ems_id ems_ref),
+    CloudVolumeSnapshot       => %i(ems_id ems_ref),
+    CloudResourceQuota        => %i(ems_id ems_ref),
+    CloudObjectStoreContainer => %i(ems_id ems_ref),
+    CloudObjectStoreObject    => %i(ems_id ems_ref),
+    Vm                        => %i(ems_id ems_ref),
+  }.freeze
+
+  def up
+    UNIQUE_INDEXES_FOR_MODELS.each do |model, unique_indexes_columns|
+      say_with_time("Cleanup duplicate data for model #{model}") do
+        cleanup_duplicate_data_delete_all(model, unique_indexes_columns)
+      end
+    end
+  end
+end

db/migrate/20180123130054_add_unique_indexes_to_cloud_tables.rb

@@ -0,0 +1,20 @@
+class AddUniqueIndexesToCloudTables < ActiveRecord::Migration[5.0]
+  def change
+    add_index :availability_zones,            %i(ems_id ems_ref), :unique => true
+    add_index :cloud_services,                %i(ems_id ems_ref), :unique => true
+    add_index :cloud_tenants,                 %i(ems_id ems_ref), :unique => true
+    add_index :cloud_object_store_containers, %i(ems_id ems_ref), :unique => true
+    add_index :cloud_object_store_objects,    %i(ems_id ems_ref), :unique => true
+    add_index :cloud_networks,                %i(ems_id ems_ref), :unique => true
+    add_index :cloud_volumes,                 %i(ems_id ems_ref), :unique => true
+    add_index :cloud_volume_backups,          %i(ems_id ems_ref), :unique => true
+    add_index :cloud_volume_snapshots,        %i(ems_id ems_ref), :unique => true
+    add_index :cloud_resource_quotas,         %i(ems_id ems_ref), :unique => true
+    add_index :flavors,                       %i(ems_id ems_ref), :unique => true
+    add_index :resource_groups,               %i(ems_id ems_ref), :unique => true
+    add_index :host_aggregates,               %i(ems_id ems_ref), :unique => true
+    add_index :orchestration_templates,       %i(ems_id ems_ref), :unique => true
+    add_index :orchestration_stacks,          %i(ems_id ems_ref), :unique => true
+    add_index :vms,                           %i(ems_id ems_ref), :unique => true
+  end
+end

spec/migrations/20180123130030_clean_up_duplicates_in_cloud_tables_spec.rb

@@ -0,0 +1,95 @@
+require_migration
+
+describe CleanUpDuplicatesInCloudTables do
+  def model_unique_keys(model)
+    models[model]
+  end
+
+  def models
+    described_class::UNIQUE_INDEXES_FOR_MODELS
+  end
+
+  def model_stub(model)
+    migration_stub(model.to_s.to_sym)
+  end
+
+  def create_test_data(model)
+    build_data(model, 10, "string_1")
+    build_data(model, 10, "string_1")
+    build_data(model, 10, "string_2")
+    build_data(model, 11, "string_1")
+    build_data(model, 11, "string_1")
+    build_data(model, 11, "string_1")
+    build_data(model, 11, "string_1")
+    build_data(model, 12, "string_1")
+    build_data(model, nil, "string_1")
+    build_data(model, nil, "string_1")
+  end
+
+  def build_data(model, foreign_key_value, string_value)
+    data_values = model_unique_keys(model).each_with_object({}) do |key, obj|
+      obj[key] = build_value(key, foreign_key_value, string_value)
+    end
+    model.create!(data_values)
+  end
+
+  def build_value(key, foreign_key_value, string_value)
+    if key.to_s.ends_with?("id")
+      foreign_key_value
+    else
+      string_value
+    end
+  end
+
+  def analyze(model)
+    original_values = {}
+    duplicate_values = {}
+    model.order("id DESC").all.each do |record|
+      index = record.attributes.symbolize_keys.slice(*model_unique_keys(model))
+      if original_values[index]
+        duplicate_values[index] << record.id
+      else
+        original_values[index] = record.id
+        duplicate_values[index] ||= []
+      end
+    end
+
+    return original_values, duplicate_values
+  end
+
+  def assert_before_migration_test_data(model, original_values, duplicate_values)
+    expect(model.count).to eq(10)
+
+    # Check there are 5 duplicates in the data
+    expect(original_values.count).to eq(5)
+    expect(duplicate_values.values.flatten.count).to eq(5)
+
+    # Check that original values ids are the max or all duplicated ids
+    original_values.each do |key, value|
+      expect((duplicate_values[key] << value).max).to eq value
+    end
+  end
+
+  def assert_after_migration_test_data(model, original_values, _duplicate_values)
+    expect(model.count).to eq(5)
+
+    model.all.each do |record|
+      expect(original_values[record.attributes.symbolize_keys.slice(*model_unique_keys(model))]).to eq record.id
+    end
+  end
+
+  migration_context :up do
+    described_class::UNIQUE_INDEXES_FOR_MODELS.each_key do |model|
+      context "with model #{model}" do
+        it "checks that the duplicate values are cleaned up" do
+          create_test_data(model)
+
+          original_values, duplicate_values = analyze(model)
+          assert_before_migration_test_data(model, original_values, duplicate_values)
+          migrate
+          assert_after_migration_test_data(model, original_values, duplicate_values)
+        end
+      end
+    end
+  end
+end