diff --git a/app/models/manager_refresh/inventory/persister.rb b/app/models/manager_refresh/inventory/persister.rb index ae8aaae2eb9..2eda7e1909d 100644 --- a/app/models/manager_refresh/inventory/persister.rb +++ b/app/models/manager_refresh/inventory/persister.rb @@ -4,6 +4,8 @@ class ManagerRefresh::Inventory::Persister attr_reader :manager, :target, :collections + include ::ManagerRefresh::InventoryCollection::Builder::PersisterHelper + # @param manager [ManageIQ::Providers::BaseManager] A manager object # @param target [Object] A refresh Target object def initialize(manager, target = nil) @@ -117,12 +119,6 @@ def initialize_inventory_collections # can be implemented in a subclass end - # @return [Hash] kwargs shared for all InventoryCollection objects - def shared_options - # can be implemented in a subclass - {} - end - # Adds 1 ManagerRefresh::InventoryCollection under a target.collections using :association key as index # # @param options [Hash] Hash used for ManagerRefresh::InventoryCollection initialize diff --git a/app/models/manager_refresh/inventory_collection.rb b/app/models/manager_refresh/inventory_collection.rb index 433122d9fee..561336d971b 100644 --- a/app/models/manager_refresh/inventory_collection.rb +++ b/app/models/manager_refresh/inventory_collection.rb @@ -44,6 +44,8 @@ module ManagerRefresh # puts @ems.vms.collect(&:ems_ref) # => ["vm2", "vm3"] # class InventoryCollection + require_nested :Builder + # @return [Boolean] A true value marks that we collected all the data of the InventoryCollection, # meaning we also collected all the references. attr_accessor :data_collection_finalized @@ -445,7 +447,6 @@ def initialize(model_class: nil, manager_ref: nil, association: nil, parent: nil @attributes_whitelist = Set.new @transitive_dependency_attributes = Set.new @dependees = Set.new - @data_storage = ::ManagerRefresh::InventoryCollection::DataStorage.new(self, secondary_refs) @references_storage = ::ManagerRefresh::InventoryCollection::ReferencesStorage.new(index_proxy) @targeted_scope = ::ManagerRefresh::InventoryCollection::ReferencesStorage.new(index_proxy).merge!(manager_uuids) diff --git a/app/models/manager_refresh/inventory_collection/builder.rb b/app/models/manager_refresh/inventory_collection/builder.rb new file mode 100644 index 00000000000..f5cc34909d6 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder.rb @@ -0,0 +1,248 @@ +module ManagerRefresh + class InventoryCollection + # @see test in /spec/models/manager_refresh/inventory_collection/builder_spec.rb + class Builder + class MissingModelClassError < StandardError; end + + require_nested :CloudManager + require_nested :InfraManager + require_nested :AutomationManager + require_nested :NetworkManager + require_nested :StorageManager + require_nested :PersisterHelper + + include ::ManagerRefresh::InventoryCollection::Builder::Shared + + # Default options for builder + # :adv_settings + # - values from Advanced settings (doesn't overwrite values specified in code) + # - @see method ManagerRefresh::Inventory::Persister.make_builder_settings() + # :shared_properties + # - any properties applied if missing (not explicitly specified) + def self.default_options + { + :adv_settings => {}, + :shared_properties => {}, + } + end + + # Entry point + # Creates builder and builds data for inventory collection + # @param name [Symbol] InventoryCollection.association value + # (optional) method with this name also used for concrete inventory collection specific properties + # @param persister_class [Class] used for "guessing" model_class + # @param options [Hash] + def self.prepare_data(name, persister_class, options = {}) + options = default_options.merge(options) + builder = new(name, persister_class, options) + builder.construct_data + + yield(builder) if block_given? + + builder + end + + # @see prepare_data() + def initialize(name, persister_class, options = self.class.default_options) + @name = name + @persister_class = persister_class + + @properties = {} + @inventory_object_attributes = [] + @builder_params = {} + @dependency_attributes = {} + + @options = options + @options[:auto_inventory_attributes] = true if @options[:auto_inventory_attributes].nil? + @options[:without_model_class] = false if @options[:without_model_class].nil? + + @adv_settings = options[:adv_settings] # Configuration/Advanced settings in GUI + @shared_properties = options[:shared_properties] # From persister + end + + # Builds data for InventoryCollection + # Calls method @name (if exists) with specific properties + # Yields for overwriting provider-specific properties + def construct_data + add_properties(:association => @name) + add_properties(:model_class => auto_model_class) unless @options[:without_model_class] + + add_properties(@adv_settings, :if_missing) + add_properties(@shared_properties, :if_missing) + + send(@name.to_sym) if respond_to?(@name.to_sym) + + add_inventory_attributes(auto_inventory_attributes) if @options[:auto_inventory_attributes] + end + + # Creates InventoryCollection + def to_inventory_collection + if @properties[:model_class].nil? && !@options[:without_model_class] + raise MissingModelClassError + end + + ::ManagerRefresh::InventoryCollection.new(to_hash) + end + + # + # Missing method + # - add_some_property(value) + # converted to: + # - add_properties(:some_property => value) + # + def method_missing(method_name, *arguments, &block) + if method_name.to_s.starts_with?('add_') + add_properties( + method_name.to_s.gsub('add_', '').to_sym => arguments[0] + ) + else + super + end + end + + def respond_to_missing?(method_name, _include_private = false) + method_name.to_s.starts_with?('add_') + end + + # Merges @properties + # @see ManagerRefresh::InventoryCollection.initialize for list of properties + # + # @param props [Hash] + # @param mode [Symbol] :overwrite | :if_missing + def add_properties(props = {}, mode = :overwrite) + @properties = merge_hashes(@properties, props, mode) + end + + # Adds inventory object attributes (part of @properties) + def add_inventory_attributes(array) + @inventory_object_attributes += (array || []) + end + + # Removes specified inventory object attributes + def remove_inventory_attributes(array) + @inventory_object_attributes -= (array || []) + end + + # Clears all inventory object attributes + def clear_inventory_attributes! + @inventory_object_attributes = [] + end + + # Adds key/values to builder params (part of @properties) + def add_builder_params(params = {}, mode = :overwrite) + @builder_params = merge_hashes(@builder_params, params, mode) + end + + # Evaluates lambda blocks + def evaluate_lambdas!(persister) + evaluate_builder_params_lambdas!(persister) + end + + # Adds key/values to dependency_attributes (part of @properties) + def add_dependency_attributes(attrs = {}, mode = :overwrite) + @dependency_attributes = merge_hashes(@dependency_attributes, attrs, mode) + end + + # Returns whole InventoryCollection properties + def to_hash + @properties.merge( + :inventory_object_attributes => @inventory_object_attributes, + :builder_params => @builder_params, + :dependency_attributes => @dependency_attributes + ) + end + + protected + + # Extends source hash with + # - a) all keys from dest (overwrite mode) + # - b) missing keys (missing mode) + # + # @param mode [Symbol] :overwrite | :if_missing + def merge_hashes(source, dest, mode) + return source if source.nil? || dest.nil? + + if mode == :overwrite + source.merge(dest) + else + dest.merge(source) + end + end + + # Derives model_class from persister class and @name + # 1) searches for class in provider + # 2) if not found, searches class in core + # Can be disabled by options :auto_model_class => false + # + # @example derives model_class from amazon + # + # @persister_class = ManageIQ::Providers::Amazon::Inventory::Persister::CloudManager + # @name = :vms + # + # returns - ::::<@name.classify> + # returns - ::ManageIQ::Providers::Amazon::CloudManager::Vm + # + # @example derives model_class from @name only + # + # @persister_class = ManagerRefresh::Inventory::Persister + # @name = :vms + # + # returns ::Vm + # + # @return [Class | nil] when class doesn't exist, returns nil + def auto_model_class + model_class = begin + # a) Provider specific class + provider_module = ManageIQ::Providers::Inflector.provider_module(@persister_class).name + manager_module = self.class.name.split('::').last + + class_name = "#{provider_module}::#{manager_module}::#{@name.to_s.classify}" + class_name.safe_constantize + rescue ::ManageIQ::Providers::Inflector::ObjectNotNamespacedError + nil + end + + if model_class + model_class + else + # b) general class + "::#{@name.to_s.classify}".safe_constantize + end + end + + # Inventory object attributes are derived from setters + # + # Can be disabled by options :auto_inventory_attributes => false + # - attributes can be manually set via method add_inventory_attributes() + def auto_inventory_attributes + return if @properties[:model_class].nil? + + (@properties[:model_class].new.methods - ApplicationRecord.methods).grep(/^[\w]+?\=$/).collect do |setter| + setter.to_s[0..setter.length - 2].to_sym + end + end + + # Evaluates lambda blocks in @builder_params + def evaluate_builder_params_lambdas!(persister) + if @builder_params + @builder_params = @builder_params.transform_values do |value| + if value.respond_to?(:call) + value.call(persister) + else + value + end + end + end + end + + def network_manager_collections? + self.class.network_manager_collections? + end + + # InventoryCollection definitions for NetworkManager? + def self.network_manager_collections? + false + end + end + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/automation_manager.rb b/app/models/manager_refresh/inventory_collection/builder/automation_manager.rb new file mode 100644 index 00000000000..71b5c9df220 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/automation_manager.rb @@ -0,0 +1,58 @@ +module ManagerRefresh + class InventoryCollection + class Builder + # TODO: (mslemr) Remove /manager_refresh/inventory/[core, automation_manager, cloud_manager, ?middleware_manager?].rb + # TODO: (mslemr) think about lib/generators/provider/templates/app/models/manageiq/providers/%provider_name%/inventory/persister/cloud_manager.rb + class AutomationManager < ::ManagerRefresh::InventoryCollection::Builder + def configuration_scripts + default_manager_ref + default_builder_params + end + + def configuration_script_payloads + add_properties( + :manager_ref => %i(configuration_script_source manager_ref) + ) + default_builder_params + end + + def configuration_script_sources + default_manager_ref + default_builder_params + end + + def configured_systems + default_manager_ref + default_builder_params + end + + def credentials + default_manager_ref + add_builder_params( + :resource => ->(persister) { persister.manager } + ) + end + + def inventory_root_groups + default_builder_params + end + + def vms + add_properties(:manager_ref => %i(uid_ems)) + end + + protected + + def default_manager_ref + add_properties(:manager_ref => %i(manager_ref)) + end + + def default_builder_params + add_builder_params( + :manager => ->(persister) { persister.manager } + ) + end + end + end + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/cloud_manager.rb b/app/models/manager_refresh/inventory_collection/builder/cloud_manager.rb new file mode 100644 index 00000000000..fe024acbe97 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/cloud_manager.rb @@ -0,0 +1,176 @@ +module ManagerRefresh + class InventoryCollection + class Builder + class CloudManager < ::ManagerRefresh::InventoryCollection::Builder + def availability_zones + shared_builder_params + end + + def flavors + shared_builder_params + end + + def key_pairs + add_properties( + :model_class => ::ManageIQ::Providers::CloudManager::AuthKeyPair, + :manager_ref => %i(name) + ) + add_builder_params( + :resource_id => ->(persister) { persister.manager.id }, + :resource_type => ->(persister) { persister.manager.class.base_class } + ) + end + + def vm_and_template_labels + # TODO(lsmola) make a generic CustomAttribute IC and move it to base class + add_properties( + :model_class => ::CustomAttribute, + :manager_ref => %i(resource name), + :parent_inventory_collections => %i(vms miq_templates) + ) + end + + def orchestration_stacks + add_properties( + :model_class => ::ManageIQ::Providers::CloudManager::OrchestrationStack, + :attributes_blacklist => %i(parent), + ) + + shared_builder_params + end + + def orchestration_stacks_resources + add_properties( + :model_class => ::OrchestrationStackResource, + :parent_inventory_collections => %i(orchestration_stacks) + ) + end + + def orchestration_stacks_outputs + add_properties( + :model_class => ::OrchestrationStackOutput, + :parent_inventory_collections => %i(orchestration_stacks) + ) + end + + def orchestration_stacks_parameters + add_properties( + :model_class => ::OrchestrationStackParameter, + :parent_inventory_collections => %i(orchestration_stacks) + ) + end + + def orchestration_templates + # TODO(lsmola) do refactoring, we shouldn't need this custom saving block\ + orchestration_templates_save_block = lambda do |_ems, inventory_collection| + hashes = inventory_collection.data.map(&:attributes) + + templates = inventory_collection.model_class.find_or_create_by_contents(hashes) + inventory_collection.data.zip(templates).each do |inventory_object, template| + inventory_object.id = template.id + end + end + + add_properties( + :custom_save_block => orchestration_templates_save_block + ) + end + + def orchestration_stack_ancestry + orchestration_stack_ancestry_save_block = lambda do |_ems, inventory_collection| + stacks_inventory_collection = inventory_collection.dependency_attributes[:orchestration_stacks].try(:first) + + return if stacks_inventory_collection.blank? + + stacks_parents = stacks_inventory_collection.data.each_with_object({}) do |x, obj| + parent_id = x.data[:parent].try(:load).try(:id) + obj[x.id] = parent_id if parent_id + end + + model_class = stacks_inventory_collection.model_class + + stacks_parents_indexed = model_class + .select(%i(id ancestry)) + .where(:id => stacks_parents.values).find_each.index_by(&:id) + + ActiveRecord::Base.transaction do + model_class.select(%i(id ancestry)) + .where(:id => stacks_parents.keys).find_each do |stack| + parent = stacks_parents_indexed[stacks_parents[stack.id]] + stack.update_attribute(:parent, parent) + end + end + end + + add_properties( + :custom_save_block => orchestration_stack_ancestry_save_block + ) + end + + def vm_and_miq_template_ancestry + vm_and_miq_template_ancestry_save_block = lambda do |_ems, inventory_collection| + vms_inventory_collection = inventory_collection.dependency_attributes[:vms].try(:first) + miq_templates_inventory_collection = inventory_collection.dependency_attributes[:miq_templates].try(:first) + + return if vms_inventory_collection.blank? || miq_templates_inventory_collection.blank? + + # Fetch IDs of all vms and genealogy_parents, only if genealogy_parent is present + vms_genealogy_parents = vms_inventory_collection.data.each_with_object({}) do |x, obj| + unless x.data[:genealogy_parent].nil? + genealogy_parent_id = x.data[:genealogy_parent].load.try(:id) + obj[x.id] = genealogy_parent_id if genealogy_parent_id + end + end + + miq_template_genealogy_parents = miq_templates_inventory_collection.data.each_with_object({}) do |x, obj| + unless x.data[:genealogy_parent].nil? + genealogy_parent_id = x.data[:genealogy_parent].load.try(:id) + obj[x.id] = genealogy_parent_id if genealogy_parent_id + end + end + + ActiveRecord::Base.transaction do + # associate parent templates to child instances + parent_miq_templates = miq_templates_inventory_collection.model_class + .select([:id]) + .where(:id => vms_genealogy_parents.values).find_each.index_by(&:id) + vms_inventory_collection.model_class + .select([:id]) + .where(:id => vms_genealogy_parents.keys).find_each do |vm| + parent = parent_miq_templates[vms_genealogy_parents[vm.id]] + vm.with_relationship_type('genealogy') { vm.parent = parent } + end + end + + ActiveRecord::Base.transaction do + # associate parent instances to child templates + parent_vms = vms_inventory_collection.model_class + .select([:id]) + .where(:id => miq_template_genealogy_parents.values).find_each.index_by(&:id) + miq_templates_inventory_collection.model_class + .select([:id]) + .where(:id => miq_template_genealogy_parents.keys).find_each do |miq_template| + parent = parent_vms[miq_template_genealogy_parents[miq_template.id]] + miq_template.with_relationship_type('genealogy') { miq_template.parent = parent } + end + end + end + + add_properties( + :custom_save_block => vm_and_miq_template_ancestry_save_block + ) + end + end + + private + + def shared_builder_params + add_builder_params(:ems_id => default_ems_id) + end + + def default_ems_id + ->(persister) { persister.manager.id } + end + end + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/infra_manager.rb b/app/models/manager_refresh/inventory_collection/builder/infra_manager.rb new file mode 100644 index 00000000000..7728c8091c0 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/infra_manager.rb @@ -0,0 +1,216 @@ +module ManagerRefresh + class InventoryCollection + class Builder + class InfraManager < ::ManagerRefresh::InventoryCollection::Builder + def networks + add_properties( + :manager_ref => %i(hardware ipaddress ipv6address) + ) + end + + def host_networks + add_properties( + :model_class => ::Network, + :manager_ref => %i(hardware ipaddress), + :parent_inventory_collections => %i(hosts) + ) + end + + def guest_devices + add_properties( + :manager_ref => %i(hardware uid_ems), + ) + end + + def host_hardwares + add_properties( + :model_class => ::Hardware, + :manager_ref => %i(host), + :parent_inventory_collections => %i(hosts) + ) + end + + def snapshots + add_properties( + :manager_ref => %i(uid) + ) + end + + def operating_systems + add_properties( + :manager_ref => %i(vm_or_template) + ) + end + + def host_operating_systems + add_properties( + :model_class => ::OperatingSystem, + :manager_ref => %i(host), + :parent_inventory_collections => %i(hosts), + ) + end + + def custom_attributes + add_properties( + :manager_ref => %i(name) + ) + end + + def ems_folders + add_properties( + :manager_ref => %i(uid_ems), + :attributes_blacklist => %i(ems_children), + ) + shared_builder_params + end + + def datacenters + shared_builder_params + end + + def resource_pools + add_properties( + :manager_ref => %i(uid_ems), + :attributes_blacklist => %i(ems_children), + ) + shared_builder_params + end + + def ems_clusters + add_properties( + :attributes_blacklist => %i(ems_children datacenter_id), + ) + + add_inventory_attributes(%i(datacenter_id)) + shared_builder_params + end + + def storages + add_properties( + :manager_ref => %i(location), + :complete => false, + :arel => Storage, + ) + end + + def hosts + shared_builder_params + + add_custom_reconnect_block( + lambda do |inventory_collection, inventory_objects_index, attributes_index| + relation = inventory_collection.model_class.where(:ems_id => nil) + + return if relation.count <= 0 + + inventory_objects_index.each_slice(100) do |batch| + relation.where(inventory_collection.manager_ref.first => batch.map(&:first)).each do |record| + index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record) + + # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it + # would be sent for create. + inventory_object = inventory_objects_index.delete(index) + hash = attributes_index.delete(index) + + record.assign_attributes(hash.except(:id, :type)) + if !inventory_collection.check_changed? || record.changed? + record.save! + inventory_collection.store_updated_records(record) + end + + inventory_object.id = record.id + end + end + end + ) + end + + def vms + super + + custom_reconnect_block = lambda do |inventory_collection, inventory_objects_index, attributes_index| + relation = inventory_collection.model_class.where(:ems_id => nil) + + return if relation.count <= 0 + + inventory_objects_index.each_slice(100) do |batch| + relation.where(inventory_collection.manager_ref.first => batch.map(&:first)).each do |record| + index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record) + + # We need to delete the record from the inventory_objects_index and attributes_index, otherwise it + # would be sent for create. + inventory_object = inventory_objects_index.delete(index) + hash = attributes_index.delete(index) + + record.assign_attributes(hash.except(:id, :type)) + if !inventory_collection.check_changed? || record.changed? + record.save! + inventory_collection.store_updated_records(record) + end + + inventory_object.id = record.id + end + end + end + + add_properties( + :custom_reconnect_block => custom_reconnect_block + ) + end + + def host_storages + add_properties( + :manager_ref => %i(host storage), + :parent_inventory_collections => %i(hosts) + ) + end + + def host_switches + add_properties( + :manager_ref => %i(host switch), + :parent_inventory_collections => %i(hosts) + ) + end + + def switches + add_properties( + :manager_ref => %i(uid_ems) + ) + end + + def lans + add_properties( + :manager_ref => %i(uid_ems), + ) + end + + def snapshot_parent + snapshot_parent_save_block = lambda do |_ems, inventory_collection| + snapshot_collection = inventory_collection.dependency_attributes[:snapshots].try(:first) + + snapshot_collection.each do |snapshot| + ActiveRecord::Base.transaction do + child = Snapshot.find(snapshot.id) + parent = Snapshot.find_by(:uid_ems => snapshot.parent_uid) + child.update_attribute(:parent_id, parent.try(:id)) + end + end + end + + add_properties( + :custom_save_block => snapshot_parent_save_block + ) + end + + private + + def shared_builder_params + add_builder_params(:ems_id => default_ems_id) + end + + def default_ems_id + ->(persister) { persister.manager.id } + end + end + end + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/network_manager.rb b/app/models/manager_refresh/inventory_collection/builder/network_manager.rb new file mode 100644 index 00000000000..2a14baccb5c --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/network_manager.rb @@ -0,0 +1,217 @@ +module ManagerRefresh + class InventoryCollection + class Builder + class NetworkManager < ::ManagerRefresh::InventoryCollection::Builder + def cloud_subnet_network_ports + add_properties( + # :model_class => ::CloudSubnetNetworkPort, + :manager_ref => %i(address cloud_subnet network_port), + :parent_inventory_collections => %i(vms network_ports load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.cloud_subnet_network_ports.references(:network_ports).where( + :network_ports => {:ems_ref => manager_uuids} + ) + end + ) + end + + def network_ports + add_properties( + :use_ar_object => true, + # TODO(lsmola) can't do batch strategy for network_ports because of security_groups relation + :saver_strategy => :default + ) + + shared_builder_params + end + + def network_groups + shared_builder_params + end + + def network_routers + shared_builder_params + end + + def floating_ips + shared_builder_params + end + + def cloud_tenants + shared_builder_params + end + + def cloud_subnets + shared_builder_params + end + + def cloud_networks + shared_builder_params + end + + def security_groups + shared_builder_params + end + + def firewall_rules + add_properties( + :manager_ref => %i(resource source_security_group direction host_protocol port end_port source_ip_range), + :parent_inventory_collections => %i(security_groups) + ) + end + + def load_balancers + shared_builder_params + end + + def load_balancer_pools + add_properties( + :parent_inventory_collections => %i(load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_pools + .joins(:load_balancers) + .where(:load_balancers => {:ems_ref => manager_uuids}) + .distinct + end + ) + + shared_builder_params + end + + def load_balancer_pool_members + add_properties( + :parent_inventory_collections => %i(load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_pool_members + .joins(:load_balancer_pool_member_pools => [:load_balancer_pool => :load_balancers]) + .where(:load_balancer_pool_member_pools => { + 'load_balancer_pools' => { + 'load_balancers' => { + :ems_ref => manager_uuids + } + } + }).distinct + end + ) + + shared_builder_params + end + + def load_balancer_pool_member_pools + add_properties( + :manager_ref => %i(load_balancer_pool load_balancer_pool_member), + :parent_inventory_collections => %i(load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_pool_member_pools + .joins(:load_balancer_pool => :load_balancers) + .where(:load_balancer_pools => { 'load_balancers' => { :ems_ref => manager_uuids } }) + .distinct + end + ) + end + + def load_balancer_listeners + add_properties( + :use_ar_object => true, + :parent_inventory_collections => %i(load_balancers), + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_listeners + .joins(:load_balancer) + .where(:load_balancers => {:ems_ref => manager_uuids}) + .distinct + end + ) + + shared_builder_params + end + + def load_balancer_listener_pools + add_properties( + :manager_ref => %i(load_balancer_listener load_balancer_pool), + :parent_inventory_collections => %i(load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_listener_pools + .joins(:load_balancer_pool => :load_balancers) + .where(:load_balancer_pools => {'load_balancers' => {:ems_ref => manager_uuids}}) + .distinct + end + ) + end + + def load_balancer_health_checks + add_properties( + :parent_inventory_collections => %i(load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_health_checks + .joins(:load_balancer) + .where(:load_balancers => {:ems_ref => manager_uuids}) + .distinct + end + ) + + shared_builder_params + end + + def load_balancer_health_check_members + add_properties( + :manager_ref => %i(load_balancer_health_check load_balancer_pool_member), + :parent_inventory_collections => %i(load_balancers) + ) + + add_targeted_arel( + lambda do |inventory_collection| + manager_uuids = inventory_collection.parent_inventory_collections.flat_map { |c| c.manager_uuids.to_a } + inventory_collection.parent.load_balancer_health_check_members + .joins(:load_balancer_health_check => :load_balancer) + .where(:load_balancer_health_checks => {'load_balancers' => {:ems_ref => manager_uuids}}) + .distinct + end + ) + end + + protected + + def shared_builder_params + add_builder_params(:ems_id => default_ems_id) + end + + def default_ems_id + ->(persister) { persister.manager.try(:network_manager).try(:id) || persister.manager.id } + end + + # InventoryCollection definitions for NetworkManager? + def self.network_manager_collections? + true + end + end + end + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/persister_helper.rb b/app/models/manager_refresh/inventory_collection/builder/persister_helper.rb new file mode 100644 index 00000000000..0cc6eed4f18 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/persister_helper.rb @@ -0,0 +1,109 @@ +module ManagerRefresh::InventoryCollection::Builder::PersisterHelper + extend ActiveSupport::Concern + + # Interface for creating InventoryCollection under @collections + # + # @param builder_class [ManagerRefresh::InventoryCollection::Builder] or subclasses + # @param collection_name [Symbol] used as InventoryCollection:association + # @param extra_properties [Hash] props from InventoryCollection.initialize list + # - adds/overwrites properties added by builder + # + # @param settings [Hash] builder settings + # - @see ManagerRefresh::InventoryCollection::Builder.default_options + # - @see make_builder_settings() + # + # @example + # add_collection(ManagerRefresh::InventoryCollection::Builder::CloudManager, :vms) do |builder| + # builder.add_properties( + # :strategy => :local_db_cache_all, + # ) + # ) + # + # @see ManagerRefresh::InventoryCollection::Builder + # + def add_collection(builder_class, collection_name, extra_properties = {}, settings = {}, &block) + builder = builder_class.prepare_data(collection_name, + self.class, + make_builder_settings(settings), + &block) + + builder.add_properties(extra_properties) if extra_properties.present? + + builder.add_properties({:manager_uuids => references(collection_name)}, :if_missing) if targeted? + + builder.evaluate_lambdas!(self) + + collections[collection_name] = builder.to_inventory_collection + end + + # builder_class for add_collection() + def cloud + ::ManagerRefresh::InventoryCollection::Builder::CloudManager + end + + # builder_class for add_collection() + def network + ::ManagerRefresh::InventoryCollection::Builder::NetworkManager + end + + # builder_class for add_collection() + def infra + ::ManagerRefresh::InventoryCollection::Builder::InfraManager + end + + # builder_class for add_collection() + def storage + ::ManagerRefresh::InventoryCollection::Builder::StorageManager + end + + # builder_class for add_collection() + def automation + ::ManagerRefresh::InventoryCollection::Builder::AutomationManager + end + + # @param extra_settings [Hash] + # :auto_inventory_attributes + # - auto creates inventory_object_attributes from target model_class setters + # - attributes used in InventoryObject.add_attributes + # :without_model_class + # - if false and no model_class derived or specified, throws exception + # - doesn't try to derive model class automatically + # - @see method ManagerRefresh::InventoryCollection::Builder.auto_model_class + def make_builder_settings(extra_settings = {}) + opts = ::ManagerRefresh::InventoryCollection::Builder.default_options + + opts[:adv_settings] = options[:inventory_collections].try(:to_hash) || {} + opts[:shared_properties] = shared_options + opts[:auto_inventory_attributes] = true + opts[:without_model_class] = false + + opts.merge(extra_settings) + end + + def strategy + nil + end + + # Persisters for targeted refresh can override to true + def targeted? + false + end + + # @return [Hash] kwargs shared for all InventoryCollection objects + def shared_options + # can be implemented in a subclass + {} + end + + # Returns list of target's ems_refs + # @return [Array] + def references(collection) + target.manager_refs_by_association.try(:[], collection).try(:[], :ems_ref).try(:to_a) || [] + end + + # Returns list of target's name + # @return [Array] + def name_references(collection) + target.manager_refs_by_association.try(:[], collection).try(:[], :name).try(:to_a) || [] + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/shared.rb b/app/models/manager_refresh/inventory_collection/builder/shared.rb new file mode 100644 index 00000000000..9678cb25383 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/shared.rb @@ -0,0 +1,90 @@ +module ManagerRefresh::InventoryCollection::Builder::Shared + extend ActiveSupport::Concern + + included do + INVENTORY_RECONNECT_BLOCK = lambda do |inventory_collection, inventory_objects_index, attributes_index| + relation = inventory_collection.model_class.where(:ems_id => nil) + + return if relation.count <= 0 + + inventory_objects_index.each_slice(100) do |batch| + batch_refs = batch.map(&:first) + relation.where(inventory_collection.manager_ref.first => batch_refs).each do |record| + index = inventory_collection.object_index_with_keys(inventory_collection.manager_ref_to_cols, record) + + # We need to delete the record from the inventory_objects_index + # and attributes_index, otherwise it would be sent for create. + inventory_object = inventory_objects_index.delete(index) + hash = attributes_index.delete(index) + + record.assign_attributes(hash.except(:id, :type)) + if !inventory_collection.check_changed? || record.changed? + record.save! + inventory_collection.store_updated_records(record) + end + + inventory_object.id = record.id + end + end + end.freeze + + def vendor + ::ManageIQ::Providers::Inflector.provider_name(@persister_class).downcase + rescue + 'unknown' + end + + def vms + vm_template_shared + end + + def miq_templates + vm_template_shared + end + + def vm_template_shared + add_properties( + :delete_method => :disconnect_inv, + :attributes_blacklist => %i(genealogy_parent), + :use_ar_object => true, # Because of raw_power_state setter and hooks are needed for settings user + :saver_strategy => :default, + :batch_extra_attributes => %i(power_state state_changed_on previous_state), + :custom_reconnect_block => INVENTORY_RECONNECT_BLOCK + ) + + add_builder_params( + :ems_id => ->(persister) { persister.manager.id }, + :vendor => vendor + ) + end + + def hardwares + add_properties( + :manager_ref => %i(vm_or_template), + :parent_inventory_collections => %i(vms miq_templates), + :use_ar_object => true, # TODO(lsmola) just because of default value on cpu_sockets, this can be fixed by separating instances_hardwares and images_hardwares + ) + end + + def operating_systems + add_properties( + :manager_ref => %i(vm_or_template), + :parent_inventory_collections => %i(vms miq_templates) + ) + end + + def networks + add_properties( + :manager_ref => %i(hardware description), + :parent_inventory_collections => %i(vms) + ) + end + + def disks + add_properties( + :manager_ref => %i(hardware device_name), + :parent_inventory_collections => %i(vms) + ) + end + end +end diff --git a/app/models/manager_refresh/inventory_collection/builder/storage_manager.rb b/app/models/manager_refresh/inventory_collection/builder/storage_manager.rb new file mode 100644 index 00000000000..1cb0cf808b1 --- /dev/null +++ b/app/models/manager_refresh/inventory_collection/builder/storage_manager.rb @@ -0,0 +1,9 @@ +module ManagerRefresh + class InventoryCollection + class Builder + class StorageManager < ::ManagerRefresh::InventoryCollection::Builder + # Nothing there + end + end + end +end diff --git a/app/models/manager_refresh/inventory_object.rb b/app/models/manager_refresh/inventory_object.rb index fd355859cf8..d12f7545013 100644 --- a/app/models/manager_refresh/inventory_object.rb +++ b/app/models/manager_refresh/inventory_object.rb @@ -153,16 +153,23 @@ def dependency? end # Adds setters and getters based on :inventory_object_attributes kwarg passed into InventoryCollection + # Methods already defined should not be redefined (causes unexpected behaviour) # # @param inventory_object_attributes [Array] def self.add_attributes(inventory_object_attributes) + defined_methods = ManagerRefresh::InventoryObject.instance_methods(false) + inventory_object_attributes.each do |attr| - define_method("#{attr}=") do |value| - data[attr] = value + unless defined_methods.include?("#{attr}=".to_sym) + define_method("#{attr}=") do |value| + data[attr] = value + end end - define_method(attr) do - data[attr] + unless defined_methods.include?(attr.to_sym) + define_method(attr) do + data[attr] + end end end end diff --git a/spec/models/manager_refresh/inventory_collection/builder_spec.rb b/spec/models/manager_refresh/inventory_collection/builder_spec.rb new file mode 100644 index 00000000000..96f3802e9bc --- /dev/null +++ b/spec/models/manager_refresh/inventory_collection/builder_spec.rb @@ -0,0 +1,201 @@ +require_relative '../persister/test_persister' + +describe ManagerRefresh::InventoryCollection::Builder do + before :each do + @zone = FactoryGirl.create(:zone) + @ems = FactoryGirl.create(:ems_cloud, + :zone => @zone, + :network_manager => FactoryGirl.create(:ems_network, :zone => @zone)) + @persister = create_persister + end + + def create_persister + TestPersister.new(@ems, ManagerRefresh::TargetCollection.new(:manager => @ems)) + end + + let(:adv_settings) { {:strategy => :local_db_find_missing_references, :saver_strategy => :concurrent_safe_batch} } + + let(:cloud) { ::ManagerRefresh::InventoryCollection::Builder::CloudManager } + + let(:network) { ::ManagerRefresh::InventoryCollection::Builder::NetworkManager } + + let(:persister_class) { ::ManagerRefresh::Inventory::Persister } + + # --- association --- + + it 'assigns association automatically to InventoryCollection' do + ic = cloud.prepare_data(:vms, persister_class).to_inventory_collection + + expect(ic.association).to eq :vms + end + + # --- model_class --- + + # TODO: move to amazon spec + it "derives existing model_class from persister's class" do + end + + it "derives existing model_class without persister's class" do + data = cloud.prepare_data(:vms, persister_class).to_hash + + expect(data[:model_class]).to eq ::Vm + end + + it "replaces derived model_class if model_class defined manually" do + data = cloud.prepare_data(:vms, persister_class) do |builder| + builder.add_properties(:model_class => ::MiqTemplate) + end.to_hash + + expect(data[:model_class]).to eq ::MiqTemplate + end + + it "doesn't try to derive model_class when disabled" do + data = cloud.prepare_data(:vms, persister_class, :without_model_class => true).to_hash + + expect(data[:model_class]).to be_nil + end + + it 'throws exception if model_class not specified' do + builder = cloud.prepare_data(:non_existing_ic, persister_class) + + expect { builder.to_inventory_collection }.to raise_error(::ManagerRefresh::InventoryCollection::Builder::MissingModelClassError) + end + + # --- adv. settings (TODO: link to gui)--- + + it 'assigns Advanced settings' do + builder = cloud.prepare_data(:tmp, persister_class, :adv_settings => adv_settings) + data = builder.to_hash + + expect(data[:strategy]).to eq :local_db_find_missing_references + expect(data[:saver_strategy]).to eq :concurrent_safe_batch + end + + it "doesn't overwrite defined properties by Advanced settings" do + data = cloud.prepare_data(:vms, persister_class, :adv_settings => adv_settings) do |builder| + builder.add_properties(:strategy => :custom) + end.to_hash + + expect(data[:strategy]).to eq :custom + expect(data[:saver_strategy]).to eq :default + end + + # --- shared properties --- + + it 'applies shared properties' do + data = cloud.prepare_data(:tmp, persister_class, :shared_properties => {:uuid => 1}).to_hash + + expect(data[:uuid]).to eq 1 + end + + it "doesn't overwrite defined properties by shared properties" do + data = cloud.prepare_data(:tmp, persister_class, :shared_properties => {:uuid => 1}) do |builder| + builder.add_properties(:uuid => 2) + end.to_hash + + expect(data[:uuid]).to eq 2 + end + + # --- properties --- + + it 'adds properties with add_properties repeatedly' do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_properties(:first => 1, :second => 2) + builder.add_properties(:third => 3) + end.to_hash + + expect(data[:first]).to eq 1 + expect(data[:second]).to eq 2 + expect(data[:third]).to eq 3 + end + + it 'overrides properties in :overwrite mode' do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_properties(:param => 1) + builder.add_properties({:param => 2}, :overwrite) + end.to_hash + + expect(data[:param]).to eq 2 + end + + it "doesn't override properties in :if_missing mode" do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_properties(:param => 1) + builder.add_properties({:param => 2}, :if_missing) + end.to_hash + + expect(data[:param]).to eq 1 + end + + it 'adds property by method_missing' do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_some_tmp_param(:some_value) + end.to_hash + + expect(data[:some_tmp_param]).to eq :some_value + end + + # --- builder params --- + + it 'adds builder_params repeatedly' do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_builder_params(:ems_id => 10) + builder.add_builder_params(:ems_id => 20) + builder.add_builder_params(:tmp_id => 30) + end.to_hash + + expect(data[:builder_params][:ems_id]).to eq 20 + expect(data[:builder_params][:tmp_id]).to eq 30 + end + + it 'transforms lambdas in builder_params' do + bldr = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_builder_params(:ems_id => ->(persister) { persister.manager.id }) + end + bldr.evaluate_lambdas!(@persister) + + data = bldr.to_hash + + expect(data[:builder_params][:ems_id]).to eq(@persister.manager.id) + end + + # --- inventory object attributes --- + + it 'derives inventory object attributes automatically' do + data = cloud.prepare_data(:vms, persister_class).to_hash + + expect(data[:inventory_object_attributes]).not_to be_empty + end + + it "doesn't derive inventory_object_attributes automatically when disabled" do + data = cloud.prepare_data(:vms, persister_class, :auto_inventory_attributes => false).to_hash + + expect(data[:inventory_object_attributes]).to be_empty + end + + it 'can add inventory_object_attributes manually' do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_inventory_attributes(%i(attr1 attr2 attr3)) + end.to_hash + + expect(data[:inventory_object_attributes]).to match_array(%i(attr1 attr2 attr3)) + end + + it 'can remove inventory_object_attributes' do + data = cloud.prepare_data(:tmp, persister_class) do |builder| + builder.add_inventory_attributes(%i(attr1 attr2 attr3)) + builder.remove_inventory_attributes(%i(attr2)) + end.to_hash + + expect(data[:inventory_object_attributes]).to match_array(%i(attr1 attr3)) + end + + it 'can clear all inventory_object_attributes' do + data = cloud.prepare_data(:vms, persister_class) do |builder| + builder.add_inventory_attributes(%i(attr1 attr2 attr3)) + builder.clear_inventory_attributes! + end.to_hash + + expect(data[:inventory_object_attributes]).to be_empty + end +end