From 0aee80d2fb42ed0f2fcf44151c010096ec8dddeb Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Fri, 3 Jan 2020 14:30:23 -0500 Subject: [PATCH 1/4] move container targets to metrics capture --- .../manageiq/providers/container_manager.rb | 1 + .../container_manager/metrics_capture.rb | 16 +++++++++-- app/models/metric/targets.rb | 27 ------------------- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/app/models/manageiq/providers/container_manager.rb b/app/models/manageiq/providers/container_manager.rb index 4b55aa71b4f..80e770de576 100644 --- a/app/models/manageiq/providers/container_manager.rb +++ b/app/models/manageiq/providers/container_manager.rb @@ -1,6 +1,7 @@ module ManageIQ::Providers class ContainerManager < BaseManager require_nested :ContainerTemplate + require_nested :MetricsCapture require_nested :OrchestrationStack include AvailabilityMixin diff --git a/app/models/manageiq/providers/container_manager/metrics_capture.rb b/app/models/manageiq/providers/container_manager/metrics_capture.rb index f1e170914a2..842ede3d4f8 100644 --- a/app/models/manageiq/providers/container_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/container_manager/metrics_capture.rb @@ -1,5 +1,17 @@ class ManageIQ::Providers::ContainerManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture - def capture_ems_targets(options = {}) - Metric::Targets.capture_container_targets([ems], options) + def capture_ems_targets(_options = {}) + return [] unless ems.supports_metrics? + + MiqPreloader.preload([ems], :container_nodes => :tags, :container_groups => [:tags, :containers => :tags]) + + with_archived(ems.all_container_nodes) + with_archived(ems.all_container_groups) + with_archived(ems.all_containers) + end + + private + + def with_archived(scope) + # We will look also for freshly archived entities, if the entity was short-lived or even sub-hour + archived_from = targets_archived_from + scope.where(:deleted_on => nil).or(scope.where(:deleted_on => (archived_from..Time.now.utc))) end end diff --git a/app/models/metric/targets.rb b/app/models/metric/targets.rb index bf8a5f17db7..db623cc22e4 100644 --- a/app/models/metric/targets.rb +++ b/app/models/metric/targets.rb @@ -17,7 +17,6 @@ def self.capture_ems_targets(ems, options = {}) case ems when EmsCloud then capture_cloud_targets([ems], options) when EmsInfra then capture_infra_targets([ems], options) - when ::ManageIQ::Providers::ContainerManager then capture_container_targets([ems], options) end end @@ -55,32 +54,6 @@ def self.capture_cloud_targets(emses, options = {}) end end - def self.with_archived(scope) - # We will look also for freshly archived entities, if the entity was short-lived or even sub-hour - archived_from = targets_archived_from - scope.where(:deleted_on => nil).or(scope.where(:deleted_on => (archived_from..Time.now.utc))) - end - - def self.capture_container_targets(emses, _options) - includes = { - :container_nodes => :tags, - :container_groups => [:tags, :containers => :tags], - } - - MiqPreloader.preload(emses, includes) - - targets = [] - emses.each do |ems| - next unless ems.supports_metrics? - - targets += with_archived(ems.all_container_nodes) - targets += with_archived(ems.all_container_groups) - targets += with_archived(ems.all_containers) - end - - targets - end - # preload emses with relations that will be used in cap&u # # tags are needed for determining if it is enabled. From 0e63febf1acf0c8b0ef7b4fd8fd298e5ea7c8c2e Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Fri, 3 Jan 2020 14:31:36 -0500 Subject: [PATCH 2/4] move cloud targets to metrics capture --- app/models/manageiq/providers/cloud_manager.rb | 1 + .../providers/cloud_manager/metrics_capture.rb | 9 ++++++++- app/models/metric/targets.rb | 12 ------------ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/app/models/manageiq/providers/cloud_manager.rb b/app/models/manageiq/providers/cloud_manager.rb index e54094db6ee..5a77c8900f3 100644 --- a/app/models/manageiq/providers/cloud_manager.rb +++ b/app/models/manageiq/providers/cloud_manager.rb @@ -3,6 +3,7 @@ class CloudManager < BaseManager require_nested :AuthKeyPair require_nested :RefreshParser require_nested :Template + require_nested :MetricsCapture require_nested :Provision require_nested :ProvisionWorkflow require_nested :Vm diff --git a/app/models/manageiq/providers/cloud_manager/metrics_capture.rb b/app/models/manageiq/providers/cloud_manager/metrics_capture.rb index f637ca2f9ec..bfd4c02e257 100644 --- a/app/models/manageiq/providers/cloud_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/cloud_manager/metrics_capture.rb @@ -1,5 +1,12 @@ class ManageIQ::Providers::CloudManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture + # @return vms under all availability zones + # and vms under no availability zone + # NOTE: some stacks (e.g. nova) default to no availability zone def capture_ems_targets(options = {}) - Metric::Targets.capture_cloud_targets([ems], options) + MiqPreloader.preload([ems], :vms => [{:availability_zone => :tags}, :ext_management_system]) + + ems.vms.select do |vm| + vm.state == 'on' && (vm.availability_zone.nil? || vm.availability_zone.perf_capture_enabled?) + end end end diff --git a/app/models/metric/targets.rb b/app/models/metric/targets.rb index db623cc22e4..6dabd640cbb 100644 --- a/app/models/metric/targets.rb +++ b/app/models/metric/targets.rb @@ -15,7 +15,6 @@ def self.targets_archived_from def self.capture_ems_targets(ems, options = {}) case ems - when EmsCloud then capture_cloud_targets([ems], options) when EmsInfra then capture_infra_targets([ems], options) end end @@ -43,17 +42,6 @@ def self.only_enabled(hosts) end end - # @return vms under all availability zones - # and vms under no availability zone - # NOTE: some stacks (e.g. nova) default to no availability zone - def self.capture_cloud_targets(emses, options = {}) - MiqPreloader.preload(emses, :vms => [{:availability_zone => :tags}, :ext_management_system]) - - emses.flat_map(&:vms).select do |vm| - vm.state == 'on' && (vm.availability_zone.nil? || vm.availability_zone.perf_capture_enabled?) - end - end - # preload emses with relations that will be used in cap&u # # tags are needed for determining if it is enabled. From 265f8aebcdcb62cd4da91b2fe6fab1a6024fc1a3 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Fri, 3 Jan 2020 14:34:31 -0500 Subject: [PATCH 3/4] move infra targets to metrics capture --- .../providers/base_manager/metrics_capture.rb | 2 +- .../manageiq/providers/infra_manager.rb | 1 + .../infra_manager/metrics_capture.rb | 97 +++++++++++++++++- app/models/metric/targets.rb | 99 ------------------- 4 files changed, 98 insertions(+), 101 deletions(-) diff --git a/app/models/manageiq/providers/base_manager/metrics_capture.rb b/app/models/manageiq/providers/base_manager/metrics_capture.rb index ce4473d25d4..003c992a50b 100644 --- a/app/models/manageiq/providers/base_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/base_manager/metrics_capture.rb @@ -80,7 +80,7 @@ def filter_perf_capture_now(targets, target_options) end def capture_ems_targets(options = {}) - Metric::Targets.capture_ems_targets(ems, options) + raise(NotImplementedError, _("must be implemented in subclass")) end # if it has not been run, or it was a very long time ago, just run it diff --git a/app/models/manageiq/providers/infra_manager.rb b/app/models/manageiq/providers/infra_manager.rb index 8a29fa6c262..f1756bb9daf 100644 --- a/app/models/manageiq/providers/infra_manager.rb +++ b/app/models/manageiq/providers/infra_manager.rb @@ -3,6 +3,7 @@ class InfraManager < BaseManager require_nested :Cluster require_nested :Datacenter require_nested :Folder + require_nested :MetricsCapture require_nested :ProvisionWorkflow require_nested :ResourcePool require_nested :Storage diff --git a/app/models/manageiq/providers/infra_manager/metrics_capture.rb b/app/models/manageiq/providers/infra_manager/metrics_capture.rb index df81a47e1e8..6fccd3f77b5 100644 --- a/app/models/manageiq/providers/infra_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/infra_manager/metrics_capture.rb @@ -1,5 +1,100 @@ class ManageIQ::Providers::InfraManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture def capture_ems_targets(options = {}) - Metric::Targets.capture_infra_targets([ems], options) + capture_infra_targets([ems], options) + end + + private + + # If a Cluster, standalone Host, or Storage is not enabled, skip it. + # If a Cluster is enabled, capture all of its Hosts. + # If a Host is enabled, capture all of its Vms. + def capture_infra_targets(emses, options) + load_infra_targets_data(emses, options) + all_hosts = capture_host_targets(emses) + targets = enabled_hosts = only_enabled(all_hosts) + targets += capture_storage_targets(all_hosts) unless options[:exclude_storages] + targets += capture_vm_targets(emses, enabled_hosts) + + targets + end + + # Filter to enabled hosts. If it has a cluster consult that, otherwise consult the host itself. + # + # NOTE: if capture_storage takes only enabled, then move + # this logic into capture_host_targets + def only_enabled(hosts) + hosts.select do |host| + host.supports_capture? && (host.ems_cluster ? host.ems_cluster.perf_capture_enabled? : host.perf_capture_enabled?) + end + end + + # preload emses with relations that will be used in cap&u + # + # tags are needed for determining if it is enabled. + # ems is needed for determining queue name + # cluster is used for hierarchies + def load_infra_targets_data(emses, options) + MiqPreloader.preload(emses, preload_hash_infra_targets_data(options)) + postload_infra_targets_data(emses, options) + end + + def preload_hash_infra_targets_data(options) + # Preload all of the objects we are going to be inspecting. + includes = {:hosts => {:ems_cluster => :tags, :tags => {}}} + includes[:hosts][:storages] = :tags unless options[:exclude_storages] + includes[:vms] = {} + includes + end + + # populate parts of the hierarchy that are not properly preloaded + # + # inverse_of does not work with :through. + # e.g.: :ems => :hosts => vms will not preload vms.ems + # + # adding in a preload for vms => :ems will fix, but different objects get assigned + # and since we also rely upon tags and clusters, this causes unnecessary data to be downloaded + # + # so we have introduced this to work around preload not working (and inverse_of) + def postload_infra_targets_data(emses, options) + # populate ems (with tags / clusters) + emses.each do |ems| + ems.hosts.each do |host| + host.ems_cluster.association(:ext_management_system).target = ems if host.ems_cluster_id + unless options[:exclude_storages] + host.storages.each do |storage| + storage.ext_management_system = ems + end + end + end + host_ids = ems.hosts.index_by(&:id) + clusters = ems.hosts.flat_map(&:ems_cluster).uniq.compact.index_by(&:id) + ems.vms.each do |vm| + vm.association(:ext_management_system).target = ems if vm.ems_id + vm.association(:ems_cluster).target = clusters[vm.ems_cluster_id] if vm.ems_cluster_id + vm.association(:host).target = host_ids[vm.host_id] if vm.host_id + end + end + end + + def capture_host_targets(emses) + # NOTE: if capture_storage_targets takes only enabled hosts + # merge only_enabled into this method + emses.flat_map(&:hosts) + end + + # @param [Host] all hosts that have an ems + # NOTE: disabled hosts are passed in. + # @return [Array] supported storages + # hosts preloaded storages and tags + def capture_storage_targets(hosts) + hosts.flat_map(&:storages).uniq.select { |s| Storage.supports?(s.store_type) & s.perf_capture_enabled? } + end + + # @param [Array] emses Typically 1 ems for this zone + # @param [Host] hosts that are enabled or cluster enabled + # we want to work with only enabled_hosts, so hosts needs to be further filtered + def capture_vm_targets(emses, hosts) + enabled_host_ids = hosts.select(&:perf_capture_enabled?).index_by(&:id) + emses.flat_map { |e| e.vms.select { |v| enabled_host_ids.key?(v.host_id) && v.state == 'on' && v.supports_capture? } } end end diff --git a/app/models/metric/targets.rb b/app/models/metric/targets.rb index 6dabd640cbb..d6dbc140639 100644 --- a/app/models/metric/targets.rb +++ b/app/models/metric/targets.rb @@ -12,103 +12,4 @@ def self.targets_archived_from archived_for_setting = Settings.performance.targets.archived_for archived_for_setting.to_i_with_method.seconds.ago.utc end - - def self.capture_ems_targets(ems, options = {}) - case ems - when EmsInfra then capture_infra_targets([ems], options) - end - end - - # If a Cluster, standalone Host, or Storage is not enabled, skip it. - # If a Cluster is enabled, capture all of its Hosts. - # If a Host is enabled, capture all of its Vms. - def self.capture_infra_targets(emses, options) - load_infra_targets_data(emses, options) - all_hosts = capture_host_targets(emses) - targets = enabled_hosts = only_enabled(all_hosts) - targets += capture_storage_targets(all_hosts) unless options[:exclude_storages] - targets += capture_vm_targets(emses, enabled_hosts) - - targets - end - - # Filter to enabled hosts. If it has a cluster consult that, otherwise consult the host itself. - # - # NOTE: if capture_storage takes only enabled, then move - # this logic into capture_host_targets - def self.only_enabled(hosts) - hosts.select do |host| - host.supports_capture? && (host.ems_cluster ? host.ems_cluster.perf_capture_enabled? : host.perf_capture_enabled?) - end - end - - # preload emses with relations that will be used in cap&u - # - # tags are needed for determining if it is enabled. - # ems is needed for determining queue name - # cluster is used for hierarchies - def self.load_infra_targets_data(emses, options) - MiqPreloader.preload(emses, preload_hash_infra_targets_data(options)) - postload_infra_targets_data(emses, options) - end - - def self.preload_hash_infra_targets_data(options) - # Preload all of the objects we are going to be inspecting. - includes = {:hosts => {:ems_cluster => :tags, :tags => {}}} - includes[:hosts][:storages] = :tags unless options[:exclude_storages] - includes[:vms] = {} - includes - end - - # populate parts of the hierarchy that are not properly preloaded - # - # inverse_of does not work with :through. - # e.g.: :ems => :hosts => vms will not preload vms.ems - # - # adding in a preload for vms => :ems will fix, but different objects get assigned - # and since we also rely upon tags and clusters, this causes unnecessary data to be downloaded - # - # so we have introduced this to work around preload not working (and inverse_of) - def self.postload_infra_targets_data(emses, options) - # populate ems (with tags / clusters) - emses.each do |ems| - ems.hosts.each do |host| - host.ems_cluster.association(:ext_management_system).target = ems if host.ems_cluster_id - unless options[:exclude_storages] - host.storages.each do |storage| - storage.ext_management_system = ems - end - end - end - host_ids = ems.hosts.index_by(&:id) - clusters = ems.hosts.flat_map(&:ems_cluster).uniq.compact.index_by(&:id) - ems.vms.each do |vm| - vm.association(:ext_management_system).target = ems if vm.ems_id - vm.association(:ems_cluster).target = clusters[vm.ems_cluster_id] if vm.ems_cluster_id - vm.association(:host).target = host_ids[vm.host_id] if vm.host_id - end - end - end - - def self.capture_host_targets(emses) - # NOTE: if capture_storage_targets takes only enabled hosts - # merge only_enabled into this method - emses.flat_map(&:hosts) - end - - # @param [Host] all hosts that have an ems - # NOTE: disabled hosts are passed in. - # @return [Array] supported storages - # hosts preloaded storages and tags - def self.capture_storage_targets(hosts) - hosts.flat_map(&:storages).uniq.select { |s| Storage.supports?(s.store_type) & s.perf_capture_enabled? } - end - - # @param [Array] emses Typically 1 ems for this zone - # @param [Host] hosts that are enabled or cluster enabled - # we want to work with only enabled_hosts, so hosts needs to be further filtered - def self.capture_vm_targets(emses, hosts) - enabled_host_ids = hosts.select(&:perf_capture_enabled?).index_by(&:id) - emses.flat_map { |e| e.vms.select { |v| enabled_host_ids.key?(v.host_id) && v.state == 'on' && v.supports_capture? } } - end end From 5f25d71e4a17b9118ab73feb72914e8505ef0922 Mon Sep 17 00:00:00 2001 From: Keenan Brock Date: Mon, 6 Jan 2020 12:17:55 -0500 Subject: [PATCH 4/4] change infra targets to be single ems centric remove cop around exclude_storage guard vs next statement --- .../infra_manager/metrics_capture.rb | 67 ++++++++----------- .../infra_manager/metrics_capture_spec.rb | 4 +- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/app/models/manageiq/providers/infra_manager/metrics_capture.rb b/app/models/manageiq/providers/infra_manager/metrics_capture.rb index 6fccd3f77b5..07326030a49 100644 --- a/app/models/manageiq/providers/infra_manager/metrics_capture.rb +++ b/app/models/manageiq/providers/infra_manager/metrics_capture.rb @@ -1,23 +1,16 @@ class ManageIQ::Providers::InfraManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture def capture_ems_targets(options = {}) - capture_infra_targets([ems], options) - end - - private - - # If a Cluster, standalone Host, or Storage is not enabled, skip it. - # If a Cluster is enabled, capture all of its Hosts. - # If a Host is enabled, capture all of its Vms. - def capture_infra_targets(emses, options) - load_infra_targets_data(emses, options) - all_hosts = capture_host_targets(emses) + load_infra_targets_data(ems, options) + all_hosts = capture_host_targets(ems) targets = enabled_hosts = only_enabled(all_hosts) targets += capture_storage_targets(all_hosts) unless options[:exclude_storages] - targets += capture_vm_targets(emses, enabled_hosts) + targets += capture_vm_targets(ems, enabled_hosts) targets end + private + # Filter to enabled hosts. If it has a cluster consult that, otherwise consult the host itself. # # NOTE: if capture_storage takes only enabled, then move @@ -33,9 +26,9 @@ def only_enabled(hosts) # tags are needed for determining if it is enabled. # ems is needed for determining queue name # cluster is used for hierarchies - def load_infra_targets_data(emses, options) - MiqPreloader.preload(emses, preload_hash_infra_targets_data(options)) - postload_infra_targets_data(emses, options) + def load_infra_targets_data(ems, options) + MiqPreloader.preload(ems, preload_hash_infra_targets_data(options)) + postload_infra_targets_data(ems, options) end def preload_hash_infra_targets_data(options) @@ -55,34 +48,32 @@ def preload_hash_infra_targets_data(options) # and since we also rely upon tags and clusters, this causes unnecessary data to be downloaded # # so we have introduced this to work around preload not working (and inverse_of) - def postload_infra_targets_data(emses, options) + def postload_infra_targets_data(ems, options) # populate ems (with tags / clusters) - emses.each do |ems| - ems.hosts.each do |host| - host.ems_cluster.association(:ext_management_system).target = ems if host.ems_cluster_id - unless options[:exclude_storages] - host.storages.each do |storage| - storage.ext_management_system = ems - end - end - end - host_ids = ems.hosts.index_by(&:id) - clusters = ems.hosts.flat_map(&:ems_cluster).uniq.compact.index_by(&:id) - ems.vms.each do |vm| - vm.association(:ext_management_system).target = ems if vm.ems_id - vm.association(:ems_cluster).target = clusters[vm.ems_cluster_id] if vm.ems_cluster_id - vm.association(:host).target = host_ids[vm.host_id] if vm.host_id + ems.hosts.each do |host| + host.ems_cluster.association(:ext_management_system).target = ems if host.ems_cluster_id + next if options[:exclude_storages] + + host.storages.each do |storage| + storage.ext_management_system = ems end end + host_ids = ems.hosts.index_by(&:id) + clusters = ems.hosts.flat_map(&:ems_cluster).uniq.compact.index_by(&:id) + ems.vms.each do |vm| + vm.association(:ext_management_system).target = ems if vm.ems_id + vm.association(:ems_cluster).target = clusters[vm.ems_cluster_id] if vm.ems_cluster_id + vm.association(:host).target = host_ids[vm.host_id] if vm.host_id + end end - def capture_host_targets(emses) + def capture_host_targets(ems) # NOTE: if capture_storage_targets takes only enabled hosts # merge only_enabled into this method - emses.flat_map(&:hosts) + ems.hosts end - # @param [Host] all hosts that have an ems + # @param [Array] all hosts that have an ems # NOTE: disabled hosts are passed in. # @return [Array] supported storages # hosts preloaded storages and tags @@ -90,11 +81,11 @@ def capture_storage_targets(hosts) hosts.flat_map(&:storages).uniq.select { |s| Storage.supports?(s.store_type) & s.perf_capture_enabled? } end - # @param [Array] emses Typically 1 ems for this zone - # @param [Host] hosts that are enabled or cluster enabled + # @param [ExtManagementSystem] ems + # @param [Array] hosts that are enabled or cluster enabled # we want to work with only enabled_hosts, so hosts needs to be further filtered - def capture_vm_targets(emses, hosts) + def capture_vm_targets(ems, hosts) enabled_host_ids = hosts.select(&:perf_capture_enabled?).index_by(&:id) - emses.flat_map { |e| e.vms.select { |v| enabled_host_ids.key?(v.host_id) && v.state == 'on' && v.supports_capture? } } + ems.vms.select { |v| enabled_host_ids.key?(v.host_id) && v.state == 'on' && v.supports_capture? } end end diff --git a/spec/models/manageiq/providers/infra_manager/metrics_capture_spec.rb b/spec/models/manageiq/providers/infra_manager/metrics_capture_spec.rb index a82908ce37d..b158461a04d 100644 --- a/spec/models/manageiq/providers/infra_manager/metrics_capture_spec.rb +++ b/spec/models/manageiq/providers/infra_manager/metrics_capture_spec.rb @@ -25,13 +25,13 @@ describe ".capture_ems_targets" do it "finds enabled targets" do - targets = described_class.new(nil, ems).send(:capture_ems_targets) + targets = described_class.new(nil, ems).capture_ems_targets assert_infra_targets_enabled targets expect(targets.map { |t| t.class.name }).to match_array(%w[ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host Storage]) end it "finds enabled targets excluding storages" do - targets = described_class.new(nil, ems).send(:capture_ems_targets, :exclude_storages => true) + targets = described_class.new(nil, ems).capture_ems_targets(:exclude_storages => true) assert_infra_targets_enabled targets expect(targets.map { |t| t.class.name }).to match_array(%w[ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host]) end