Skip to content

Commit

Permalink
Merge pull request #19684 from kbrock/metrics_capture_hierarchy
Browse files Browse the repository at this point in the history 
Metrics capture hierarchy
  • Loading branch information
@agrare
agrare authored Jan 7, 2020
2 parents 7a5a5f7 + 5f25d71 commit 953fcd8
Show file tree
Hide file tree
Showing 9 changed files with 115 additions and 145 deletions.
2 changes: 1 addition & 1 deletion app/models/manageiq/providers/base_manager/metrics_capture.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ def filter_perf_capture_now(targets, target_options)
end

def capture_ems_targets(options = {})
Metric::Targets.capture_ems_targets(ems, options)
raise(NotImplementedError, _("must be implemented in subclass"))
end

# if it has not been run, or it was a very long time ago, just run it
Expand Down
1 change: 1 addition & 0 deletions app/models/manageiq/providers/cloud_manager.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ class CloudManager < BaseManager
require_nested :AuthKeyPair
require_nested :RefreshParser
require_nested :Template
require_nested :MetricsCapture
require_nested :Provision
require_nested :ProvisionWorkflow
require_nested :Vm
Expand Down
9 changes: 8 additions & 1 deletion app/models/manageiq/providers/cloud_manager/metrics_capture.rb
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
class ManageIQ::Providers::CloudManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture
# @return vms under all availability zones
# and vms under no availability zone
# NOTE: some stacks (e.g. nova) default to no availability zone
def capture_ems_targets(options = {})
Metric::Targets.capture_cloud_targets([ems], options)
MiqPreloader.preload([ems], :vms => [{:availability_zone => :tags}, :ext_management_system])

ems.vms.select do |vm|
vm.state == 'on' && (vm.availability_zone.nil? || vm.availability_zone.perf_capture_enabled?)
end
end
end
1 change: 1 addition & 0 deletions app/models/manageiq/providers/container_manager.rb
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
module ManageIQ::Providers
class ContainerManager < BaseManager
require_nested :ContainerTemplate
require_nested :MetricsCapture
require_nested :OrchestrationStack

include AvailabilityMixin
Expand Down
16 changes: 14 additions & 2 deletions app/models/manageiq/providers/container_manager/metrics_capture.rb
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
class ManageIQ::Providers::ContainerManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture
def capture_ems_targets(options = {})
Metric::Targets.capture_container_targets([ems], options)
def capture_ems_targets(_options = {})
return [] unless ems.supports_metrics?

MiqPreloader.preload([ems], :container_nodes => :tags, :container_groups => [:tags, :containers => :tags])

with_archived(ems.all_container_nodes) + with_archived(ems.all_container_groups) + with_archived(ems.all_containers)
end

private

def with_archived(scope)
# We will look also for freshly archived entities, if the entity was short-lived or even sub-hour
archived_from = targets_archived_from
scope.where(:deleted_on => nil).or(scope.where(:deleted_on => (archived_from..Time.now.utc)))
end
end
1 change: 1 addition & 0 deletions app/models/manageiq/providers/infra_manager.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ class InfraManager < BaseManager
require_nested :Cluster
require_nested :Datacenter
require_nested :Folder
require_nested :MetricsCapture
require_nested :ProvisionWorkflow
require_nested :ResourcePool
require_nested :Storage
Expand Down
88 changes: 87 additions & 1 deletion app/models/manageiq/providers/infra_manager/metrics_capture.rb
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,91 @@
class ManageIQ::Providers::InfraManager::MetricsCapture < ManageIQ::Providers::BaseManager::MetricsCapture
def capture_ems_targets(options = {})
Metric::Targets.capture_infra_targets([ems], options)
load_infra_targets_data(ems, options)
all_hosts = capture_host_targets(ems)
targets = enabled_hosts = only_enabled(all_hosts)
targets += capture_storage_targets(all_hosts) unless options[:exclude_storages]
targets += capture_vm_targets(ems, enabled_hosts)

targets
end

private

# Filter to enabled hosts. If it has a cluster consult that, otherwise consult the host itself.
#
# NOTE: if capture_storage takes only enabled, then move
# this logic into capture_host_targets
def only_enabled(hosts)
hosts.select do |host|
host.supports_capture? && (host.ems_cluster ? host.ems_cluster.perf_capture_enabled? : host.perf_capture_enabled?)
end
end

# preload emses with relations that will be used in cap&u
#
# tags are needed for determining if it is enabled.
# ems is needed for determining queue name
# cluster is used for hierarchies
def load_infra_targets_data(ems, options)
MiqPreloader.preload(ems, preload_hash_infra_targets_data(options))
postload_infra_targets_data(ems, options)
end

def preload_hash_infra_targets_data(options)
# Preload all of the objects we are going to be inspecting.
includes = {:hosts => {:ems_cluster => :tags, :tags => {}}}
includes[:hosts][:storages] = :tags unless options[:exclude_storages]
includes[:vms] = {}
includes
end

# populate parts of the hierarchy that are not properly preloaded
#
# inverse_of does not work with :through.
# e.g.: :ems => :hosts => vms will not preload vms.ems
#
# adding in a preload for vms => :ems will fix, but different objects get assigned
# and since we also rely upon tags and clusters, this causes unnecessary data to be downloaded
#
# so we have introduced this to work around preload not working (and inverse_of)
def postload_infra_targets_data(ems, options)
# populate ems (with tags / clusters)
ems.hosts.each do |host|
host.ems_cluster.association(:ext_management_system).target = ems if host.ems_cluster_id
next if options[:exclude_storages]

host.storages.each do |storage|
storage.ext_management_system = ems
end
end
host_ids = ems.hosts.index_by(&:id)
clusters = ems.hosts.flat_map(&:ems_cluster).uniq.compact.index_by(&:id)
ems.vms.each do |vm|
vm.association(:ext_management_system).target = ems if vm.ems_id
vm.association(:ems_cluster).target = clusters[vm.ems_cluster_id] if vm.ems_cluster_id
vm.association(:host).target = host_ids[vm.host_id] if vm.host_id
end
end

def capture_host_targets(ems)
# NOTE: if capture_storage_targets takes only enabled hosts
# merge only_enabled into this method
ems.hosts
end

# @param [Array<Host>] all hosts that have an ems
# NOTE: disabled hosts are passed in.
# @return [Array<Storage>] supported storages
# hosts preloaded storages and tags
def capture_storage_targets(hosts)
hosts.flat_map(&:storages).uniq.select { |s| Storage.supports?(s.store_type) & s.perf_capture_enabled? }
end

# @param [ExtManagementSystem] ems
# @param [Array<Host>] hosts that are enabled or cluster enabled
# we want to work with only enabled_hosts, so hosts needs to be further filtered
def capture_vm_targets(ems, hosts)
enabled_host_ids = hosts.select(&:perf_capture_enabled?).index_by(&:id)
ems.vms.select { |v| enabled_host_ids.key?(v.host_id) && v.state == 'on' && v.supports_capture? }
end
end
138 changes: 0 additions & 138 deletions app/models/metric/targets.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,142 +12,4 @@ def self.targets_archived_from
archived_for_setting = Settings.performance.targets.archived_for
archived_for_setting.to_i_with_method.seconds.ago.utc
end

def self.capture_ems_targets(ems, options = {})
case ems
when EmsCloud then capture_cloud_targets([ems], options)
when EmsInfra then capture_infra_targets([ems], options)
when ::ManageIQ::Providers::ContainerManager then capture_container_targets([ems], options)
end
end

# If a Cluster, standalone Host, or Storage is not enabled, skip it.
# If a Cluster is enabled, capture all of its Hosts.
# If a Host is enabled, capture all of its Vms.
def self.capture_infra_targets(emses, options)
load_infra_targets_data(emses, options)
all_hosts = capture_host_targets(emses)
targets = enabled_hosts = only_enabled(all_hosts)
targets += capture_storage_targets(all_hosts) unless options[:exclude_storages]
targets += capture_vm_targets(emses, enabled_hosts)

targets
end

# Filter to enabled hosts. If it has a cluster consult that, otherwise consult the host itself.
#
# NOTE: if capture_storage takes only enabled, then move
# this logic into capture_host_targets
def self.only_enabled(hosts)
hosts.select do |host|
host.supports_capture? && (host.ems_cluster ? host.ems_cluster.perf_capture_enabled? : host.perf_capture_enabled?)
end
end

# @return vms under all availability zones
# and vms under no availability zone
# NOTE: some stacks (e.g. nova) default to no availability zone
def self.capture_cloud_targets(emses, options = {})
MiqPreloader.preload(emses, :vms => [{:availability_zone => :tags}, :ext_management_system])

emses.flat_map(&:vms).select do |vm|
vm.state == 'on' && (vm.availability_zone.nil? || vm.availability_zone.perf_capture_enabled?)
end
end

def self.with_archived(scope)
# We will look also for freshly archived entities, if the entity was short-lived or even sub-hour
archived_from = targets_archived_from
scope.where(:deleted_on => nil).or(scope.where(:deleted_on => (archived_from..Time.now.utc)))
end

def self.capture_container_targets(emses, _options)
includes = {
:container_nodes => :tags,
:container_groups => [:tags, :containers => :tags],
}

MiqPreloader.preload(emses, includes)

targets = []
emses.each do |ems|
next unless ems.supports_metrics?

targets += with_archived(ems.all_container_nodes)
targets += with_archived(ems.all_container_groups)
targets += with_archived(ems.all_containers)
end

targets
end

# preload emses with relations that will be used in cap&u
#
# tags are needed for determining if it is enabled.
# ems is needed for determining queue name
# cluster is used for hierarchies
def self.load_infra_targets_data(emses, options)
MiqPreloader.preload(emses, preload_hash_infra_targets_data(options))
postload_infra_targets_data(emses, options)
end

def self.preload_hash_infra_targets_data(options)
# Preload all of the objects we are going to be inspecting.
includes = {:hosts => {:ems_cluster => :tags, :tags => {}}}
includes[:hosts][:storages] = :tags unless options[:exclude_storages]
includes[:vms] = {}
includes
end

# populate parts of the hierarchy that are not properly preloaded
#
# inverse_of does not work with :through.
# e.g.: :ems => :hosts => vms will not preload vms.ems
#
# adding in a preload for vms => :ems will fix, but different objects get assigned
# and since we also rely upon tags and clusters, this causes unnecessary data to be downloaded
#
# so we have introduced this to work around preload not working (and inverse_of)
def self.postload_infra_targets_data(emses, options)
# populate ems (with tags / clusters)
emses.each do |ems|
ems.hosts.each do |host|
host.ems_cluster.association(:ext_management_system).target = ems if host.ems_cluster_id
unless options[:exclude_storages]
host.storages.each do |storage|
storage.ext_management_system = ems
end
end
end
host_ids = ems.hosts.index_by(&:id)
clusters = ems.hosts.flat_map(&:ems_cluster).uniq.compact.index_by(&:id)
ems.vms.each do |vm|
vm.association(:ext_management_system).target = ems if vm.ems_id
vm.association(:ems_cluster).target = clusters[vm.ems_cluster_id] if vm.ems_cluster_id
vm.association(:host).target = host_ids[vm.host_id] if vm.host_id
end
end
end

def self.capture_host_targets(emses)
# NOTE: if capture_storage_targets takes only enabled hosts
# merge only_enabled into this method
emses.flat_map(&:hosts)
end

# @param [Host] all hosts that have an ems
# NOTE: disabled hosts are passed in.
# @return [Array<Storage>] supported storages
# hosts preloaded storages and tags
def self.capture_storage_targets(hosts)
hosts.flat_map(&:storages).uniq.select { |s| Storage.supports?(s.store_type) & s.perf_capture_enabled? }
end

# @param [Array<ExtManagementSystem>] emses Typically 1 ems for this zone
# @param [Host] hosts that are enabled or cluster enabled
# we want to work with only enabled_hosts, so hosts needs to be further filtered
def self.capture_vm_targets(emses, hosts)
enabled_host_ids = hosts.select(&:perf_capture_enabled?).index_by(&:id)
emses.flat_map { |e| e.vms.select { |v| enabled_host_ids.key?(v.host_id) && v.state == 'on' && v.supports_capture? } }
end
end
4 changes: 2 additions & 2 deletions spec/models/manageiq/providers/infra_manager/metrics_capture_spec.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@

describe ".capture_ems_targets" do
it "finds enabled targets" do
targets = described_class.new(nil, ems).send(:capture_ems_targets)
targets = described_class.new(nil, ems).capture_ems_targets
assert_infra_targets_enabled targets
expect(targets.map { |t| t.class.name }).to match_array(%w[ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host Storage])
end

it "finds enabled targets excluding storages" do
targets = described_class.new(nil, ems).send(:capture_ems_targets, :exclude_storages => true)
targets = described_class.new(nil, ems).capture_ems_targets(:exclude_storages => true)
assert_infra_targets_enabled targets
expect(targets.map { |t| t.class.name }).to match_array(%w[ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Host ManageIQ::Providers::Vmware::InfraManager::Vm ManageIQ::Providers::Vmware::InfraManager::Host])
end
Expand Down

0 comments on commit 953fcd8

Please sign in to comment.