Implement graph inventory refresh for cloud manager

With this commit we implement graph inventory refresh for cloud manager.
Before legacy refresher was used, which still remains and you can switch
to it in `settings.yml`. Both refreshers use and pass the same unit tests.

Signed-off-by: sasoc <saso.cvitkovic@gmail.com>

app/models/manageiq/providers/vmware/builder.rb

@@ -0,0 +1,30 @@
+class ManageIQ::Providers::Vmware::Builder
+ class << self
+ def build_inventory(ems, target)
+ cloud_manager_inventory(ems, target)
+ end
+
+ private
+
+ def cloud_manager_inventory(ems, target)
+ inventory(
+ ems,
+ target,
+ ManageIQ::Providers::Vmware::Inventory::Collector::CloudManager,
+ ManageIQ::Providers::Vmware::Inventory::Persister::CloudManager,
+ [ManageIQ::Providers::Vmware::Inventory::Parser::CloudManager]
+ )
+ end
+
+ def inventory(manager, raw_target, collector_class, persister_class, parsers_classes)
+ collector = collector_class.new(manager, raw_target)
+ persister = persister_class.new(manager, raw_target)
+
+ ::ManageIQ::Providers::Vmware::Inventory.new(
+ persister,
+ collector,
+ parsers_classes.map(&:new)
+ )
+ end
+ end
+end

app/models/manageiq/providers/vmware/cloud_manager.rb

@@ -15,6 +15,8 @@ class ManageIQ::Providers::Vmware::CloudManager < ManageIQ::Providers::CloudMana
  include ManageIQ::Providers::Vmware::CloudManager::ManagerEventsMixin
  include HasNetworkManagerMixin
 
+ has_many :snapshots, :through => :vms_and_templates
+
  before_create :ensure_managers
 
  def ensure_network_manager

app/models/manageiq/providers/vmware/cloud_manager/refresher.rb

@@ -13,4 +13,37 @@ def save_inventory(ems, target, hashes)
  def post_process_refresh_classes
  [::Vm]
  end
+
+ def collect_inventory_for_targets(ems, targets)
+ log_header = format_ems_for_logging(ems)
+ targets_with_data = targets.collect do |target|
+ target_name = target.try(:name) || target.try(:event_type)
+
+ _log.info("#{log_header} Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]...")
+
+ if refresher_options.try(:[], :inventory_object_refresh)
+ inventory = ManageIQ::Providers::Vmware::Builder.build_inventory(ems, target)
+ end
+
+ _log.info("#{log_header} Filtering inventory...Complete")
+ [target, inventory]
+ end
+
+ targets_with_data
+ end
+
+ def parse_targeted_inventory(ems, _target, inventory)
+ log_header = format_ems_for_logging(ems)
+ _log.debug("#{log_header} Parsing inventory...")
+ hashes, = Benchmark.realtime_block(:parse_inventory) do
+ if refresher_options.try(:[], :inventory_object_refresh)
+ inventory.inventory_collections
+ else
+ ManageIQ::Providers::Vmware::CloudManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+ end
+ end
+ _log.debug("#{log_header} Parsing inventory...Complete")
+
+ hashes
+ end
 end

app/models/manageiq/providers/vmware/cloud_manager/snapshot.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Vmware::CloudManager::Snapshot < ::Snapshot
+end

app/models/manageiq/providers/vmware/inventory.rb

@@ -0,0 +1,5 @@
+class ManageIQ::Providers::Vmware::Inventory < ManagerRefresh::Inventory
+ require_nested :Collector
+ require_nested :Parser
+ require_nested :Persister
+end

app/models/manageiq/providers/vmware/inventory/collector.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Collector < ManagerRefresh::Inventory::Collector
+ require_nested :CloudManager
+
+ def initialize(_manager, _target)
+ super
+
+ initialize_inventory_sources
+ end
+
+ def initialize_inventory_sources
+ @orgs = []
+ @vdcs = []
+ @vapps = []
+ @vms = []
+ @vapp_templates = []
+ @images = []
+ end
+
+ def connection
+ @connection ||= manager.connect
+ end
+
+ def public_images?
+ options.try(:get_public_images)
+ end
+end

app/models/manageiq/providers/vmware/inventory/collector/cloud_manager.rb

@@ -0,0 +1,75 @@
+class ManageIQ::Providers::Vmware::Inventory::Collector::CloudManager < ManageIQ::Providers::Vmware::Inventory::Collector
+ VAPP_TEMPLATE_STATUS_READY = "8".freeze
+
+ def orgs
+ return @orgs if @orgs.any?
+ @orgs = connection.organizations
+ end
+
+ def vdcs
+ return @vdcs if @vdcs.any?
+ @orgs.each do |org|
+ @vdcs += org.vdcs.all
+ end
+
+ @vdcs
+ end
+
+ def vapps
+ return @vapps if @vapps.any?
+ @vdcs.each do |vdc|
+ @vapps += vdc.vapps.all
+ end
+
+ @vapps
+ end
+
+ def vms
+ return @vms if @vms.any?
+ @vapps.each do |vapp|
+ # Remove this each loop, once fog api will be updated to send hostname and snapshot together with vms
+ vapp.vms.each do |vm|
+ @vms << {
+ :vm => vm,
+ :hostname => vm.customization.try(:computer_name),
+ :snapshot => connection.get_snapshot_section(vm.id).data
+ }
+ end
+ end
+
+ @vms
+ end
+
+ def vapp_templates
+ return @vapp_templates if @vapp_templates.any?
+ @orgs.each do |org|
+ org.catalogs.each do |catalog|
+ next if catalog.is_published && !public_images?
+
+ catalog.catalog_items.each do |item|
+ # Skip all Catalog Items which are not vApp Templates (e.g. Media & Other)
+ next unless item.vapp_template_id.starts_with?('vappTemplate-')
+
+ if (t = item.vapp_template) && t.status == VAPP_TEMPLATE_STATUS_READY
+ @vapp_templates << {
+ :vapp_template => t,
+ :is_published => catalog.is_published,
+ :content => connection.get_vapp_template_ovf_descriptor(t.id).body
+ }
+ end
+ end
+ end
+ end
+
+ @vapp_templates
+ end
+
+ def images
+ return @images if @images.any?
+ @vapp_templates.each do |template_obj|
+ @images += template_obj[:vapp_template].vms.map { |image| { :image => image, :is_published => template_obj[:is_published] } }
+ end
+
+ @images
+ end
+end

app/models/manageiq/providers/vmware/inventory/parser.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Parser < ManagerRefresh::Inventory::Parser
+ require_nested :CloudManager
+
+ # See https://pubs.vmware.com/vcd-80/index.jsp#com.vmware.vcloud.api.sp.doc_90/GUID-E1BA999D-87FA-4E2C-B638-24A211AB8160.html
+ def controller_description(bus_subtype)
+ case bus_subtype
+ when 'buslogic'
+ 'BusLogic Parallel SCSI controller'
+ when 'lsilogic'
+ 'LSI Logic Parallel SCSI controller'
+ when 'lsilogicsas'
+ 'LSI Logic SAS SCSI controller'
+ when 'VirtualSCSI'
+ 'Paravirtual SCSI controller'
+ when 'vmware.sata.ahci'
+ 'SATA controller'
+ else
+ 'IDE controller'
+ end
+ end
+
+ # See https://pubs.vmware.com/vcd-80/index.jsp#com.vmware.vcloud.api.sp.doc_90/GUID-E1BA999D-87FA-4E2C-B638-24A211AB8160.html
+ def hdd?(bus_type)
+ [5, 6, 20].include?(bus_type)
+ end
+end

app/models/manageiq/providers/vmware/inventory/parser/cloud_manager.rb

@@ -0,0 +1,119 @@
+class ManageIQ::Providers::Vmware::Inventory::Parser::CloudManager < ManageIQ::Providers::Vmware::Inventory::Parser
+ def parse
+ orgs
+ vdcs
+ vapps
+ vms
+ vapp_templates
+ images
+ end
+
+ private
+
+ def orgs
+ collector.orgs
+ end
+
+ def vdcs
+ collector.vdcs.each do |vdc|
+ persister.availability_zones.find_or_build(vdc.id).assign_attributes(
+ :ems_ref => vdc.id,
+ :name => vdc.name
+ )
+ end
+ end
+
+ def vapps
+ collector.vapps.each do |vapp|
+ persister.orchestration_stacks.find_or_build(vapp.id).assign_attributes(
+ :ems_ref => vapp.id,
+ :name => vapp.name,
+ :description => vapp.name,
+ :status => vapp.human_status,
+ )
+ end
+ end
+
+ def vms
+ collector.vms.each do |vm|
+ parsed_vm = persister.vms.find_or_build(vm[:vm].id).assign_attributes(
+ :uid_ems => vm[:vm].id,
+ :ems_ref => vm[:vm].id,
+ :name => vm[:vm].name,
+ :hostname => vm[:hostname],
+ :vendor => 'vmware',
+ :raw_power_state => vm[:vm].status,
+ :orchestration_stack => persister.orchestration_stacks.lazy_find(vm[:vm].vapp_id),
+ )
+
+ if (snapshot_resp = vm[:snapshot].fetch_path(:body, :Snapshot))
+ snapshot = persister.snapshots.find_or_build(parsed_vm).assign_attributes(
+ :name => "#{vm[:vm].name} (snapshot)",
+ :vm_or_template_id => parsed_vm.id,
+ :uid => "#{vm[:vm].id}_#{snapshot_resp[:created]}",
+ :ems_ref => "#{vm[:vm].id}_#{snapshot_resp[:created]}",
+ :parent_id => vm[:vm].id,
+ :parent_uid => vm[:vm].id,
+ :create_time => snapshot_resp[:created],
+ :total_size => snapshot_resp[:size]
+ )
+
+ parsed_vm.snapshots = [snapshot]
+ end
+
+ hardware = persister.hardwares.find_or_build(parsed_vm).assign_attributes(
+ :guest_os => vm[:vm].operating_system,
+ :guest_os_full_name => vm[:vm].operating_system,
+ :bitness => vm[:vm].operating_system =~ /64-bit/ ? 64 : 32,
+ :cpu_sockets => vm[:vm].cpu,
+ :cpu_cores_per_socket => 1,
+ :cpu_total_cores => vm[:vm].cpu,
+ :memory_mb => vm[:vm].memory,
+ :disk_capacity => vm[:vm].hard_disks.inject(0) { |sum, x| sum + x.values[0] } * 1.megabyte,
+ )
+
+ vm[:vm].disks.all.select { |d| hdd? d.bus_type }.map do |disk|
+ persister.disks.find_or_build_by(:hardware => hardware, :device_name => disk.name).assign_attributes(
+ :device_type => 'disk',
+ :controller_type => controller_description(disk.bus_sub_type),
+ :size => disk.capacity * 1.megabyte,
+ :location => "#{vm[:vm].id}-#{disk.id}",
+ :filename => "#{vm[:vm].id}-#{disk.id}",
+ )
+ end
+
+ persister.operating_systems.find_or_build(parsed_vm).assign_attributes(
+ :product_name => vm[:vm].operating_system,
+ )
+ end
+ end
+
+ def vapp_templates
+ collector.vapp_templates.each do |vapp_template|
+ persister.orchestration_templates.find_or_build(vapp_template[:vapp_template].id).assign_attributes(
+ :ems_ref => vapp_template[:vapp_template].id,
+ :name => vapp_template[:vapp_template].name,
+ :description => vapp_template[:vapp_template].description,
+ :orderable => true,
+ :content => "<!-- #{vapp_template[:vapp_template].id} -->\n#{vapp_template[:content]}",
+ # By default #save_orchestration_templates_inventory does not set the EMS
+ # ID because templates are not EMS specific. We are setting the EMS
+ # explicitly here, because vapps are specific to concrete EMS.
+ :ems_id => collector.manager.id
+ )
+ end
+ end
+
+ def images
+ collector.images.each do |image|
+ persister.miq_templates.find_or_build(image[:image].id).assign_attributes(
+ :uid_ems => image[:image].id,
+ :ems_ref => image[:image].id,
+ :name => image[:image].name,
+ :vendor => 'vmware',
+ :raw_power_state => 'never',
+ :publicly_available => image[:is_published]
+ )
+ end
+ end
+end

app/models/manageiq/providers/vmware/inventory/persister.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Persister < ManagerRefresh::Inventory::Persister
+ require_nested :CloudManager
+
+ protected
+
+ def cloud
+ ManageIQ::Providers::Vmware::InventoryCollectionDefault::CloudManager
+ end
+
+ def targeted
+ false
+ end
+
+ def strategy
+ nil
+ end
+
+ def shared_options
+ settings_options = options[:inventory_collections].try(:to_hash) || {}
+
+ settings_options.merge(
+ :strategy => strategy,
+ :targeted => targeted,
+ )
+ end
+end

app/models/manageiq/providers/vmware/inventory/persister/cloud_manager.rb

@@ -0,0 +1,18 @@
+class ManageIQ::Providers::Vmware::Inventory::Persister::CloudManager < ManageIQ::Providers::Vmware::Inventory::Persister
+ def initialize_inventory_collections
+ add_inventory_collections(
+ cloud,
+ %i(
+ availability_zones
+ orchestration_stacks
+ vms
+ snapshots
+ hardwares
+ disks
+ operating_systems
+ orchestration_templates
+ miq_templates
+ )
+ )
+ end
+end