Implement graph inventory refresh for cloud manager

With this commit we implement graph inventory refresh for cloud manager.
Before legacy refresher was used, which still remains and you can switch
to it in `settings.yml`. Both refreshers use and pass the same unit tests.

Signed-off-by: sasoc <saso.cvitkovic@gmail.com>

app/models/manageiq/providers/vmware/builder.rb

@@ -0,0 +1,30 @@
+class ManageIQ::Providers::Vmware::Builder
+  class << self
+    def build_inventory(ems, target)
+      cloud_manager_inventory(ems, target)
+    end
+
+    private
+
+    def cloud_manager_inventory(ems, target)
+      inventory(
+        ems,
+        target,
+        ManageIQ::Providers::Vmware::Inventory::Collector::CloudManager,
+        ManageIQ::Providers::Vmware::Inventory::Persister::CloudManager,
+        [ManageIQ::Providers::Vmware::Inventory::Parser::CloudManager]
+      )
+    end
+
+    def inventory(manager, raw_target, collector_class, persister_class, parsers_classes)
+      collector = collector_class.new(manager, raw_target)
+      persister = persister_class.new(manager, raw_target)
+
+      ::ManageIQ::Providers::Vmware::Inventory.new(
+        persister,
+        collector,
+        parsers_classes.map(&:new)
+      )
+    end
+  end
+end

app/models/manageiq/providers/vmware/cloud_manager.rb

@@ -15,6 +15,8 @@ class ManageIQ::Providers::Vmware::CloudManager < ManageIQ::Providers::CloudMana
   include ManageIQ::Providers::Vmware::CloudManager::ManagerEventsMixin
   include HasNetworkManagerMixin
 
+  has_many :snapshots, :through => :vms_and_templates
+
   before_create :ensure_managers
 
   def ensure_network_manager

app/models/manageiq/providers/vmware/cloud_manager/refresher.rb

@@ -1,4 +1,4 @@
-class ManageIQ::Providers::Vmware::CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
+class ManageIQ::Providers::Vmware::CloudManager::Refresher < ManageIQ::Providers::BaseManager::ManagerRefresher
   include ::EmsRefresh::Refreshers::EmsRefresherMixin
 
   def parse_legacy_inventory(ems)

app/models/manageiq/providers/vmware/cloud_manager/snapshot.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Vmware::CloudManager::Snapshot < ::Snapshot
+end

app/models/manageiq/providers/vmware/inventory.rb

@@ -0,0 +1,5 @@
+class ManageIQ::Providers::Vmware::Inventory < ManagerRefresh::Inventory
+  require_nested :Collector
+  require_nested :Parser
+  require_nested :Persister
+end

app/models/manageiq/providers/vmware/inventory/collector.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Collector < ManagerRefresh::Inventory::Collector
+  require_nested :CloudManager
+
+  def initialize(_manager, _target)
+    super
+
+    initialize_inventory_sources
+  end
+
+  def initialize_inventory_sources
+    @orgs           = []
+    @vdcs           = []
+    @vapps          = []
+    @vms            = []
+    @vapp_templates = []
+    @images         = []
+  end
+
+  def connection
+    @connection ||= manager.connect
+  end
+
+  def public_images?
+    options.try(:get_public_images)
+  end
+end

app/models/manageiq/providers/vmware/inventory/collector/cloud_manager.rb

@@ -0,0 +1,64 @@
+class ManageIQ::Providers::Vmware::Inventory::Collector::CloudManager < ManageIQ::Providers::Vmware::Inventory::Collector
+  VAPP_TEMPLATE_STATUS_READY = "8".freeze
+
+  def orgs
+    return @orgs if @orgs.any?
+    @orgs = connection.organizations
+  end
+
+  def vdcs
+    return @vdcs if @vdcs.any?
+    @vdcs = orgs.each_with_object([]) do |org, res|
+      res.concat(org.vdcs.all)
+    end
+  end
+
+  def vapps
+    return @vapps if @vapps.any?
+    @vapps = vdcs.each_with_object([]) do |vdc, res|
+      res.concat(vdc.vapps.all)
+    end
+  end
+
+  def vms
+    return @vms if @vms.any?
+    @vms = vapps.each_with_object([]) do |vapp, res|
+      # Remove this each loop, once fog api will be updated to send hostname and snapshot together with vms
+      vapp.vms.each do |vm|
+        res << {
+          :vm       => vm,
+          :hostname => vm.customization.try(:computer_name),
+          :snapshot => connection.get_snapshot_section(vm.id).try(:data)
+        }
+      end
+    end
+  end
+
+  def vapp_templates
+    return @vapp_templates if @vapp_templates.any?
+    @vapp_templates = orgs.each_with_object([]) do |org, res|
+      org.catalogs.each do |catalog|
+        next if catalog.is_published && !public_images?
+
+        catalog.catalog_items.each do |item|
+          # Skip all Catalog Items which are not vApp Templates (e.g. Media & Other)
+          next unless item.vapp_template_id.starts_with?('vappTemplate-')
+          next unless (t = item.vapp_template) && t.status == VAPP_TEMPLATE_STATUS_READY
+
+          res << {
+            :vapp_template => t,
+            :is_published  => catalog.is_published,
+            :content       => connection.get_vapp_template_ovf_descriptor(t.id).body
+          }
+        end
+      end
+    end
+  end
+
+  def images
+    return @images if @images.any?
+    @images = vapp_templates.each_with_object([]) do |template_obj, res|
+      res.concat(template_obj[:vapp_template].vms.map { |image| { :image => image, :is_published => template_obj[:is_published] } } )
+    end
+  end
+end

app/models/manageiq/providers/vmware/inventory/parser.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Parser < ManagerRefresh::Inventory::Parser
+  require_nested :CloudManager
+
+  # See https://pubs.vmware.com/vcd-80/index.jsp#com.vmware.vcloud.api.sp.doc_90/GUID-E1BA999D-87FA-4E2C-B638-24A211AB8160.html
+  def controller_description(bus_subtype)
+    case bus_subtype
+    when 'buslogic'
+      'BusLogic Parallel SCSI controller'
+    when 'lsilogic'
+      'LSI Logic Parallel SCSI controller'
+    when 'lsilogicsas'
+      'LSI Logic SAS SCSI controller'
+    when 'VirtualSCSI'
+      'Paravirtual SCSI controller'
+    when 'vmware.sata.ahci'
+      'SATA controller'
+    else
+      'IDE controller'
+    end
+  end
+
+  # See https://pubs.vmware.com/vcd-80/index.jsp#com.vmware.vcloud.api.sp.doc_90/GUID-E1BA999D-87FA-4E2C-B638-24A211AB8160.html
+  def hdd?(bus_type)
+    [5, 6, 20].include?(bus_type)
+  end
+end

app/models/manageiq/providers/vmware/inventory/parser/cloud_manager.rb

@@ -0,0 +1,108 @@
+class ManageIQ::Providers::Vmware::Inventory::Parser::CloudManager < ManageIQ::Providers::Vmware::Inventory::Parser
+  def parse
+    vdcs
+    vapps
+    vms
+    vapp_templates
+    images
+  end
+
+  private
+
+  def vdcs
+    collector.vdcs.each do |vdc|
+      persister.availability_zones.find_or_build(vdc.id).assign_attributes(
+        :name => vdc.name
+      )
+    end
+  end
+
+  def vapps
+    collector.vapps.each do |vapp|
+      persister.orchestration_stacks.find_or_build(vapp.id).assign_attributes(
+        :name        => vapp.name,
+        :description => vapp.name,
+        :status      => vapp.human_status,
+      )
+    end
+  end
+
+  def vms
+    collector.vms.each do |vm|
+      parsed_vm = persister.vms.find_or_build(vm[:vm].id).assign_attributes(
+        :uid_ems             => vm[:vm].id,
+        :name                => vm[:vm].name,
+        :hostname            => vm[:hostname],
+        :vendor              => 'vmware',
+        :raw_power_state     => vm[:vm].status,
+        :orchestration_stack => persister.orchestration_stacks.lazy_find(vm[:vm].vapp_id),
+        :snapshots           => []
+      )
+
+      if (resp = vm[:snapshot]) && (snapshot = resp.fetch_path(:body, :Snapshot))
+        parsed_vm.snapshots = [
+          persister.snapshots.find_or_build(parsed_vm.ems_ref).assign_attributes(
+            :name        => "#{vm[:vm].name} (snapshot)",
+            :uid         => "#{vm[:vm].id}_#{snapshot[:created]}",
+            :ems_ref     => "#{vm[:vm].id}_#{snapshot[:created]}",
+            :parent_uid  => vm[:vm].id,
+            :create_time => DateTime.parse(snapshot[:created]),
+            :total_size  => snapshot[:size]
+          )
+        ]
+      end
+
+      hardware = persister.hardwares.find_or_build(parsed_vm).assign_attributes(
+        :guest_os             => vm[:vm].operating_system,
+        :guest_os_full_name   => vm[:vm].operating_system,
+        :bitness              => vm[:vm].operating_system =~ /64-bit/ ? 64 : 32,
+        :cpu_sockets          => vm[:vm].cpu,
+        :cpu_cores_per_socket => 1,
+        :cpu_total_cores      => vm[:vm].cpu,
+        :memory_mb            => vm[:vm].memory,
+        :disk_capacity        => vm[:vm].hard_disks.inject(0) { |sum, x| sum + x.values[0] } * 1.megabyte,
+      )
+
+      vm[:vm].disks.all.select { |d| hdd? d.bus_type }.map do |disk|
+        persister.disks.find_or_build_by(:hardware => hardware, :device_name => disk.name).assign_attributes(
+          :device_type     => 'disk',
+          :controller_type => controller_description(disk.bus_sub_type),
+          :size            => disk.capacity * 1.megabyte,
+          :location        => "#{vm[:vm].id}-#{disk.id}",
+          :filename        => "#{vm[:vm].id}-#{disk.id}",
+        )
+      end
+
+      persister.operating_systems.find_or_build(parsed_vm).assign_attributes(
+        :product_name => vm[:vm].operating_system,
+      )
+    end
+  end
+
+  def vapp_templates
+    collector.vapp_templates.each do |vapp_template|
+      persister.orchestration_templates.find_or_build(vapp_template[:vapp_template].id).assign_attributes(
+        :name        => vapp_template[:vapp_template].name,
+        :description => vapp_template[:vapp_template].description,
+        :orderable   => true,
+        :content     => "<!-- #{vapp_template[:vapp_template].id} -->\n#{vapp_template[:content]}",
+        # By default #save_orchestration_templates_inventory does not set the EMS
+        # ID because templates are not EMS specific. We are setting the EMS
+        # explicitly here, because vapps are specific to concrete EMS.
+        :ems_id      => collector.manager.id
+      )
+    end
+  end
+
+  def images
+    collector.images.each do |image|
+      persister.miq_templates.find_or_build(image[:image].id).assign_attributes(
+        :uid_ems            => image[:image].id,
+        :name               => image[:image].name,
+        :vendor             => 'vmware',
+        :raw_power_state    => 'never',
+        :publicly_available => image[:is_published]
+      )
+    end
+  end
+end

app/models/manageiq/providers/vmware/inventory/persister.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Persister < ManagerRefresh::Inventory::Persister
+  require_nested :CloudManager
+
+  protected
+
+  def cloud
+    ManageIQ::Providers::Vmware::InventoryCollectionDefault::CloudManager
+  end
+
+  def targeted
+    false
+  end
+
+  def strategy
+    nil
+  end
+
+  def shared_options
+    settings_options = options[:inventory_collections].try(:to_hash) || {}
+
+    settings_options.merge(
+      :strategy => strategy,
+      :targeted => targeted,
+    )
+  end
+end

app/models/manageiq/providers/vmware/inventory/persister/cloud_manager.rb

@@ -0,0 +1,18 @@
+class ManageIQ::Providers::Vmware::Inventory::Persister::CloudManager < ManageIQ::Providers::Vmware::Inventory::Persister
+  def initialize_inventory_collections
+    add_inventory_collections(
+      cloud,
+      %i(
+        availability_zones
+        orchestration_stacks
+        vms
+        snapshots
+        hardwares
+        disks
+        operating_systems
+        orchestration_templates
+        miq_templates
+      )
+    )
+  end
+end