Merge pull request #217 from sasoc/graph-refresh-for-cloud-manager

Implement graph inventory refresh for cloud manager
(cherry picked from commit b14b2f3)

https://bugzilla.redhat.com/show_bug.cgi?id=1584727

app/models/manageiq/providers/vmware/builder.rb

@@ -0,0 +1,30 @@
+class ManageIQ::Providers::Vmware::Builder
+  class << self
+    def build_inventory(ems, target)
+      cloud_manager_inventory(ems, target)
+    end
+
+    private
+
+    def cloud_manager_inventory(ems, target)
+      inventory(
+        ems,
+        target,
+        ManageIQ::Providers::Vmware::Inventory::Collector::CloudManager,
+        ManageIQ::Providers::Vmware::Inventory::Persister::CloudManager,
+        [ManageIQ::Providers::Vmware::Inventory::Parser::CloudManager]
+      )
+    end
+
+    def inventory(manager, raw_target, collector_class, persister_class, parsers_classes)
+      collector = collector_class.new(manager, raw_target)
+      persister = persister_class.new(manager, raw_target)
+
+      ::ManageIQ::Providers::Vmware::Inventory.new(
+        persister,
+        collector,
+        parsers_classes.map(&:new)
+      )
+    end
+  end
+end

app/models/manageiq/providers/vmware/cloud_manager.rb

@@ -15,6 +15,8 @@ class ManageIQ::Providers::Vmware::CloudManager < ManageIQ::Providers::CloudMana
   include ManageIQ::Providers::Vmware::CloudManager::ManagerEventsMixin
   include HasNetworkManagerMixin
 
+  has_many :snapshots, :through => :vms_and_templates
+
   before_create :ensure_managers
 
   def ensure_network_manager

app/models/manageiq/providers/vmware/cloud_manager/refresher.rb

@@ -1,4 +1,4 @@
-class ManageIQ::Providers::Vmware::CloudManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
+class ManageIQ::Providers::Vmware::CloudManager::Refresher < ManageIQ::Providers::BaseManager::ManagerRefresher
   include ::EmsRefresh::Refreshers::EmsRefresherMixin
 
   def parse_legacy_inventory(ems)

app/models/manageiq/providers/vmware/cloud_manager/snapshot.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Vmware::CloudManager::Snapshot < ::Snapshot
+end

app/models/manageiq/providers/vmware/inventory.rb

@@ -0,0 +1,5 @@
+class ManageIQ::Providers::Vmware::Inventory < ManagerRefresh::Inventory
+  require_nested :Collector
+  require_nested :Parser
+  require_nested :Persister
+end

app/models/manageiq/providers/vmware/inventory/collector.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Collector < ManagerRefresh::Inventory::Collector
+  require_nested :CloudManager
+
+  def initialize(_manager, _target)
+    super
+
+    initialize_inventory_sources
+  end
+
+  def initialize_inventory_sources
+    @orgs           = []
+    @vdcs           = []
+    @vapps          = []
+    @vms            = []
+    @vapp_templates = []
+    @images         = []
+  end
+
+  def connection
+    @connection ||= manager.connect
+  end
+
+  def public_images?
+    options.try(:get_public_images)
+  end
+end

app/models/manageiq/providers/vmware/inventory/collector/cloud_manager.rb

@@ -0,0 +1,64 @@
+class ManageIQ::Providers::Vmware::Inventory::Collector::CloudManager < ManageIQ::Providers::Vmware::Inventory::Collector
+  VAPP_TEMPLATE_STATUS_READY = "8".freeze
+
+  def orgs
+    return @orgs if @orgs.any?
+    @orgs = connection.organizations
+  end
+
+  def vdcs
+    return @vdcs if @vdcs.any?
+    @vdcs = orgs.each_with_object([]) do |org, res|
+      res.concat(org.vdcs.all)
+    end
+  end
+
+  def vapps
+    return @vapps if @vapps.any?
+    @vapps = vdcs.each_with_object([]) do |vdc, res|
+      res.concat(vdc.vapps.all)
+    end
+  end
+
+  def vms
+    return @vms if @vms.any?
+    @vms = vapps.each_with_object([]) do |vapp, res|
+      # Remove this each loop, once fog api will be updated to send hostname and snapshot together with vms
+      vapp.vms.each do |vm|
+        res << {
+          :vm       => vm,
+          :hostname => vm.customization.try(:computer_name),
+          :snapshot => connection.get_snapshot_section(vm.id).try(:data)
+        }
+      end
+    end
+  end
+
+  def vapp_templates
+    return @vapp_templates if @vapp_templates.any?
+    @vapp_templates = orgs.each_with_object([]) do |org, res|
+      org.catalogs.each do |catalog|
+        next if !public_images? && catalog.is_published
+
+        catalog.catalog_items.each do |item|
+          # Skip all Catalog Items which are not vApp Templates (e.g. Media & Other)
+          next unless item.vapp_template_id.starts_with?('vappTemplate-')
+          next unless (t = item.vapp_template) && t.status == VAPP_TEMPLATE_STATUS_READY
+
+          res << {
+            :vapp_template => t,
+            :is_published  => catalog.is_published,
+            :content       => connection.get_vapp_template_ovf_descriptor(t.id).try(:body)
+          }
+        end
+      end
+    end
+  end
+
+  def images
+    return @images if @images.any?
+    @images = vapp_templates.each_with_object([]) do |template_obj, res|
+      res.concat(template_obj[:vapp_template].vms.map { |image| { :image => image, :is_published => template_obj[:is_published] } })
+    end
+  end
+end

app/models/manageiq/providers/vmware/inventory/parser.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Parser < ManagerRefresh::Inventory::Parser
+  require_nested :CloudManager
+
+  # See https://pubs.vmware.com/vcd-80/index.jsp#com.vmware.vcloud.api.sp.doc_90/GUID-E1BA999D-87FA-4E2C-B638-24A211AB8160.html
+  def controller_description(bus_subtype)
+    case bus_subtype
+    when 'buslogic'
+      'BusLogic Parallel SCSI controller'
+    when 'lsilogic'
+      'LSI Logic Parallel SCSI controller'
+    when 'lsilogicsas'
+      'LSI Logic SAS SCSI controller'
+    when 'VirtualSCSI'
+      'Paravirtual SCSI controller'
+    when 'vmware.sata.ahci'
+      'SATA controller'
+    else
+      'IDE controller'
+    end
+  end
+
+  # See https://pubs.vmware.com/vcd-80/index.jsp#com.vmware.vcloud.api.sp.doc_90/GUID-E1BA999D-87FA-4E2C-B638-24A211AB8160.html
+  def hdd?(bus_type)
+    [5, 6, 20].include?(bus_type)
+  end
+end

app/models/manageiq/providers/vmware/inventory/parser/cloud_manager.rb

@@ -0,0 +1,102 @@
+class ManageIQ::Providers::Vmware::Inventory::Parser::CloudManager < ManageIQ::Providers::Vmware::Inventory::Parser
+  def parse
+    vdcs
+    vapps
+    vms
+    vapp_templates
+    images
+  end
+
+  private
+
+  def vdcs
+    collector.vdcs.each do |vdc|
+      persister.availability_zones.find_or_build(vdc.id).assign_attributes(
+        :name => vdc.name
+      )
+    end
+  end
+
+  def vapps
+    collector.vapps.each do |vapp|
+      persister.orchestration_stacks.find_or_build(vapp.id).assign_attributes(
+        :name        => vapp.name,
+        :description => vapp.name,
+        :status      => vapp.human_status,
+      )
+    end
+  end
+
+  def vms
+    collector.vms.each do |vm|
+      parsed_vm = persister.vms.find_or_build(vm[:vm].id).assign_attributes(
+        :uid_ems             => vm[:vm].id,
+        :name                => vm[:vm].name,
+        :hostname            => vm[:hostname],
+        :vendor              => 'vmware',
+        :raw_power_state     => vm[:vm].status,
+        :orchestration_stack => persister.orchestration_stacks.lazy_find(vm[:vm].vapp_id),
+        :snapshots           => []
+      )
+
+      if (resp = vm[:snapshot]) && (snapshot = resp.fetch_path(:body, :Snapshot))
+        uid = "#{vm[:vm].id}_#{snapshot[:created]}"
+        persister.snapshots.find_or_build_by(:vm_or_template => parsed_vm, :ems_ref => uid).assign_attributes(
+          :name        => "#{vm[:vm].name} (snapshot)",
+          :uid         => uid,
+          :parent_uid  => vm[:vm].id,
+          :create_time => Time.zone.parse(snapshot[:created]),
+          :total_size  => snapshot[:size]
+        )
+      end
+
+      hardware = persister.hardwares.find_or_build(parsed_vm).assign_attributes(
+        :guest_os             => vm[:vm].operating_system,
+        :guest_os_full_name   => vm[:vm].operating_system,
+        :bitness              => vm[:vm].operating_system =~ /64-bit/ ? 64 : 32,
+        :cpu_sockets          => vm[:vm].cpu,
+        :cpu_cores_per_socket => 1,
+        :cpu_total_cores      => vm[:vm].cpu,
+        :memory_mb            => vm[:vm].memory,
+        :disk_capacity        => vm[:vm].hard_disks.inject(0) { |sum, x| sum + x.values[0] } * 1.megabyte,
+      )
+
+      vm[:vm].disks.all.select { |d| hdd? d.bus_type }.map do |disk|
+        persister.disks.find_or_build_by(:hardware => hardware, :device_name => disk.name).assign_attributes(
+          :device_type     => 'disk',
+          :controller_type => controller_description(disk.bus_sub_type),
+          :size            => disk.capacity * 1.megabyte,
+          :location        => "#{vm[:vm].id}-#{disk.id}",
+          :filename        => "#{vm[:vm].id}-#{disk.id}",
+        )
+      end
+
+      persister.operating_systems.find_or_build(parsed_vm).assign_attributes(
+        :product_name => vm[:vm].operating_system,
+      )
+    end
+  end
+
+  def vapp_templates
+    collector.vapp_templates.each do |vapp_template|
+      persister.orchestration_templates.find_or_build(vapp_template[:vapp_template].id).assign_attributes(
+        :name        => vapp_template[:vapp_template].name,
+        :description => vapp_template[:vapp_template].description,
+        :orderable   => true,
+        :content     => "<!-- #{vapp_template[:vapp_template].id} -->\n#{vapp_template[:content]}",
+      )
+    end
+  end
+
+  def images
+    collector.images.each do |image|
+      persister.miq_templates.find_or_build(image[:image].id).assign_attributes(
+        :uid_ems            => image[:image].id,
+        :name               => image[:image].name,
+        :vendor             => 'vmware',
+        :raw_power_state    => 'never',
+        :publicly_available => image[:is_published]
+      )
+    end
+  end
+end

app/models/manageiq/providers/vmware/inventory/persister.rb

@@ -0,0 +1,26 @@
+class ManageIQ::Providers::Vmware::Inventory::Persister < ManagerRefresh::Inventory::Persister
+  require_nested :CloudManager
+
+  protected
+
+  def cloud
+    ManageIQ::Providers::Vmware::InventoryCollectionDefault::CloudManager
+  end
+
+  def targeted
+    false
+  end
+
+  def strategy
+    nil
+  end
+
+  def shared_options
+    settings_options = options[:inventory_collections].try(:to_hash) || {}
+
+    settings_options.merge(
+      :strategy => strategy,
+      :targeted => targeted,
+    )
+  end
+end

app/models/manageiq/providers/vmware/inventory/persister/cloud_manager.rb

@@ -0,0 +1,18 @@
+class ManageIQ::Providers::Vmware::Inventory::Persister::CloudManager < ManageIQ::Providers::Vmware::Inventory::Persister
+  def initialize_inventory_collections
+    add_inventory_collections(
+      cloud,
+      %i(
+        availability_zones
+        orchestration_stacks
+        vms
+        snapshots
+        hardwares
+        disks
+        operating_systems
+        orchestration_templates
+        miq_templates
+      )
+    )
+  end
+end