Separate collector from inventory and targeted refresh

app/models/manageiq/providers/amazon/cloud_manager/refresh_parser_inventory_object.rb

@@ -30,27 +30,27 @@ def populate_inventory_collections
   private
 
   def get_flavors
-    process_inventory_collection(inventory.flavors, :flavors) { |flavor| parse_flavor(flavor) }
+    process_inventory_collection(inventory.collector.flavors, :flavors) { |flavor| parse_flavor(flavor) }
   end
 
   def get_availability_zones
-    process_inventory_collection(inventory.availability_zones, :availability_zones) { |az| parse_availability_zone(az) }
+    process_inventory_collection(inventory.collector.availability_zones, :availability_zones) { |az| parse_availability_zone(az) }
   end
 
   def get_key_pairs
-    process_inventory_collection(inventory.key_pairs, :key_pairs) { |kp| parse_key_pair(kp) }
+    process_inventory_collection(inventory.collector.key_pairs, :key_pairs) { |kp| parse_key_pair(kp) }
   end
 
   def get_private_images
-    get_images(inventory.private_images)
+    get_images(inventory.collector.private_images)
   end
 
   def get_shared_images
-    get_images(inventory.shared_images)
+    get_images(inventory.collector.shared_images)
   end
 
   def get_public_images
-    get_images(inventory.public_images, true)
+    get_images(inventory.collector.public_images, true)
   end
 
   def get_images(images, is_public = false)
@@ -66,7 +66,7 @@ def get_image_hardware(image)
   end
 
   def get_stacks
-    process_inventory_collection(inventory.stacks, :orchestration_stacks) do |stack|
+    process_inventory_collection(inventory.collector.stacks, :orchestration_stacks) do |stack|
       get_stack_resources(stack)
       get_stack_outputs(stack)
       get_stack_parameters(stack)
@@ -89,7 +89,7 @@ def get_stack_outputs(stack)
   end
 
   def get_stack_resources(stack)
-    resources = inventory.stack_resources(stack['stack_name'])
+    resources = inventory.collector.stack_resources(stack['stack_name'])
 
     process_inventory_collection(resources, :orchestration_stacks_resources) do |resource|
       parse_stack_resource(resource, stack)
@@ -101,7 +101,7 @@ def get_stack_template(stack)
   end
 
   def get_instances
-    process_inventory_collection(inventory.instances, :vms) do |instance|
+    process_inventory_collection(inventory.collector.instances, :vms) do |instance|
       # TODO(lsmola) we have a non lazy dependency, can we remove that?
       flavor = inventory_collections[:flavors].find(instance['instance_type']) || inventory_collections[:flavors].find("unknown")
 
@@ -318,7 +318,7 @@ def parse_stack_template(stack)
       :ems_ref     => stack['stack_id'],
       :name        => stack['stack_name'],
       :description => stack['description'],
-      :content     => inventory.stack_template(stack['stack_name']),
+      :content     => inventory.collector.stack_template(stack['stack_name']),
       :orderable   => false
     }
   end

app/models/manageiq/providers/amazon/inventory.rb

@@ -1,102 +1,19 @@
 class ManageIQ::Providers::Amazon::Inventory
   require_nested :Factory
   require_nested :HashCollection
+  require_nested :Collectors
   require_nested :Targets
+  require_nested :TargetCollection
 
-  attr_reader :ems, :target, :inventory_collections, :options
+  attr_reader :ems, :target, :collector, :inventory_collections, :options
 
   def initialize(ems, target)
     @ems                   = ems
     @target                = target
     @options               = Settings.ems_refresh[ems.class.ems_type]
     @inventory_collections = {:_inventory_collection => true}
-
-    @known_flavors = Set.new
+    @collector             = initialize_collector
 
     initialize_inventory_collections
   end
-
-  def aws_ec2
-    @aws_ec2 ||= ems.connect
-  end
-
-  def aws_cloud_formation
-    @aws_cloud_formation ||= ems.connect(:service => :CloudFormation)
-  end
-
-  def aws_elb
-    @aws_elb ||= ems.connect(:service => :ElasticLoadBalancing)
-  end
-
-  def aws_s3
-    @aws_s3 ||= ems.connect(:service => :S3)
-  end
-
-  def instances
-    []
-  end
-
-  def flavors
-    []
-  end
-
-  def availability_zones
-    []
-  end
-
-  def key_pairs
-    []
-  end
-
-  def private_images
-    []
-  end
-
-  def shared_images
-    []
-  end
-
-  def public_images
-    []
-  end
-
-  def stacks
-    []
-  end
-
-  def stack_resources(_stack_name)
-    []
-  end
-
-  def stack_template(_stack_name)
-    []
-  end
-
-  def cloud_networks
-    []
-  end
-
-  def cloud_subnets
-    []
-  end
-
-  def security_groups
-    []
-  end
-
-  def network_ports
-    []
-  end
-
-  def load_balancers
-    []
-  end
-
-  def health_check_members(_load_balancer_name)
-    []
-  end
-
-  def floating_ips
-    []
-  end
 end

app/models/manageiq/providers/amazon/inventory/collectors.rb

@@ -0,0 +1,127 @@
+class ManageIQ::Providers::Amazon::Inventory::Collectors
+  attr_reader :ems, :target, :options
+
+  attr_reader :instances, :instances_refs, :instances_deleted
+  attr_reader :flavors, :flavors_refs, :flavors_deleted
+  attr_reader :availability_zones, :availability_zones_refs, :availability_zones_deleted
+  attr_reader :key_pairs, :key_pairs_refs, :key_pairs_deleted
+  attr_reader :private_images, :private_images_refs, :private_images_deleted
+  attr_reader :shared_images, :shared_images_refs, :shared_images_deleted
+  attr_reader :public_images, :public_images_refs, :public_images_deleted
+  attr_reader :cloud_networks, :cloud_networks_refs, :cloud_networks_deleted
+  attr_reader :cloud_subnets, :cloud_subnets_refs, :cloud_subnets_deleted
+  attr_reader :security_groups, :security_groups_refs, :security_groups_deleted
+  attr_reader :floating_ips, :floating_ips_refs, :floating_ips_deleted
+  attr_reader :network_ports, :network_ports_refs, :network_ports_deleted
+  attr_reader :load_balancers, :load_balancers_refs, :load_balancers_deleted
+  attr_reader :stacks, :stacks_refs, :stacks_deleted
+  attr_reader :cloud_volumes, :cloud_volumes_refs
+  attr_reader :cloud_volume_snapshots, :cloud_volume_snapshots_refs
+
+
+  def initialize(ems, target)
+    @ems     = ems
+    @target  = target
+    @options = Settings.ems_refresh[ems.class.ems_type]
+
+    initialize_inventory_sources
+  end
+
+  def initialize_inventory_sources
+    @instances                   = []
+    @instances_refs              = Set.new
+    @instances_deleted           = []
+    @flavors                     = []
+    @flavors_refs                = Set.new
+    @flavors_deleted             = []
+    @availability_zones          = []
+    @availability_zones_refs     = Set.new
+    @availability_zones_deleted  = []
+    @key_pairs                   = []
+    @key_pairs_refs              = Set.new
+    @key_pairs_deleted           = []
+    @private_images              = []
+    @private_images_refs         = Set.new
+    @private_images_deleted      = []
+    @shared_images               = []
+    @shared_images_refs          = Set.new
+    @shared_images_deleted       = []
+    @public_images               = []
+    @public_images_refs          = Set.new
+    @public_images_deleted       = []
+    @cloud_networks              = []
+    @cloud_networks_refs         = Set.new
+    @cloud_networks_deleted      = []
+    @cloud_subnets               = []
+    @cloud_subnets_refs          = Set.new
+    @cloud_subnets_deleted       = []
+    @security_groups             = []
+    @security_groups_refs        = Set.new
+    @security_groups_deleted     = []
+    @floating_ips                = []
+    @floating_ips_refs           = Set.new
+    @floating_ips_deleted        = []
+    @network_ports               = []
+    @network_ports_refs          = Set.new
+    @network_ports_deleted       = []
+    @load_balancers              = []
+    @load_balancers_refs         = Set.new
+    @stacks                      = []
+    @stacks_refs                 = Set.new
+    @cloud_volumes               = []
+    @cloud_volumes_refs          = Set.new
+    @cloud_volume_snapshots      = []
+    @cloud_volume_snapshots_refs = Set.new
+    # Nested resources
+    @stack_resources             = {}
+    @stack_resources_refs        = {}
+    @stack_template              = {}
+    @stack_template_refs         = {}
+    @health_check_members        = {}
+    @health_check_members_refs   = {}
+  end
+
+  def hash_collection
+    ::ManageIQ::Providers::Amazon::Inventory::HashCollection
+  end
+
+  def aws_ec2
+    @aws_ec2 ||= ems.connect
+  end
+
+  def aws_cloud_formation
+    @aws_cloud_formation ||= ems.connect(:service => :CloudFormation)
+  end
+
+  def aws_elb
+    @aws_elb ||= ems.connect(:service => :ElasticLoadBalancing)
+  end
+
+  def aws_s3
+    @aws_s3 ||= ems.connect(:service => :S3)
+  end
+
+  def stack_resources(stack_name)
+    @stack_resources.try(:[], stack_name) || []
+  end
+
+  def stack_resources_refs(stack_name)
+    @stack_resources_refs.try(:[], stack_name) || []
+  end
+
+  def stack_template(stack_name)
+    @stack_template.try(:[], stack_name) || []
+  end
+
+  def stack_template_refs(stack_name)
+    @stack_template_refs.try(:[], stack_name) || []
+  end
+
+  def health_check_members(load_balancer_name)
+    @health_check_members.try(:[], load_balancer_name) || []
+  end
+
+  def health_check_members_refs(load_balancer_name)
+    @health_check_members_refs.try(:[], load_balancer_name) || []
+  end
+end

app/models/manageiq/providers/amazon/inventory/collectors/cloud_manager.rb

@@ -0,0 +1,49 @@
+class ManageIQ::Providers::Amazon::Inventory::Collectors::CloudManager < ManageIQ::Providers::Amazon::Inventory::Collectors
+  def instances
+    hash_collection.new(aws_ec2.instances)
+  end
+
+  def flavors
+    ManageIQ::Providers::Amazon::InstanceTypes.all
+  end
+
+  def availability_zones
+    hash_collection.new(aws_ec2.client.describe_availability_zones[:availability_zones])
+  end
+
+  def key_pairs
+    hash_collection.new(aws_ec2.client.describe_key_pairs[:key_pairs])
+  end
+
+  def private_images
+    hash_collection.new(aws_ec2.client.describe_images(:owners  => [:self],
+                                                      :filters => [{:name   => "image-type",
+                                                                    :values => ["machine"]}])[:images])
+  end
+
+  def shared_images
+    hash_collection.new(aws_ec2.client.describe_images(:executable_users => [:self],
+                                                      :filters          => [{:name   => "image-type",
+                                                                             :values => ["machine"]}])[:images])
+  end
+
+  def public_images
+    filters = options.public_images_filters
+    hash_collection.new(aws_ec2.client.describe_images(:executable_users => [:all],
+                                                       :filters          => filters)[:images])
+  end
+
+  def stacks
+    hash_collection.new(aws_cloud_formation.client.describe_stacks[:stacks])
+  end
+
+  def stack_resources(stack_name)
+    stack_resources = aws_cloud_formation.client.list_stack_resources(:stack_name => stack_name).try(:stack_resource_summaries)
+
+    hash_collection.new(stack_resources || [])
+  end
+
+  def stack_template(stack_name)
+    aws_cloud_formation.client.get_template(:stack_name => stack_name).template_body
+  end
+end

app/models/manageiq/providers/amazon/inventory/collectors/network_manager.rb

@@ -0,0 +1,34 @@
+class ManageIQ::Providers::Amazon::Inventory::Collectors::NetworkManager < ManageIQ::Providers::Amazon::Inventory::Collectors
+  def cloud_networks
+    hash_collection.new(aws_ec2.client.describe_vpcs[:vpcs])
+  end
+
+  def cloud_subnets
+    hash_collection.new(aws_ec2.client.describe_subnets[:subnets])
+  end
+
+  def security_groups
+    hash_collection.new(aws_ec2.security_groups)
+  end
+
+  def network_ports
+    hash_collection.new(aws_ec2.client.describe_network_interfaces.network_interfaces)
+  end
+
+  def load_balancers
+    hash_collection.new(aws_elb.client.describe_load_balancers.load_balancer_descriptions)
+  end
+
+  def health_check_members(load_balancer_name)
+    hash_collection.new(aws_elb.client.describe_instance_health(
+      :load_balancer_name => load_balancer_name).instance_states)
+  end
+
+  def floating_ips
+    hash_collection.new(aws_ec2.client.describe_addresses.addresses)
+  end
+
+  def instances
+    hash_collection.new(aws_ec2.instances)
+  end
+end

app/models/manageiq/providers/amazon/inventory/collectors/storage_manager/ebs.rb

@@ -0,0 +1,9 @@
+class ManageIQ::Providers::Amazon::Inventory::Collectors::StorageManager::Ebs < ManageIQ::Providers::Amazon::Inventory::Collectors
+  def cloud_volumes
+    hash_collection.new(aws_ec2.client.describe_volumes[:volumes])
+  end
+
+  def cloud_volume_snapshots
+    hash_collection.new(aws_ec2.client.describe_snapshots(:owner_ids => [:self])[:snapshots])
+  end
+end