Merge pull request #101 from gberginc/introduce_ebs_manager

Introduce Amazon Block Storage  Manager (EBS)

app/models/manageiq/providers/amazon/block_storage_manager.rb

@@ -0,0 +1,37 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager < ManageIQ::Providers::StorageManager
+ require_nested :RefreshParser
+ require_nested :RefreshWorker
+ require_nested :Refresher
+
+ include ManageIQ::Providers::Amazon::ManagerMixin
+ include ManageIQ::Providers::StorageManager::BlockMixin
+
+ delegate :availability_zones,
+ :authentication_check,
+ :authentication_status,
+ :authentications,
+ :authentication_for_summary,
+ :zone,
+ :connect,
+ :verify_credentials,
+ :with_provider_connection,
+ :address,
+ :ip_address,
+ :hostname,
+ :default_endpoint,
+ :endpoints,
+ :to => :parent_manager,
+ :allow_nil => true
+
+ def self.ems_type
+ @ems_type ||= "ec2_block_storage".freeze
+ end
+
+ def self.description
+ @description ||= "Amazon EBS".freeze
+ end
+
+ def self.hostname_required?
+ false
+ end
+end

app/models/manageiq/providers/amazon/block_storage_manager/cloud_volume.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolume < ::CloudVolume
+end

app/models/manageiq/providers/amazon/block_storage_manager/cloud_volume_snapshot.rb

@@ -0,0 +1,2 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolumeSnapshot < ::CloudVolumeSnapshot
+end

app/models/manageiq/providers/amazon/block_storage_manager/refresh_parser.rb

@@ -0,0 +1,76 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager::RefreshParser
+ include ManageIQ::Providers::Amazon::RefreshHelperMethods
+
+ def initialize(ems, options = nil)
+ @ems = ems
+ @aws_ec2 = ems.connect
+ @data = {}
+ @data_index = {}
+ @options = options || {}
+ end
+
+ def ems_inv_to_hashes
+ log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{@ems.name}] id: [#{@ems.id}]"
+
+ $aws_log.info("#{log_header}...")
+ get_volumes
+ get_snapshots
+
+ $aws_log.info("#{log_header}...Complete")
+
+ @data
+ end
+
+ def get_volumes
+ volumes = @aws_ec2.client.describe_volumes[:volumes]
+ process_collection(volumes, :cloud_volumes) { |volume| parse_volume(volume) }
+ end
+
+ def get_snapshots
+ snapshots = @aws_ec2.client.describe_snapshots(:owner_ids => [:self])[:snapshots]
+ process_collection(snapshots, :cloud_volume_snapshots) { |snap| parse_snapshot(snap) }
+ end
+
+ def parse_volume(volume)
+ uid = volume.volume_id
+
+ new_result = {
+ :type => self.class.volume_type,
+ :ems_ref => uid,
+ :name => uid,
+ :status => volume.state,
+ :creation_time => volume.create_time,
+ :volume_type => volume.volume_type,
+ :size => volume.size.to_i.gigabytes
+ }
+
+ return uid, new_result
+ end
+
+ def parse_snapshot(snap)
+ uid = snap.snapshot_id
+
+ new_result = {
+ :ems_ref => uid,
+ :type => self.class.volume_snapshot_type,
+ :name => snap.snapshot_id,
+ :status => snap.state,
+ :creation_time => snap.start_time,
+ :description => snap.description,
+ :size => snap.volume_size,
+ :volume => @data_index.fetch_path(:cloud_volumes, snap.volume_id)
+ }
+
+ return uid, new_result
+ end
+
+ class << self
+ def volume_type
+ "ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolume"
+ end
+
+ def volume_snapshot_type
+ "ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolumeSnapshot"
+ end
+ end
+end

app/models/manageiq/providers/amazon/block_storage_manager/refresh_parser_inventory_object.rb

@@ -0,0 +1,63 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager::RefreshParserInventoryObject < ::ManagerRefresh::RefreshParserInventoryObject
+ include ManageIQ::Providers::Amazon::RefreshHelperMethods
+
+ def populate_inventory_collections
+ log_header = "MIQ(#{self.class.name}.#{__method__}) Collecting data for EMS name: [#{inventory.ems.name}] id: [#{inventory.ems.id}]"
+
+ $aws_log.info("#{log_header}...}")
+ get_volumes
+ get_snapshots
+ $aws_log.info("#{log_header}...Complete")
+
+ inventory_collections
+ end
+
+ private
+
+ def get_volumes
+ process_inventory_collection(inventory.cloud_volumes, :cloud_volumes) { |volume| parse_volume(volume) }
+ end
+
+ def get_snapshots
+ process_inventory_collection(inventory.cloud_volume_snapshots, :cloud_volume_snapshots) { |snap| parse_snapshot(snap) }
+ end
+
+ def parse_volume(volume)
+ uid = volume['volume_id']
+
+ {
+ :type => self.class.volume_type,
+ :ems_ref => uid,
+ :name => uid,
+ :status => volume['state'],
+ :creation_time => volume['create_time'],
+ :volume_type => volume['volume_type'],
+ :size => volume['size'].to_i.gigabytes
+ }
+ end
+
+ def parse_snapshot(snap)
+ uid = snap['snapshot_id']
+
+ {
+ :ems_ref => uid,
+ :type => self.class.volume_snapshot_type,
+ :name => snap['snapshot_id'],
+ :status => snap['state'],
+ :creation_time => snap['start_time'],
+ :description => snap['description'],
+ :size => snap['volume_size'],
+ :cloud_volume => inventory_collections[:cloud_volumes].lazy_find(snap['volume_id'])
+ }
+ end
+
+ class << self
+ def volume_type
+ "ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolume"
+ end
+
+ def volume_snapshot_type
+ "ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolumeSnapshot"
+ end
+ end
+end

app/models/manageiq/providers/amazon/block_storage_manager/refresh_worker.rb

@@ -0,0 +1,11 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager::RefreshWorker < ::MiqEmsRefreshWorker
+ require_nested :Runner
+
+ def self.ems_class
+ ManageIQ::Providers::Amazon::BlockStorageManager
+ end
+
+ def self.settings_name
+ :ems_refresh_worker_amazon_block_storage
+ end
+end

app/models/manageiq/providers/amazon/block_storage_manager/refresh_worker/runner.rb

@@ -0,0 +1,3 @@
+class ManageIQ::Providers::Amazon::BlockStorageManager::RefreshWorker::Runner <
+ ManageIQ::Providers::BaseManager::RefreshWorker::Runner
+end

app/models/manageiq/providers/amazon/block_storage_manager/refresher.rb

@@ -0,0 +1,43 @@
+module ManageIQ::Providers
+ class Amazon::BlockStorageManager::Refresher < ManageIQ::Providers::BaseManager::Refresher
+ include ::EmsRefresh::Refreshers::EmsRefresherMixin
+
+ def collect_inventory_for_targets(ems, targets)
+ targets_with_data = targets.collect do |target|
+ target_name = target.try(:name) || target.try(:event_type)
+
+ _log.info "Filtering inventory for #{target.class} [#{target_name}] id: [#{target.id}]..."
+
+ inventory = if refresher_options.try(:[], :inventory_object_refresh)
+ ManageIQ::Providers::Amazon::Inventory::Factory.inventory(ems, target)
+ else
+ nil
+ end
+
+ _log.info "Filtering inventory...Complete"
+ [target, inventory]
+ end
+
+ targets_with_data
+ end
+
+ def parse_targeted_inventory(ems, _target, inventory)
+ log_header = format_ems_for_logging(ems)
+ _log.debug "#{log_header} Parsing inventory..."
+ hashes, = Benchmark.realtime_block(:parse_inventory) do
+ if refresher_options.try(:[], :inventory_object_refresh)
+ ManageIQ::Providers::Amazon::BlockStorageManager::RefreshParserInventoryObject.new(inventory).populate_inventory_collections
+ else
+ ManageIQ::Providers::Amazon::BlockStorageManager::RefreshParser.ems_inv_to_hashes(ems, refresher_options)
+ end
+ end
+ _log.debug "#{log_header} Parsing inventory...Complete"
+
+ hashes
+ end
+
+ def post_process_refresh_classes
+ []
+ end
+ end
+end

app/models/manageiq/providers/amazon/cloud_manager.rb

@@ -29,6 +29,12 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana
  :autosave => true,
  :dependent => :destroy
 
+ has_many :storage_managers,
+ :foreign_key => :parent_ems_id,
+ :class_name => "ManageIQ::Providers::StorageManager",
+ :autosave => true,
+ :dependent => :destroy
+
  delegate :floating_ips,
  :security_groups,
  :cloud_networks,
@@ -41,6 +47,17 @@ class ManageIQ::Providers::Amazon::CloudManager < ManageIQ::Providers::CloudMana
  :to => :network_manager,
  :allow_nil => true
 
+ has_one :block_storage_manager,
+ :foreign_key => :parent_ems_id,
+ :class_name => "ManageIQ::Providers::Amazon::BlockStorageManager",
+ :autosave => true,
+ :dependent => :destroy
+
+ delegate :cloud_volumes,
+ :cloud_volume_snapshots,
+ :to => :block_storage_manager,
+ :allow_nil => true
+
  before_create :ensure_managers
 
  supports :provisioning
@@ -52,6 +69,11 @@ def ensure_managers
  network_manager.name = "#{name} Network Manager"
  network_manager.zone_id = zone_id
  network_manager.provider_region = provider_region
+
+ build_block_storage_manager unless block_storage_manager
+ block_storage_manager.name = "#{name} Block Storage Manager"
+ block_storage_manager.zone_id = zone_id
+ block_storage_manager.provider_region = provider_region
  end
 
  def self.ems_type

app/models/manageiq/providers/amazon/inventory/factory.rb

@@ -14,6 +14,8 @@ def target(ems, target)
  ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager.new(ems, target)
  when ManageIQ::Providers::Amazon::NetworkManager
  ManageIQ::Providers::Amazon::Inventory::Targets::NetworkManager.new(ems, target)
+ when ManageIQ::Providers::Amazon::BlockStorageManager
+ ManageIQ::Providers::Amazon::Inventory::Targets::BlockStorageManager.new(ems, target)
  else
  ManageIQ::Providers::Amazon::Inventory::Targets::CloudManager.new(ems, target)
  end

app/models/manageiq/providers/amazon/inventory/inventory_collection_default_init_data.rb

@@ -278,4 +278,20 @@ def load_balancer_health_check_members_init_data(extra_attributes = {})
 
  init_data(::LoadBalancerHealthCheckMember, attributes, extra_attributes)
  end
+
+ def cloud_volumes_init_data(extra_attributes = {})
+ attributes = {
+ :association => :cloud_volumes,
+ }
+
+ init_data(::ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolume, attributes, extra_attributes)
+ end
+
+ def cloud_volume_snapshots_init_data(extra_attributes = {})
+ attributes = {
+ :association => :cloud_volume_snapshots,
+ }
+
+ init_data(::ManageIQ::Providers::Amazon::BlockStorageManager::CloudVolumeSnapshot, attributes, extra_attributes)
+ end
 end

app/models/manageiq/providers/amazon/inventory/targets/block_storage_manager.rb

@@ -0,0 +1,13 @@
+class ManageIQ::Providers::Amazon::Inventory::Targets::BlockStorageManager < ManageIQ::Providers::Amazon::Inventory::Targets
+ def initialize_inventory_collections
+ add_inventory_collections(%i(cloud_volumes cloud_volume_snapshots))
+ end
+
+ def cloud_volumes
+ HashCollection.new(aws_ec2.client.describe_volumes[:volumes])
+ end
+
+ def cloud_volume_snapshots
+ HashCollection.new(aws_ec2.client.describe_snapshots(:owner_ids => [:self])[:snapshots])
+ end
+end

config/settings.yml

@@ -43,3 +43,4 @@
  :ems_refresh_worker:
  :ems_refresh_worker_amazon: {}
  :ems_refresh_worker_amazon_network: {}
+ :ems_refresh_worker_amazon_block_storage: {}

spec/models/manageiq/providers/amazon/aws_refresher_spec_common.rb

@@ -33,7 +33,7 @@ def expected_table_counts
  :cloud_subnet => 10,
  :custom_attribute => 0,
  :disk => 10,
- :ext_management_system => 2,
+ :ext_management_system => 3,
  :firewall_rule => 119,
  :flavor => 56,
  :floating_ip => 12,

spec/models/manageiq/providers/amazon/aws_stubs.rb

@@ -30,6 +30,8 @@ def test_counts(scaling = nil)
  :security_group_count => scaling * 20,
  :inbound_firewall_rule_per_security_group_count => scaling * 5,
  :outbound_firewall_rule_per_security_group_count => scaling * 5,
+ :cloud_volume_count => scaling * 5,
+ :cloud_volume_snapshot_count => scaling * 5,
  }
  end
 
@@ -356,4 +358,36 @@ def mocked_instance_health
  end
  mocked_instance_healths
  end
+
+ def mocked_cloud_volumes
+ mocked_cloud_volumes = []
+ test_counts[:cloud_volume_count].times do |i|
+ mocked_cloud_volumes << {
+ :availability_zone => "us-east-1e",
+ :create_time => Time.now,
+ :size => i,
+ :state => "in-use",
+ :volume_id => "volume_id_#{i}",
+ :volume_type => "standard"
+ }
+ end
+
+ { :volumes => mocked_cloud_volumes }
+ end
+
+ def mocked_cloud_volume_snapshots
+ mocked_cloud_volume_snapshots = []
+ test_counts[:cloud_volume_snapshot_count].times do |i|
+ mocked_cloud_volume_snapshots << {
+ :snapshot_id => "snapshot_id_#{i}",
+ :description => "snapshot_desc_#{i}",
+ :start_time => Time.now,
+ :volume_size => i,
+ :state => "completed",
+ :volume_id => "volume_id_#{i}",
+ }
+ end
+
+ { :snapshots => mocked_cloud_volume_snapshots }
+ end
 end