Merge pull request #18033 from fdupont-redhat/conversion_host_backpor…

…t_from_automate

Add methods to conversion_host to build virt-v2v wrapper options

(cherry picked from commit fcd7161)

https://bugzilla.redhat.com/show_bug.cgi?id=1634029

app/models/conversion_host.rb

@@ -4,4 +4,41 @@ class ConversionHost < ApplicationRecord
   acts_as_miq_taggable
 
   belongs_to :resource, :polymorphic => true
+  has_many :service_template_transformation_plan_tasks, :dependent => :nullify
+  has_many :active_tasks, -> { where(:state => 'active') }, :class_name => ServiceTemplateTransformationPlanTask, :inverse_of => :conversion_host
+
+  # To be eligible, a conversion host must have the following properties
+  #  - A transport mechanism is configured for source (set by 3rd party)
+  #  - Credentials are set on the resource
+  #  - The number of concurrent tasks has not reached the limit
+  def eligible?
+    source_transport_method.present? && check_resource_credentials && check_concurrent_tasks
+  end
+
+  def check_concurrent_tasks
+    max_tasks = max_concurrent_tasks || Settings.transformation.limits.max_concurrent_tasks_per_host
+    active_tasks.size < max_tasks
+  end
+
+  def check_resource_credentials
+    send("check_resource_credentials_#{resource.ext_management_system.emstype}")
+  end
+
+  def source_transport_method
+    return 'vddk' if vddk_transport_supported
+    return 'ssh' if ssh_transport_supported
+  end
+
+  private
+
+  def check_resource_credentials_rhevm
+    !(resource.authentication_userid.nil? || resource.authentication_password.nil?)
+  end
+
+  def check_resource_credentials_openstack
+    ssh_authentications = resource.ext_management_system.authentications
+                                  .where(:authtype => 'ssh_keypair')
+                                  .where.not(:userid => nil, :auth_key => nil)
+    !ssh_authentications.empty?
+  end
 end

app/models/service_template_transformation_plan_task.rb

@@ -1,4 +1,7 @@
 class ServiceTemplateTransformationPlanTask < ServiceTemplateProvisionTask
+  belongs_to :conversion_host
+  delegate :source_transport_method, :to => :conversion_host
+
   def self.base_model
     ServiceTemplateTransformationPlanTask
   end
@@ -46,21 +49,72 @@ def task_active
     vm_resource.update_attributes(:status => ServiceResource::STATUS_ACTIVE)
   end
 
-  def conversion_host
-    Host.find_by(:id => options[:transformation_host_id])
+  def source_ems
+    source.ext_management_system
+  end
+
+  def destination_ems
+    transformation_destination(source.ems_cluster).ext_management_system
+  end
+
+  def source_disks
+    options[:source_disks] ||= source.hardware.disks.select { |d| d.device_type == 'disk' }.collect do |disk|
+      source_storage = disk.storage
+      destination_storage = transformation_destination(disk.storage)
+      raise "[#{source.name}] Disk #{disk.device_name} [#{source_storage.name}] has no mapping. Aborting." if destination_storage.nil?
+      {
+        :path    => disk.filename,
+        :size    => disk.size,
+        :percent => 0,
+        :weight  => disk.size.to_f / source.allocated_disk_storage.to_f * 100
+      }
+    end
+  end
+
+  def network_mappings
+    options[:network_mappings] ||= source.hardware.nics.select { |n| n.device_type == 'ethernet' }.collect do |nic|
+      source_network = nic.lan
+      destination_network = transformation_destination(source_network)
+      raise "[#{source.name}] NIC #{nic.device_name} [#{source_network.name}] has no mapping. Aborting." if destination_network.nil?
+      {
+        :source      => source_network.name,
+        :destination => destination_network_ref(destination_network),
+        :mac_address => nic.address
+      }
+    end
+  end
+
+  def destination_network_ref(network)
+    send("destination_network_ref_#{destination_ems.emstype}", network)
+  end
+
+  def destination_network_ref_rhevm(network)
+    network.name
+  end
+
+  def destination_network_ref_openstack(network)
+    network.ems_ref
+  end
+
+  def destination_flavor
+    Flavor.find_by(:id => miq_request.source.options[:config_info][:osp_flavor])
+  end
+
+  def destination_security_group
+    SecurityGroup.find_by(:id => miq_request.source.options[:config_info][:osp_security_group])
   end
 
   def transformation_log
     host = conversion_host
     if host.nil?
-      msg = "Conversion host was not found: ID [#{options[:transformation_host_id]}]. Download of transformation log aborted."
+      msg = "Conversion host was not found. Download of transformation log aborted."
       _log.error(msg)
       raise MiqException::Error, msg
     end
 
-    userid, password = host.auth_user_pwd(:remote)
+    userid, password = host.resource.auth_user_pwd(:remote)
     if userid.blank? || password.blank?
-      msg = "Credential was not found for host #{host.name}. Download of transformation log aborted."
+      msg = "Credential was not found for host #{host.resource.name}. Download of transformation log aborted."
       _log.error(msg)
       raise MiqException::Error, msg
     end
@@ -74,7 +128,7 @@ def transformation_log
 
     begin
       require 'net/scp'
-      Net::SCP.download!(host.ipaddress, userid, logfile, nil, :ssh => {:password => password})
+      Net::SCP.download!(host.resource.ipaddress, userid, logfile, nil, :ssh => {:password => password})
     rescue Net::SCP::Error => scp_err
       _log.error("Download of transformation log for #{description} with ID [#{id}] failed with error: #{scp_err.message}")
       raise scp_err
@@ -87,7 +141,7 @@ def transformation_log_queue(userid = nil)
     userid ||= User.current_userid || 'system'
     host = conversion_host
     if host.nil?
-      msg = "Conversion host was not found: ID [#{options[:transformation_host_id]}]. Cannot queue the download of transformation log."
+      msg = "Conversion host was not found. Cannot queue the download of transformation log."
       return create_error_status_task(userid, msg).id
     end
 
@@ -98,7 +152,7 @@ def transformation_log_queue(userid = nil)
                      :instance_id => id,
                      :priority    => MiqQueue::HIGH_PRIORITY,
                      :args        => [],
-                     :zone        => host.my_zone}
+                     :zone        => host.resource.my_zone}
     MiqTask.generic_action_with_callback(options, queue_options)
   end
 
@@ -114,6 +168,23 @@ def canceled
     update_attributes(:cancelation_status => MiqRequestTask::CANCEL_STATUS_FINISHED)
   end
 
+  def conversion_options
+    source_cluster = source.ems_cluster
+    source_storage = source.hardware.disks.select { |d| d.device_type == 'disk' }.first.storage
+    destination_cluster = transformation_destination(source_cluster)
+    destination_storage = transformation_destination(source_storage)
+
+    options = {
+      :source_disks     => source_disks.map { |disk| disk[:path] },
+      :network_mappings => network_mappings
+    }
+
+    options.merge!(send("conversion_options_source_provider_#{source_ems.emstype}_#{source_transport_method}", source_storage))
+    options.merge!(send("conversion_options_destination_provider_#{destination_ems.emstype}", destination_cluster, destination_storage))
+
+    options
+  end
+
   private
 
   def vm_resource
@@ -129,4 +200,60 @@ def create_error_status_task(userid, msg)
       :message => msg
     )
   end
+
+  def conversion_options_source_provider_vmwarews_vddk(_storage)
+    {
+      :vm_name            => source.name,
+      :transport_method   => 'vddk',
+      :vmware_fingerprint => source.host.thumbprint_sha1,
+      :vmware_uri         => URI::Generic.build(
+        :scheme   => 'esx',
+        :userinfo => CGI.escape(source.host.authentication_userid),
+        :host     => source.host.ipaddress,
+        :path     => '/',
+        :query    => { :no_verify => 1 }.to_query
+      ).to_s,
+      :vmware_password    => source.host.authentication_password
+    }
+  end
+
+  def conversion_options_source_provider_vmwarews_ssh(storage)
+    {
+      :vm_name          => URI::Generic.build(:scheme => 'ssh', :userinfo => 'root', :host => source.host.ipaddress, :path => "/vmfs/volumes").to_s + "/#{storage.name}/#{source.location}",
+      :transport_method => 'ssh'
+    }
+  end
+
+  def conversion_options_destination_provider_rhevm(cluster, storage)
+    {
+      :rhv_url             => URI::Generic.build(:scheme => 'https', :host => destination_ems.hostname, :path => '/ovirt-engine/api').to_s,
+      :rhv_cluster         => cluster.name,
+      :rhv_storage         => storage.name,
+      :rhv_password        => destination_ems.authentication_password,
+      :install_drivers     => true,
+      :insecure_connection => true
+    }
+  end
+
+  def conversion_options_destination_provider_openstack(cluster, storage)
+    {
+      :osp_environment            => {
+        :os_no_cache         => true,
+        :os_auth_url         => URI::Generic.build(
+          :scheme => destination_ems.security_protocol == 'non-ssl' ? 'http' : 'https',
+          :host   => destination_ems.hostname,
+          :port   => destination_ems.port,
+          :path   => destination_ems.api_version
+        ),
+        :os_user_domain_name => destination_ems.uid_ems,
+        :os_username         => destination_ems.authentication_userid,
+        :os_password         => destination_ems.authentication_password,
+        :os_project_name     => cluster.name
+      },
+      :osp_destination_project_id => cluster.ems_ref,
+      :osp_volume_type_id         => storage.ems_ref,
+      :osp_flavor_id              => destination_flavor.ems_ref,
+      :osp_security_groups_ids    => [destination_security_group.ems_ref]
+    }
+  end
 end

config/settings.yml

@@ -1098,7 +1098,9 @@
   :history:
     :keep_tasks: 1.week
     :purge_window_size: 1000
-
+:transformation:
+  :limits:
+    :max_concurrent_tasks_per_host: 10
 :ui:
   :mark_translated_strings: false
   :url:

spec/models/conversion_host_spec.rb

@@ -0,0 +1,101 @@
+describe ConversionHost do
+  let(:apst) { FactoryGirl.create(:service_template_ansible_playbook) }
+
+  context "provider independent methods" do
+    let(:host) { FactoryGirl.create(:host) }
+    let(:vm) { FactoryGirl.create(:vm_or_template) }
+    let(:conversion_host_1) { FactoryGirl.create(:conversion_host, :resource => host) }
+    let(:conversion_host_2) { FactoryGirl.create(:conversion_host, :resource => vm) }
+    let(:task_1) { FactoryGirl.create(:service_template_transformation_plan_task, :state => 'active', :conversion_host => conversion_host_1) }
+    let(:task_2) { FactoryGirl.create(:service_template_transformation_plan_task, :conversion_host => conversion_host_1) }
+    let(:task_3) { FactoryGirl.create(:service_template_transformation_plan_task, :state => 'active', :conversion_host => conversion_host_2) }
+
+    before do
+      allow(conversion_host_1).to receive(:active_tasks).and_return([task_1])
+      allow(conversion_host_2).to receive(:active_tasks).and_return([task_3])
+    end
+
+    describe "#check_concurrent_tasks" do
+      context "default max concurrent tasks is equal to current active tasks" do
+        before { stub_settings_merge(:transformation => {:limits => {:max_concurrent_tasks_per_host => 1}}) }
+        it { expect(conversion_host_1.check_concurrent_tasks).to eq(false) }
+      end
+
+      context "default max concurrent tasks is greater than current active tasks" do
+        before { stub_settings_merge(:transformation => {:limits => {:max_concurrent_tasks_per_host => 10}}) }
+        it { expect(conversion_host_1.check_concurrent_tasks).to eq(true) }
+      end
+
+      context "host's max concurrent tasks is equal to current active tasks" do
+        before { conversion_host_1.max_concurrent_tasks = "1" }
+        it { expect(conversion_host_1.check_concurrent_tasks).to eq(false) }
+      end
+
+      context "host's max concurrent tasks greater than current active tasks" do
+        before { conversion_host_2.max_concurrent_tasks = "2" }
+        it { expect(conversion_host_2.check_concurrent_tasks).to eq(true) }
+      end
+    end
+
+    context "#source_transport_method" do
+      it { expect(conversion_host_2.source_transport_method).to be_nil }
+
+      context "ssh transport enabled" do
+        before { conversion_host_2.ssh_transport_supported = true }
+        it { expect(conversion_host_2.source_transport_method).to eq('ssh') }
+
+        context "vddk transport enabled" do
+          before { conversion_host_2.vddk_transport_supported = true }
+          it { expect(conversion_host_2.source_transport_method).to eq('vddk') }
+        end
+      end
+    end
+  end
+
+  context "resource provider is rhevm" do
+    let(:ems) { FactoryGirl.create(:ems_redhat, :zone => FactoryGirl.create(:zone)) }
+    let(:host) { FactoryGirl.create(:host, :ext_management_system => ems) }
+    let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => host, :vddk_transport_supported => true) }
+
+    context "host userid is nil" do
+      before { allow(host).to receive(:authentication_userid).and_return(nil) }
+      it { expect(conversion_host.check_resource_credentials).to eq(false) }
+    end
+
+    context "host userid is set" do
+      before { allow(host).to receive(:authentication_userid).and_return('root') }
+
+      context "and host password is nil" do
+        before { allow(host).to receive(:authentication_password).and_return(nil) }
+        it { expect(conversion_host.check_resource_credentials).to eq(false) }
+      end
+
+      context "and host password is set" do
+        before { allow(host).to receive(:authentication_password).and_return('password') }
+        it { expect(conversion_host.check_resource_credentials).to eq(true) }
+      end
+    end
+  end
+
+  context "resource provider is openstack" do
+    let(:ems) { FactoryGirl.create(:ems_openstack, :zone => FactoryGirl.create(:zone)) }
+    let(:vm) { FactoryGirl.create(:vm, :ext_management_system => ems) }
+    let(:conversion_host) { FactoryGirl.create(:conversion_host, :resource => vm, :vddk_transport_supported => true) }
+
+    context "ems authentications is empty" do
+      it { expect(conversion_host.check_resource_credentials).to be(false) }
+    end
+
+    context "ems authentications contains ssh_auth" do
+      let(:ssh_auth) { FactoryGirl.create(:authentication_ssh_keypair, :resource => ems) }
+
+      it "with fake auth" do
+        allow(ems).to receive(:authentications).and_return(ssh_auth)
+        allow(ssh_auth).to receive(:where).with(:authype => 'ssh_keypair').and_return(ssh_auth)
+        allow(ssh_auth).to receive(:where).and_return(ssh_auth)
+        allow(ssh_auth).to receive(:not).with(:userid => nil, :auth_key => nil).and_return([ssh_auth])
+        expect(conversion_host.check_resource_credentials).to be(true)
+      end
+    end
+  end
+end