diff --git a/src/vmm_mad/remotes/vcenter/cancel b/src/vmm_mad/remotes/vcenter/cancel index bc3e5269d24..987fab5311f 100755 --- a/src/vmm_mad/remotes/vcenter/cancel +++ b/src/vmm_mad/remotes/vcenter/cancel @@ -29,6 +29,8 @@ $LOAD_PATH << File.dirname(__FILE__) require 'vcenter_driver' +CONFIG = VCenterConf.new + vm_ref = ARGV[0] host = ARGV[1] vm_id = ARGV[-2] @@ -43,16 +45,26 @@ check_valid(lcm_state, 'lcm_state') lcm_state_str = OpenNebula::VirtualMachine::LCM_STATE[lcm_state.to_i] begin + retries ||= 0 vi_client = VCenterDriver::VIClient.new_from_host(host_id) vm = VCenterDriver::VirtualMachine.new(vi_client, vm_ref, vm_id) vm.poweroff_hard rescue StandardError => e - message = "Cancel VM #{vm_ref} failed due to "\ - "\"#{e.message}\"\n#{e.backtrace.join("\n")}" + if (retries += 1) < CONFIG[:retries] + message = "Cancel VM #{vm_ref} failed due to "\ + "\"#{e.message}\"\n#{e.backtrace.join("\n")} "\ + "on the attempt \##{retries}\n#{e.backtrace.join("\n")}" + else + message = "Cancel VM #{vm_ref} failed due to "\ + "\"#{e.message}\"\n#{e.backtrace.join("\n")} "\ + "on the final attempt\n#{e.backtrace.join("\n")}" + end OpenNebula.log_error(message) STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + sleep CONFIG[:retry_interval].to_i + retry if retries < CONFIG[:retries] exit(-1) ensure diff --git a/src/vmm_mad/remotes/vcenter/deploy b/src/vmm_mad/remotes/vcenter/deploy index f2d8dd18484..28bf5caa6a3 100755 --- a/src/vmm_mad/remotes/vcenter/deploy +++ b/src/vmm_mad/remotes/vcenter/deploy @@ -29,6 +29,8 @@ $LOAD_PATH << File.dirname(__FILE__) require 'vcenter_driver' +CONFIG = VCenterConf.new + dfile = ARGV[0] cluster_name = ARGV[1] vm_id = ARGV[2] @@ -47,6 +49,7 @@ host_id = drv_action['HISTORY_RECORDS/HISTORY/HID'] deploy = {} begin + retries ||= 0 vi_client = VCenterDriver::VIClient.new_from_host(host_id) one_vm = VCenterDriver::VIHelper.one_item(OpenNebula::VirtualMachine, vm_id) @@ -67,10 +70,19 @@ begin puts vm['_ref'] rescue StandardError => e - message = "Deploy of VM #{vm_id} on vCenter cluster #{cluster_name} " \ - "with #{dfile} failed due to \"#{e.message}\"." + if (retries += 1) < CONFIG[:retries] + message = "Deploy of VM #{vm_id} on vCenter cluster #{cluster_name} " \ + "with #{dfile} failed due to \"#{e.message}\"" \ + "on the attempt \##{retries}." + else + message = "Deploy of VM #{vm_id} on vCenter cluster #{cluster_name} " \ + "with #{dfile} failed due to \"#{e.message}\"" \ + "on the final attempt." + end OpenNebula.log_error(message) STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + sleep CONFIG[:retry_interval].to_i + retry if retries < CONFIG[:retries] exit(-1) ensure diff --git a/src/vmm_mad/remotes/vcenter/shutdown b/src/vmm_mad/remotes/vcenter/shutdown index 0a6166cb66a..a7103c0fd7b 100755 --- a/src/vmm_mad/remotes/vcenter/shutdown +++ b/src/vmm_mad/remotes/vcenter/shutdown @@ -29,6 +29,8 @@ $LOAD_PATH << File.dirname(__FILE__) require 'vcenter_driver' +CONFIG = VCenterConf.new + vm_ref = ARGV[0] vc_cluster_name = ARGV[1] vm_id = ARGV[2] @@ -50,6 +52,7 @@ if !(%{'SAVE_MIGRATE', 'SHUTDOWN', 'SHUTDOWN_POWEROFF', 'SHUTDOWN_UNDEPLOY'}).in end begin + retries ||= 0 vi_client = VCenterDriver::VIClient.new_from_host(host_id) if vm_ref.empty? @@ -63,11 +66,19 @@ begin vm.shutdown # Undeploy, Poweroff or Terminate rescue StandardError => e - message = "Shutdown of VM #{vm_ref} on vCenter cluster "\ - "#{vc_cluster_name} failed due to "\ - "\"#{e.message}\"\n#{e.backtrace.join("\n")}" + if (retries += 1) < CONFIG[:retries] + message = "Shutdown of VM #{vm_ref} on vCenter cluster "\ + "#{vc_cluster_name} failed due to "\ + "\"#{e.message}\" on the attempt \##{retries}\n#{e.backtrace.join("\n")}" + else + message = "Shutdown of VM #{vm_ref} on vCenter cluster "\ + "#{vc_cluster_name} failed due to "\ + "\"#{e.message}\" on the final attempt\n#{e.backtrace.join("\n")}" + end OpenNebula.log_error(message) STDERR.puts "#{message} #{e.backtrace}" if VCenterDriver::CONFIG[:debug_information] + sleep CONFIG[:retry_interval].to_i + retry if retries < CONFIG[:retries] exit(-1) ensure diff --git a/src/vmm_mad/remotes/vcenter/vcenter_driver.rb b/src/vmm_mad/remotes/vcenter/vcenter_driver.rb index 808b403e475..519e564efc2 100644 --- a/src/vmm_mad/remotes/vcenter/vcenter_driver.rb +++ b/src/vmm_mad/remotes/vcenter/vcenter_driver.rb @@ -44,7 +44,9 @@ class VCenterConf < Hash DEFAULT_CONFIGURATION = { :delete_images => false, :vm_poweron_wait_default => 300, - :debug_information => false + :debug_information => false, + :retries => 3, + :retry_interval => 1 } def initialize