Merge pull request #17452 from slemrmartin/suspend-workers

Suspend providers

app/models/ext_management_system.rb

@@ -72,6 +72,7 @@ def self.api_allowed_attributes
  has_one :iso_datastore, :foreign_key => "ems_id", :dependent => :destroy, :inverse_of => :ext_management_system
 
  belongs_to :zone
+ belongs_to :zone_before_pause, :class_name => "Zone", :inverse_of => :paused_ext_management_systems # used for maintenance mode
 
  has_many :metrics, :as => :resource # Destroy will be handled by purger
  has_many :metric_rollups, :as => :resource # Destroy will be handled by purger
@@ -91,6 +92,8 @@ def self.api_allowed_attributes
  validates :hostname, :presence => true, :if => :hostname_required?
  validate :hostname_uniqueness_valid?, :hostname_format_valid?, :if => :hostname_required?
 
+ validate :validate_ems_enabled_when_zone_changed?, :validate_zone_not_maintenance_when_ems_enabled?
+
  scope :with_eligible_manager_types, ->(eligible_types) { where(:type => eligible_types) }
 
  serialize :options
@@ -112,6 +115,22 @@ def hostname_format_valid?
  errors.add(:hostname, _("format is invalid."))
  end
 
+ # validation - Zone cannot be changed when enabled == false
+ def validate_ems_enabled_when_zone_changed?
+ return if enabled_changed?
+
+ if zone_id_changed? && !enabled?
+ errors.add(:zone, N_("cannot be changed because the provider is paused"))
+ end
+ end
+
+ # validation - Zone cannot be maintenance_zone when enabled == true
+ def validate_zone_not_maintenance_when_ems_enabled?
+ if enabled? && zone.present? && zone == Zone.maintenance_zone
+ errors.add(:zone, N_("cannot be the maintenance zone when provider is active"))
+ end
+ end
+
  include NewWithTypeStiMixin
  include UuidMixin
  include EmsRefresh::Manager
@@ -189,6 +208,51 @@ def hostname_format_valid?
 
  default_value_for :enabled, true
 
+ after_save :change_maintenance_for_child_managers, :if => proc { |ems| ems.enabled_changed? }
+
+ def disable!
+ _log.info("Disabling EMS [#{name}] id [#{id}].")
+ update!(:enabled => false)
+ _log.info("Disabling EMS [#{name}] id [#{id}] successful.")
+ end
+
+ def enable!
+ _log.info("Enabling EMS [#{name}] id [#{id}].")
+ update!(:enabled => true)
+ _log.info("Enabling EMS [#{name}] id [#{id}] successful.")
+ end
+
+ # Move ems to maintenance zone and backup current one
+ # @param orig_zone [Integer] because of zone of child manager can be changed by parent manager's ensure_managers() callback
+ # we need to specify original zone for children explicitly
+ def pause!(orig_zone = nil)
+ _log.info("Pausing EMS [#{name}] id [#{id}].")
+ update!(
+ :zone_before_pause => orig_zone || zone,
+ :zone => Zone.maintenance_zone,
+ :enabled => false
+ )
+ _log.info("Pausing EMS [#{name}] id [#{id}] successful.")
+ end
+
+ # Move ems to original zone, reschedule task/jobs/.. collected during maintenance
+ def resume!
+ _log.info("Resuming EMS [#{name}] id [#{id}].")
+
+ new_zone = if zone_before_pause.nil?
+ zone == Zone.maintenance_zone ? Zone.default_zone : zone
+ else
+ zone_before_pause
+ end
+
+ update!(
+ :zone_before_pause => nil,
+ :zone => new_zone,
+ :enabled => true
+ )
+ _log.info("Resuming EMS [#{name}] id [#{id}] successful.")
+ end
+
  def self.with_ipaddress(ipaddress)
  joins(:endpoints).where(:endpoints => {:ipaddress => ipaddress})
  end
@@ -458,16 +522,6 @@ def self.ems_physical_infra_discovery_types
  @ems_physical_infra_discovery_types ||= %w(lenovo_ph_infra)
  end
 
- def disable!
- _log.info("Disabling EMS [#{name}] id [#{id}].")
- update!(:enabled => false)
- end
-
- def enable!
- _log.info("Enabling EMS [#{name}] id [#{id}].")
- update!(:enabled => true)
- end
-
  # override destroy_queue from AsyncDeleteMixin
  def self.destroy_queue(ids)
  find(Array.wrap(ids)).map(&:destroy_queue)
@@ -752,6 +806,17 @@ def allow_targeted_refresh?
 
  private
 
+ # Child managers went to/from maintenance mode with parent
+ def change_maintenance_for_child_managers
+ child_managers.each do |child_manager|
+ if enabled?
+ child_manager.resume!
+ else
+ child_manager.pause!(zone_before_pause)
+ end
+ end
+ end
+
  def build_connection(options = {})
  build_endpoint_by_role(options[:endpoint])
  build_authentication_by_role(options[:authentication])
@@ -772,7 +837,7 @@ def build_authentication_by_role(options)
  role = options.delete(:role)
  creds = {}
  creds[role] = options
- update_authentication(creds,options)
+ update_authentication(creds, options)
  end
 
  def clear_association_cache

app/models/miq_queue.rb

@@ -128,6 +128,11 @@ def self.put(options)
  :handler_id => nil,
  )
 
+ if options[:zone].present? && options[:zone] == Zone.maintenance_zone&.name
+ _log.debug("MiqQueue#put skipped: #{options.inspect}")
+ return
+ end
+
  create_with_options = all.values[:create_with] || {}
  options[:priority] ||= create_with_options[:priority] || NORMAL_PRIORITY
  options[:queue_name] ||= create_with_options[:queue_name] || "generic"

app/models/miq_region.rb

@@ -1,4 +1,6 @@
 class MiqRegion < ApplicationRecord
+ belongs_to :maintenance_zone, :class_name => 'Zone', :inverse_of => false
+
  has_many :metrics, :as => :resource # Destroy will be handled by purger
  has_many :metric_rollups, :as => :resource # Destroy will be handled by purger
  has_many :vim_performance_states, :as => :resource # Destroy will be handled by purger

app/models/miq_server.rb

@@ -39,6 +39,8 @@ class MiqServer < ApplicationRecord
  scope :with_zone_id, ->(zone_id) { where(:zone_id => zone_id) }
  virtual_delegate :description, :to => :zone, :prefix => true
 
+ validate :validate_zone_not_maintenance?
+
  STATUS_STARTING = 'starting'.freeze
  STATUS_STARTED = 'started'.freeze
  STATUS_RESTARTING = 'restarting'.freeze
@@ -53,6 +55,10 @@ class MiqServer < ApplicationRecord
 
  RESTART_EXIT_STATUS = 123
 
+ def validate_zone_not_maintenance?
+ errors.add(:zone, N_('cannot be maintenance zone')) if zone == Zone.maintenance_zone
+ end
+
  def hostname
  h = super
  h if h.to_s.hostname?

app/models/zone.rb

@@ -4,10 +4,11 @@ class Zone < ApplicationRecord
 
  serialize :settings, Hash
 
- belongs_to  :log_file_depot, :class_name => "FileDepot"
+ belongs_to :log_file_depot, :class_name => "FileDepot"
 
  has_many :miq_servers
  has_many :ext_management_systems
+ has_many :paused_ext_management_systems, :class_name => 'ExtManagementSystem', :foreign_key => :zone_before_pause_id
  has_many :container_managers, :class_name => "ManageIQ::Providers::ContainerManager"
  has_many :miq_schedules, :dependent => :destroy
  has_many :providers
@@ -36,6 +37,9 @@ class Zone < ApplicationRecord
  include AggregationMixin
  include ConfigurationManagementMixin
 
+ scope :visible, -> { where(:visible => true) }
+ default_value_for :visible, true
+
  def active_miq_servers
  MiqServer.active_miq_servers.where(:zone_id => id)
  end
@@ -48,7 +52,31 @@ def find_master_server
  active_miq_servers.detect(&:is_master?)
  end
 
+ def self.create_maintenance_zone
+ return MiqRegion.my_region.maintenance_zone if MiqRegion.my_region.maintenance_zone.present?
+
+ begin
+ # 1) Create Maintenance zone
+ zone = create!(:name => "__maintenance__#{SecureRandom.uuid}",
+ :description => "Maintenance Zone",
+ :visible => false)
+
+ # 2) Assign to MiqRegion
+ MiqRegion.my_region.update_attributes(:maintenance_zone => zone)
+ rescue ActiveRecord::RecordInvalid
+ raise if zone.errors[:name].blank?
+ retry
+ end
+ _log.info("Creating maintenance zone...")
+ zone
+ end
+
+ private_class_method :create_maintenance_zone
+
  def self.seed
+ MiqRegion.seed
+ create_maintenance_zone
+
  create_with(:description => "Default Zone").find_or_create_by!(:name => 'default') do |_z|
  _log.info("Creating default zone...")
  end
@@ -78,6 +106,11 @@ def self.default_zone
  in_my_region.find_by(:name => "default")
  end
 
+ # Zone for paused providers (no servers in it), not visible by default
+ cache_with_timeout(:maintenance_zone) do
+ MiqRegion.my_region&.maintenance_zone
+ end
+
  def remote_cockpit_ws_miq_server
  role_active?("cockpit_ws") ? miq_servers.find_by(:has_active_cockpit_ws => true) : nil
  end
@@ -209,6 +242,7 @@ def ntp_reload_queue
 
  def check_zone_in_use_on_destroy
  raise _("cannot delete default zone") if name == "default"
+ raise _("cannot delete maintenance zone") if self == miq_region.maintenance_zone
  raise _("zone name '%{name}' is used by a server") % {:name => name} unless miq_servers.blank?
  end
 end

spec/models/automation_request_spec.rb

@@ -1,6 +1,7 @@
 describe AutomationRequest do
  let(:admin) { FactoryBot.create(:user, :role => "admin") }
  before do
+ MiqRegion.seed
  allow(MiqServer).to receive(:my_zone).and_return(Zone.seed.name)
  @zone = FactoryBot.create(:zone, :name => "fred")
  @approver = FactoryBot.create(:user_miq_request_approver)

spec/models/custom_button_spec.rb

@@ -91,6 +91,7 @@
 
  context "when invoking for a particular VM" do
  before do
+ MiqRegion.seed
  @vm = FactoryBot.create(:vm_vmware)
  @user2 = FactoryBot.create(:user_with_group)
  EvmSpecHelper.local_miq_server(:is_master => true, :zone => Zone.seed)
@@ -321,6 +322,7 @@
  let(:custom_button) { FactoryBot.create(:custom_button, :applies_to => vm.class, :resource_action => resource_action) }
 
  before do
+ MiqRegion.seed
  EvmSpecHelper.local_miq_server(:is_master => true, :zone => Zone.seed)
  end