From 2947aac92c27defc0818e00fa4760af0b4974bab Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Tue, 19 Nov 2019 14:26:14 -0500 Subject: [PATCH 1/6] Add MiqWorkerType This adds a model which will replace the worker type constant. It seeds the table based on the leaf subclasses of MiqWorker and each worker class needs to define its own attributes, specifically its priority in the kill order and bundler groups. This allows new workers to be added by simply adding a new subclass of MiqWorker rather than maintaining a list of worker classes in the core repo. Additionally this brings us closer to the ability to run a separate process that will access the database and determine which workers should be running on which servers, all outside of the main application code-base. --- .../providers/base_manager/event_catcher.rb | 4 ++++ .../base_manager/metrics_collector_worker.rb | 4 ++++ .../base_manager/operations_worker.rb | 2 +- .../providers/base_manager/refresh_worker.rb | 6 ++++- app/models/miq_cockpit_ws_worker.rb | 4 ++++ .../miq_ems_metrics_processor_worker.rb | 4 ++++ app/models/miq_event_handler.rb | 4 ++++ app/models/miq_generic_worker.rb | 4 ++++ app/models/miq_priority_worker.rb | 4 ++++ app/models/miq_remote_console_worker.rb | 4 ++++ app/models/miq_reporting_worker.rb | 4 ++++ app/models/miq_schedule_worker.rb | 4 ++++ app/models/miq_smart_proxy_worker.rb | 4 ++++ app/models/miq_ui_worker.rb | 8 +++++++ app/models/miq_web_service_worker.rb | 8 +++++++ app/models/miq_worker.rb | 8 +++++++ app/models/miq_worker_type.rb | 24 +++++++++++++++++++ config/replication_exclude_tables.yml | 1 + lib/evm_database.rb | 1 + spec/models/miq_worker_type_spec.rb | 21 ++++++++++++++++ 20 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 app/models/miq_worker_type.rb create mode 100644 spec/models/miq_worker_type_spec.rb diff --git a/app/models/manageiq/providers/base_manager/event_catcher.rb b/app/models/manageiq/providers/base_manager/event_catcher.rb index aa11664dce5..8d9aaea7dd3 100644 --- a/app/models/manageiq/providers/base_manager/event_catcher.rb +++ b/app/models/manageiq/providers/base_manager/event_catcher.rb @@ -15,4 +15,8 @@ def friendly_name end end end + + def self.kill_priority + 130 + end end diff --git a/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb b/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb index 1b97bd31207..44e793a7a37 100644 --- a/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb +++ b/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb @@ -14,4 +14,8 @@ def self.supports_container? def self.normalized_type @normalized_type ||= "ems_metrics_collector_worker" end + + def self.kill_priority + 20 + end end diff --git a/app/models/manageiq/providers/base_manager/operations_worker.rb b/app/models/manageiq/providers/base_manager/operations_worker.rb index f0bc53add3c..c0189130972 100644 --- a/app/models/manageiq/providers/base_manager/operations_worker.rb +++ b/app/models/manageiq/providers/base_manager/operations_worker.rb @@ -3,7 +3,7 @@ class ManageIQ::Providers::BaseManager::OperationsWorker < MiqQueueWorkerBase include PerEmsWorkerMixin - self.required_roles = "ems_operations" + self.required_roles = %w[ems_operations] def friendly_name @friendly_name ||= begin diff --git a/app/models/manageiq/providers/base_manager/refresh_worker.rb b/app/models/manageiq/providers/base_manager/refresh_worker.rb index c5bb6c003e5..5b8e4860f3e 100644 --- a/app/models/manageiq/providers/base_manager/refresh_worker.rb +++ b/app/models/manageiq/providers/base_manager/refresh_worker.rb @@ -5,7 +5,7 @@ class ManageIQ::Providers::BaseManager::RefreshWorker < MiqQueueWorkerBase # Don't allow multiple refresh workers to run at once self.include_stopping_workers_on_synchronize = true - self.required_roles = "ems_inventory" + self.required_roles = %w[ems_inventory] def friendly_name @friendly_name ||= begin @@ -21,4 +21,8 @@ def friendly_name def self.normalized_type @normalized_type ||= "ems_refresh_worker" end + + def self.kill_priority + 70 + end end diff --git a/app/models/miq_cockpit_ws_worker.rb b/app/models/miq_cockpit_ws_worker.rb index a6ee0374e18..f352d4d200f 100644 --- a/app/models/miq_cockpit_ws_worker.rb +++ b/app/models/miq_cockpit_ws_worker.rb @@ -9,6 +9,10 @@ def friendly_name @friendly_name ||= "Cockpit Worker" end + def self.kill_priority + 160 + end + def self.can_start_cockpit_ws? @supports_cockpit_ws ||= MiqCockpit::WS.can_start_cockpit_ws? end diff --git a/app/models/miq_ems_metrics_processor_worker.rb b/app/models/miq_ems_metrics_processor_worker.rb index 02cd7bd744e..6720520080d 100644 --- a/app/models/miq_ems_metrics_processor_worker.rb +++ b/app/models/miq_ems_metrics_processor_worker.rb @@ -13,4 +13,8 @@ def friendly_name def self.supports_container? true end + + def self.kill_priority + 10 + end end diff --git a/app/models/miq_event_handler.rb b/app/models/miq_event_handler.rb index 569fd90225e..be913b315e5 100644 --- a/app/models/miq_event_handler.rb +++ b/app/models/miq_event_handler.rb @@ -9,4 +9,8 @@ class MiqEventHandler < MiqQueueWorkerBase def self.supports_container? true end + + def self.kill_priority + 60 + end end diff --git a/app/models/miq_generic_worker.rb b/app/models/miq_generic_worker.rb index 5ec84d42851..cab9878af89 100644 --- a/app/models/miq_generic_worker.rb +++ b/app/models/miq_generic_worker.rb @@ -10,4 +10,8 @@ class MiqGenericWorker < MiqQueueWorkerBase def self.supports_container? true end + + def self.kill_priority + 50 + end end diff --git a/app/models/miq_priority_worker.rb b/app/models/miq_priority_worker.rb index 04f39bcf20f..60882a52a0f 100644 --- a/app/models/miq_priority_worker.rb +++ b/app/models/miq_priority_worker.rb @@ -12,4 +12,8 @@ def self.queue_priority def self.supports_container? true end + + def self.kill_priority + 90 + end end diff --git a/app/models/miq_remote_console_worker.rb b/app/models/miq_remote_console_worker.rb index c855a168617..b058cb87edc 100644 --- a/app/models/miq_remote_console_worker.rb +++ b/app/models/miq_remote_console_worker.rb @@ -16,4 +16,8 @@ def friendly_name def self.supports_container? true end + + def self.kill_priority + 150 + end end diff --git a/app/models/miq_reporting_worker.rb b/app/models/miq_reporting_worker.rb index 4c41597e406..a4b3b632d89 100644 --- a/app/models/miq_reporting_worker.rb +++ b/app/models/miq_reporting_worker.rb @@ -9,4 +9,8 @@ class MiqReportingWorker < MiqQueueWorkerBase def self.supports_container? true end + + def self.kill_priority + 30 + end end diff --git a/app/models/miq_schedule_worker.rb b/app/models/miq_schedule_worker.rb index 04092005544..2008d09728e 100644 --- a/app/models/miq_schedule_worker.rb +++ b/app/models/miq_schedule_worker.rb @@ -13,4 +13,8 @@ class MiqScheduleWorker < MiqWorker def self.supports_container? true end + + def self.kill_priority + 80 + end end diff --git a/app/models/miq_smart_proxy_worker.rb b/app/models/miq_smart_proxy_worker.rb index 5f3e4ba591d..41a41afbf62 100644 --- a/app/models/miq_smart_proxy_worker.rb +++ b/app/models/miq_smart_proxy_worker.rb @@ -3,4 +3,8 @@ class MiqSmartProxyWorker < MiqQueueWorkerBase self.required_roles = ["smartproxy"] self.default_queue_name = "smartproxy" + + def self.kill_priority + 40 + end end diff --git a/app/models/miq_ui_worker.rb b/app/models/miq_ui_worker.rb index c9c25f9007c..d3641a00568 100644 --- a/app/models/miq_ui_worker.rb +++ b/app/models/miq_ui_worker.rb @@ -26,6 +26,14 @@ def self.supports_container? true end + def self.bundler_groups + %w[manageiq_default ui_dependencies graphql_api] + end + + def self.kill_priority + 140 + end + def container_port 3001 end diff --git a/app/models/miq_web_service_worker.rb b/app/models/miq_web_service_worker.rb index a640edb630c..d492489cfb6 100644 --- a/app/models/miq_web_service_worker.rb +++ b/app/models/miq_web_service_worker.rb @@ -15,4 +15,12 @@ def friendly_name def self.supports_container? true end + + def self.bundler_groups + %w[manageiq_default graphql_api] + end + + def self.kill_priority + 100 + end end diff --git a/app/models/miq_worker.rb b/app/models/miq_worker.rb index 2dbded968d7..9e3190e4b39 100644 --- a/app/models/miq_worker.rb +++ b/app/models/miq_worker.rb @@ -46,6 +46,14 @@ class << self attr_writer :workers end + def self.bundler_groups + %w[manageiq_default] + end + + def self.kill_priority + raise NotImplementedError, ".kill_priority must be implemented in a subclass" + end + def self.workers return (self.has_minimal_env_option? ? 1 : 0) if MiqServer.minimal_env? && check_for_minimal_role return 0 unless has_required_role? diff --git a/app/models/miq_worker_type.rb b/app/models/miq_worker_type.rb new file mode 100644 index 00000000000..473ab55cee5 --- /dev/null +++ b/app/models/miq_worker_type.rb @@ -0,0 +1,24 @@ +class MiqWorkerType < ApplicationRecord + EXCLUDED_CLASS_NAMES = %w[ManageIQ::Providers::BaseManager::OperationsWorker].freeze + + scope :in_kill_order, -> { order(:kill_priority => :asc) } + + def self.seed + transaction do + classes_for_seed.each { |klass| seed_worker(klass) } + end + end + + private_class_method def self.classes_for_seed + MiqWorker.descendants.select { |w| w.subclasses.empty? } - EXCLUDED_CLASS_NAMES.map(&:constantize) + end + + private_class_method def self.seed_worker(klass) + instance = find_or_initialize_by(:worker_type => klass.name) + + instance.update!( + :bundler_groups => klass.bundler_groups, + :kill_priority => klass.kill_priority + ) + end +end diff --git a/config/replication_exclude_tables.yml b/config/replication_exclude_tables.yml index 3316380019d..f4fdec9b84f 100644 --- a/config/replication_exclude_tables.yml +++ b/config/replication_exclude_tables.yml @@ -74,6 +74,7 @@ - miq_user_roles - miq_widgets - miq_widget_contents +- miq_worker_types - miq_workers - notifications - notification_recipients diff --git a/lib/evm_database.rb b/lib/evm_database.rb index a9c84f51d99..b07c60bac7e 100644 --- a/lib/evm_database.rb +++ b/lib/evm_database.rb @@ -9,6 +9,7 @@ class EvmDatabase Zone MiqServer ServerRole + MiqWorkerType Tenant MiqProductFeature MiqUserRole diff --git a/spec/models/miq_worker_type_spec.rb b/spec/models/miq_worker_type_spec.rb new file mode 100644 index 00000000000..56cd8262698 --- /dev/null +++ b/spec/models/miq_worker_type_spec.rb @@ -0,0 +1,21 @@ +describe MiqWorkerType do + describe ".seed" do + before { described_class.seed } + + it "doesn't create a row for excluded classes" do + expect(described_class.count).to be > 0 + expect(described_class.pluck(:worker_type)).not_to include(*described_class::EXCLUDED_CLASS_NAMES) + end + + it "correctly creates records for workers" do + generic_worker = described_class.find_by(:worker_type => "MiqGenericWorker") + ui_worker = described_class.find_by(:worker_type => "MiqUiWorker") + + expect(generic_worker.bundler_groups).to match_array(MiqGenericWorker.bundler_groups) + expect(generic_worker.kill_priority).to eq(MiqGenericWorker.kill_priority) + + expect(ui_worker.bundler_groups).to match_array(MiqUiWorker.bundler_groups) + expect(ui_worker.kill_priority).to eq(MiqUiWorker.kill_priority) + end + end +end From 03e1c8658c1454031e6e75e59517bf145c34f030 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Tue, 19 Nov 2019 15:35:49 -0500 Subject: [PATCH 2/6] Replace MIQ_WORKER_TYPES constant by using MiqWorkerType model --- .../worker_management/monitor/class_names.rb | 11 +- .../manageiq/provider/provider_generator.rb | 6 - lib/workers/bin/run_single_worker.rb | 12 +- lib/workers/miq_worker_types.rb | 146 ------------------ spec/lib/workers/miq_worker_types_spec.rb | 14 -- spec/models/message_timeout_handling_spec.rb | 1 + spec/models/miq_server/worker_monitor_spec.rb | 1 + .../miq_worker/container_common_spec.rb | 3 +- 8 files changed, 12 insertions(+), 182 deletions(-) delete mode 100644 lib/workers/miq_worker_types.rb delete mode 100644 spec/lib/workers/miq_worker_types_spec.rb diff --git a/app/models/miq_server/worker_management/monitor/class_names.rb b/app/models/miq_server/worker_management/monitor/class_names.rb index 973d8efc67c..4de09fa0de7 100644 --- a/app/models/miq_server/worker_management/monitor/class_names.rb +++ b/app/models/miq_server/worker_management/monitor/class_names.rb @@ -1,20 +1,13 @@ -require "workers/miq_worker_types" - module MiqServer::WorkerManagement::Monitor::ClassNames extend ActiveSupport::Concern - # This are loaded from `lib/workers/miq_worker_types.rb`, and are just a - # memory reference to them for compatability. - MONITOR_CLASS_NAMES = MIQ_WORKER_TYPES.keys - MONITOR_CLASS_NAMES_IN_KILL_ORDER = MIQ_WORKER_TYPES_IN_KILL_ORDER - module ClassMethods def monitor_class_names - MONITOR_CLASS_NAMES + MiqWorkerType.pluck(:worker_type) end def monitor_class_names_in_kill_order - MONITOR_CLASS_NAMES_IN_KILL_ORDER + MiqWorkerType.in_kill_order.pluck(:worker_type) end end end diff --git a/lib/generators/manageiq/provider/provider_generator.rb b/lib/generators/manageiq/provider/provider_generator.rb index 69bd7d427ec..c787d76f578 100644 --- a/lib/generators/manageiq/provider/provider_generator.rb +++ b/lib/generators/manageiq/provider/provider_generator.rb @@ -42,12 +42,6 @@ def create_dummy template "app/models/%plugin_path%/inventory/persister/cloud_manager.rb" template "app/models/%plugin_path%/inventory/persister.rb" template "app/models/%plugin_path%/cloud_manager.rb" - - inject_into_file Rails.root.join('lib/workers/miq_worker_types.rb'), <<~RB.indent(2), :after => "MIQ_WORKER_TYPES = {\n" - "#{class_name}::CloudManager::EventCatcher" => %i(manageiq_default), - "#{class_name}::CloudManager::MetricsCollectorWorker" => %i(manageiq_default), - "#{class_name}::CloudManager::RefreshWorker" => %i(manageiq_default), - RB end def create_vcr diff --git a/lib/workers/bin/run_single_worker.rb b/lib/workers/bin/run_single_worker.rb index 47d55f3bd6d..e5c54a61c38 100755 --- a/lib/workers/bin/run_single_worker.rb +++ b/lib/workers/bin/run_single_worker.rb @@ -60,27 +60,27 @@ def all_role_names opt_parser.parse! worker_class = ARGV[0] -require File.expand_path("../miq_worker_types", __dir__) +require File.expand_path("../../../config/environment", __dir__) if options[:list] - puts ::MIQ_WORKER_TYPES.keys + puts MiqWorkerType.pluck(:worker_type) exit end opt_parser.abort(opt_parser.help) unless worker_class -unless ::MIQ_WORKER_TYPES.keys.include?(worker_class) +worker_type = MiqWorkerType.find_by(:worker_type => worker_class) + +unless worker_type STDERR.puts "ERR: `#{worker_class}` WORKER CLASS NOT FOUND! Please run with `-l` to see possible worker class names." exit 1 end # Skip heartbeating with single worker ENV["DISABLE_MIQ_WORKER_HEARTBEAT"] ||= options[:heartbeat] ? nil : '1' -ENV["BUNDLER_GROUPS"] = MIQ_WORKER_TYPES[worker_class].join(',') +ENV["BUNDLER_GROUPS"] = worker_type.bundler_groups.join(',') options[:ems_id] ||= ENV["EMS_ID"] -require File.expand_path("../../../config/environment", __dir__) - if options[:roles].present? MiqServer.my_server.server_role_names += options[:roles] MiqServer.my_server.activate_roles(MiqServer.my_server.server_role_names) diff --git a/lib/workers/miq_worker_types.rb b/lib/workers/miq_worker_types.rb deleted file mode 100644 index 3c41149b548..00000000000 --- a/lib/workers/miq_worker_types.rb +++ /dev/null @@ -1,146 +0,0 @@ -# TODO: Consider putting this in a module namespace instead of poluting the -# global namespace - -MIQ_WORKER_TYPES = { - "ManageIQ::Providers::Amazon::AgentCoordinatorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Amazon::CloudManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Amazon::CloudManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Amazon::CloudManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Amazon::StorageManager::S3::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::AnsibleTower::AutomationManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::AnsibleTower::AutomationManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Azure::CloudManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Azure::CloudManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Azure::CloudManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::AzureStack::CloudManager::EventCatcher" => %i[manageiq_default], - "ManageIQ::Providers::AzureStack::CloudManager::RefreshWorker" => %i[manageiq_default], - "ManageIQ::Providers::AzureStack::NetworkManager::RefreshWorker" => %i[manageiq_default], - "ManageIQ::Providers::Foreman::ConfigurationManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Foreman::ProvisioningManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Google::CloudManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Google::CloudManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Google::CloudManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Google::NetworkManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Kubernetes::ContainerManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Kubernetes::ContainerManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Kubernetes::ContainerManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Kubernetes::MonitoringManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Kubevirt::InfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Lenovo::PhysicalInfraManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Lenovo::PhysicalInfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Microsoft::InfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Nuage::NetworkManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Nuage::NetworkManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Openshift::ContainerManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Openshift::ContainerManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openshift::ContainerManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openshift::MonitoringManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::CloudManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::CloudManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::CloudManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::InfraManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::InfraManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::InfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::NetworkManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::NetworkManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Openstack::StorageManager::CinderManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Redfish::PhysicalInfraManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Redfish::PhysicalInfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Redhat::InfraManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Redhat::InfraManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Redhat::InfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Redhat::NetworkManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Redhat::NetworkManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Redhat::NetworkManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Vmware::CloudManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Vmware::CloudManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Vmware::InfraManager::EventCatcher" => %i(manageiq_default), - "ManageIQ::Providers::Vmware::InfraManager::MetricsCollectorWorker" => %i(manageiq_default), - "ManageIQ::Providers::Vmware::InfraManager::RefreshWorker" => %i(manageiq_default), - "ManageIQ::Providers::Vmware::NetworkManager::RefreshWorker" => %i(manageiq_default), - "MiqCockpitWsWorker" => %i(manageiq_default), - "MiqEmsMetricsProcessorWorker" => %i(manageiq_default), - "MiqEmsRefreshCoreWorker" => %i(manageiq_default), - "MiqEventHandler" => %i(manageiq_default), - "MiqGenericWorker" => %i(manageiq_default), - "MiqPriorityWorker" => %i(manageiq_default), - "MiqReportingWorker" => %i(manageiq_default), - "MiqScheduleWorker" => %i(manageiq_default), - "MiqSmartProxyWorker" => %i(manageiq_default), - "MiqUiWorker" => %i(manageiq_default ui_dependencies graphql_api), - "MiqVimBrokerWorker" => %i(manageiq_default), - "MiqWebServiceWorker" => %i(manageiq_default graphql_api), - "MiqRemoteConsoleWorker" => %i(manageiq_default), -}.freeze - -MIQ_WORKER_TYPES_IN_KILL_ORDER = %w( - MiqEmsMetricsProcessorWorker - ManageIQ::Providers::Azure::CloudManager::MetricsCollectorWorker - ManageIQ::Providers::Amazon::CloudManager::MetricsCollectorWorker - ManageIQ::Providers::Redhat::InfraManager::MetricsCollectorWorker - ManageIQ::Providers::Kubernetes::ContainerManager::MetricsCollectorWorker - ManageIQ::Providers::Openshift::ContainerManager::MetricsCollectorWorker - ManageIQ::Providers::Google::CloudManager::MetricsCollectorWorker - ManageIQ::Providers::Vmware::InfraManager::MetricsCollectorWorker - ManageIQ::Providers::Openstack::CloudManager::MetricsCollectorWorker - ManageIQ::Providers::Openstack::NetworkManager::MetricsCollectorWorker - ManageIQ::Providers::Redhat::NetworkManager::MetricsCollectorWorker - ManageIQ::Providers::Openstack::InfraManager::MetricsCollectorWorker - MiqReportingWorker - MiqSmartProxyWorker - MiqGenericWorker - MiqEventHandler - ManageIQ::Providers::Azure::CloudManager::RefreshWorker - ManageIQ::Providers::AzureStack::CloudManager::RefreshWorker - ManageIQ::Providers::AzureStack::NetworkManager::RefreshWorker - ManageIQ::Providers::Amazon::CloudManager::RefreshWorker - ManageIQ::Providers::Amazon::StorageManager::S3::RefreshWorker - ManageIQ::Providers::Amazon::AgentCoordinatorWorker - ManageIQ::Providers::Google::CloudManager::RefreshWorker - ManageIQ::Providers::Google::NetworkManager::RefreshWorker - ManageIQ::Providers::AnsibleTower::AutomationManager::RefreshWorker - ManageIQ::Providers::Foreman::ConfigurationManager::RefreshWorker - ManageIQ::Providers::Foreman::ProvisioningManager::RefreshWorker - ManageIQ::Providers::Kubernetes::ContainerManager::RefreshWorker - ManageIQ::Providers::Lenovo::PhysicalInfraManager::RefreshWorker - ManageIQ::Providers::Openshift::ContainerManager::RefreshWorker - ManageIQ::Providers::Microsoft::InfraManager::RefreshWorker - ManageIQ::Providers::Redfish::PhysicalInfraManager::RefreshWorker - ManageIQ::Providers::Redhat::InfraManager::RefreshWorker - ManageIQ::Providers::Openstack::CloudManager::RefreshWorker - ManageIQ::Providers::Redhat::NetworkManager::RefreshWorker - ManageIQ::Providers::Openstack::InfraManager::RefreshWorker - ManageIQ::Providers::Vmware::CloudManager::RefreshWorker - ManageIQ::Providers::Vmware::NetworkManager::RefreshWorker - ManageIQ::Providers::Vmware::InfraManager::RefreshWorker - ManageIQ::Providers::Nuage::NetworkManager::RefreshWorker - ManageIQ::Providers::Nuage::NetworkManager::EventCatcher - MiqScheduleWorker - MiqPriorityWorker - MiqWebServiceWorker - MiqEmsRefreshCoreWorker - MiqVimBrokerWorker - ManageIQ::Providers::Vmware::CloudManager::EventCatcher - ManageIQ::Providers::Vmware::InfraManager::EventCatcher - ManageIQ::Providers::Redhat::InfraManager::EventCatcher - ManageIQ::Providers::Openstack::CloudManager::EventCatcher - ManageIQ::Providers::Openstack::NetworkManager::EventCatcher - ManageIQ::Providers::Openstack::StorageManager::CinderManager::EventCatcher - ManageIQ::Providers::Redfish::PhysicalInfraManager::EventCatcher - ManageIQ::Providers::Redhat::NetworkManager::EventCatcher - ManageIQ::Providers::Openstack::InfraManager::EventCatcher - ManageIQ::Providers::Amazon::CloudManager::EventCatcher - ManageIQ::Providers::Azure::CloudManager::EventCatcher - ManageIQ::Providers::AzureStack::CloudManager::EventCatcher - ManageIQ::Providers::AnsibleTower::AutomationManager::EventCatcher - ManageIQ::Providers::Google::CloudManager::EventCatcher - ManageIQ::Providers::Kubernetes::ContainerManager::EventCatcher - ManageIQ::Providers::Kubernetes::MonitoringManager::EventCatcher - ManageIQ::Providers::Kubevirt::InfraManager::RefreshWorker - ManageIQ::Providers::Openshift::ContainerManager::EventCatcher - ManageIQ::Providers::Openshift::MonitoringManager::EventCatcher - ManageIQ::Providers::Lenovo::PhysicalInfraManager::EventCatcher - MiqUiWorker - MiqRemoteConsoleWorker - MiqCockpitWsWorker -).freeze diff --git a/spec/lib/workers/miq_worker_types_spec.rb b/spec/lib/workers/miq_worker_types_spec.rb deleted file mode 100644 index 30d5b9dba8b..00000000000 --- a/spec/lib/workers/miq_worker_types_spec.rb +++ /dev/null @@ -1,14 +0,0 @@ -describe "worker types lists" do - let(:all_defined_workers) do - exceptions = %w[ManageIQ::Providers::BaseManager::OperationsWorker] - MiqWorker.descendants.select { |w| w.subclasses.empty? }.map(&:name) - exceptions - end - - it "should include all the worker types" do - expect(MIQ_WORKER_TYPES.keys).to match_array(all_defined_workers) - end - - it "kill list should match type list" do - expect(MIQ_WORKER_TYPES_IN_KILL_ORDER).to match_array(all_defined_workers) - end -end diff --git a/spec/models/message_timeout_handling_spec.rb b/spec/models/message_timeout_handling_spec.rb index 3c0452235ee..91ac6a1da06 100644 --- a/spec/models/message_timeout_handling_spec.rb +++ b/spec/models/message_timeout_handling_spec.rb @@ -46,6 +46,7 @@ :handler_id => @worker.id, :handler_type => 'MiqWorker' ) + MiqWorkerType.seed @miq_server.sync_child_worker_settings end diff --git a/spec/models/miq_server/worker_monitor_spec.rb b/spec/models/miq_server/worker_monitor_spec.rb index 8521559ed9a..923f1021179 100644 --- a/spec/models/miq_server/worker_monitor_spec.rb +++ b/spec/models/miq_server/worker_monitor_spec.rb @@ -5,6 +5,7 @@ allow_any_instance_of(MiqServer).to receive(:get_time_threshold).and_return(120) allow_any_instance_of(MiqServer).to receive(:get_memory_threshold).and_return(100.megabytes) + MiqWorkerType.seed @miq_server = EvmSpecHelper.local_miq_server end diff --git a/spec/models/miq_worker/container_common_spec.rb b/spec/models/miq_worker/container_common_spec.rb index c35dfae271d..a9f1721fbd2 100644 --- a/spec/models/miq_worker/container_common_spec.rb +++ b/spec/models/miq_worker/container_common_spec.rb @@ -16,7 +16,8 @@ it "no worker deployment names are over 60 characters" do # OpenShift does not allow deployment names over 63 characters # We also want to leave some for the ems_id so we compare against 60 to be safe - MIQ_WORKER_TYPES_IN_KILL_ORDER.each do |klass| + MiqWorkerType.seed + MiqWorkerType.pluck(:worker_type).each do |klass| expect(klass.constantize.new.worker_deployment_name.length).to be <= 60 end end From c2e03208bd5702533cf2df3c6fa4219a761afee0 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Wed, 20 Nov 2019 15:40:07 -0500 Subject: [PATCH 3/6] Define and use constants for worker kill priorities --- .../providers/base_manager/event_catcher.rb | 2 +- .../base_manager/metrics_collector_worker.rb | 2 +- .../providers/base_manager/refresh_worker.rb | 2 +- app/models/miq_cockpit_ws_worker.rb | 2 +- app/models/miq_ems_metrics_processor_worker.rb | 2 +- app/models/miq_event_handler.rb | 2 +- app/models/miq_generic_worker.rb | 2 +- app/models/miq_priority_worker.rb | 2 +- app/models/miq_remote_console_worker.rb | 2 +- app/models/miq_reporting_worker.rb | 2 +- app/models/miq_schedule_worker.rb | 2 +- app/models/miq_smart_proxy_worker.rb | 2 +- app/models/miq_ui_worker.rb | 2 +- app/models/miq_web_service_worker.rb | 2 +- app/models/miq_worker_type.rb | 17 +++++++++++++++++ 15 files changed, 31 insertions(+), 14 deletions(-) diff --git a/app/models/manageiq/providers/base_manager/event_catcher.rb b/app/models/manageiq/providers/base_manager/event_catcher.rb index 8d9aaea7dd3..ffe35300eae 100644 --- a/app/models/manageiq/providers/base_manager/event_catcher.rb +++ b/app/models/manageiq/providers/base_manager/event_catcher.rb @@ -17,6 +17,6 @@ def friendly_name end def self.kill_priority - 130 + MiqWorkerType::KILL_PRIORITY_EVENT_CATCHERS end end diff --git a/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb b/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb index 44e793a7a37..d6b26840d44 100644 --- a/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb +++ b/app/models/manageiq/providers/base_manager/metrics_collector_worker.rb @@ -16,6 +16,6 @@ def self.normalized_type end def self.kill_priority - 20 + MiqWorkerType::KILL_PRIORITY_METRICS_COLLECTOR_WORKERS end end diff --git a/app/models/manageiq/providers/base_manager/refresh_worker.rb b/app/models/manageiq/providers/base_manager/refresh_worker.rb index 5b8e4860f3e..5752be408c5 100644 --- a/app/models/manageiq/providers/base_manager/refresh_worker.rb +++ b/app/models/manageiq/providers/base_manager/refresh_worker.rb @@ -23,6 +23,6 @@ def self.normalized_type end def self.kill_priority - 70 + MiqWorkerType::KILL_PRIORITY_REFRESH_WORKERS end end diff --git a/app/models/miq_cockpit_ws_worker.rb b/app/models/miq_cockpit_ws_worker.rb index f352d4d200f..357ceebabae 100644 --- a/app/models/miq_cockpit_ws_worker.rb +++ b/app/models/miq_cockpit_ws_worker.rb @@ -10,7 +10,7 @@ def friendly_name end def self.kill_priority - 160 + MiqWorkerType::KILL_PRIORITY_COCKPIT_WS_WORKERS end def self.can_start_cockpit_ws? diff --git a/app/models/miq_ems_metrics_processor_worker.rb b/app/models/miq_ems_metrics_processor_worker.rb index 6720520080d..dbaecbe6708 100644 --- a/app/models/miq_ems_metrics_processor_worker.rb +++ b/app/models/miq_ems_metrics_processor_worker.rb @@ -15,6 +15,6 @@ def self.supports_container? end def self.kill_priority - 10 + MiqWorkerType::KILL_PRIORITY_METRICS_PROCESSOR_WORKERS end end diff --git a/app/models/miq_event_handler.rb b/app/models/miq_event_handler.rb index be913b315e5..5f36008f4c3 100644 --- a/app/models/miq_event_handler.rb +++ b/app/models/miq_event_handler.rb @@ -11,6 +11,6 @@ def self.supports_container? end def self.kill_priority - 60 + MiqWorkerType::KILL_PRIORITY_EVENT_HANDLERS end end diff --git a/app/models/miq_generic_worker.rb b/app/models/miq_generic_worker.rb index cab9878af89..13f6f046ece 100644 --- a/app/models/miq_generic_worker.rb +++ b/app/models/miq_generic_worker.rb @@ -12,6 +12,6 @@ def self.supports_container? end def self.kill_priority - 50 + MiqWorkerType::KILL_PRIORITY_GENERIC_WORKERS end end diff --git a/app/models/miq_priority_worker.rb b/app/models/miq_priority_worker.rb index 60882a52a0f..8ade2bd3db9 100644 --- a/app/models/miq_priority_worker.rb +++ b/app/models/miq_priority_worker.rb @@ -14,6 +14,6 @@ def self.supports_container? end def self.kill_priority - 90 + MiqWorkerType::KILL_PRIORITY_PRIORITY_WORKERS end end diff --git a/app/models/miq_remote_console_worker.rb b/app/models/miq_remote_console_worker.rb index b058cb87edc..1494b4b036f 100644 --- a/app/models/miq_remote_console_worker.rb +++ b/app/models/miq_remote_console_worker.rb @@ -18,6 +18,6 @@ def self.supports_container? end def self.kill_priority - 150 + MiqWorkerType::KILL_PRIORITY_REMOTE_CONSOLE_WORKERS end end diff --git a/app/models/miq_reporting_worker.rb b/app/models/miq_reporting_worker.rb index a4b3b632d89..8ca6b05f8c3 100644 --- a/app/models/miq_reporting_worker.rb +++ b/app/models/miq_reporting_worker.rb @@ -11,6 +11,6 @@ def self.supports_container? end def self.kill_priority - 30 + MiqWorkerType::KILL_PRIORITY_REPORTING_WORKERS end end diff --git a/app/models/miq_schedule_worker.rb b/app/models/miq_schedule_worker.rb index 2008d09728e..fc7aab60eaf 100644 --- a/app/models/miq_schedule_worker.rb +++ b/app/models/miq_schedule_worker.rb @@ -15,6 +15,6 @@ def self.supports_container? end def self.kill_priority - 80 + MiqWorkerType::KILL_PRIORITY_SCHEDULE_WORKERS end end diff --git a/app/models/miq_smart_proxy_worker.rb b/app/models/miq_smart_proxy_worker.rb index 41a41afbf62..cac00ca3279 100644 --- a/app/models/miq_smart_proxy_worker.rb +++ b/app/models/miq_smart_proxy_worker.rb @@ -5,6 +5,6 @@ class MiqSmartProxyWorker < MiqQueueWorkerBase self.default_queue_name = "smartproxy" def self.kill_priority - 40 + MiqWorkerType::KILL_PRIORITY_SMART_PROXY_WORKERS end end diff --git a/app/models/miq_ui_worker.rb b/app/models/miq_ui_worker.rb index d3641a00568..6e92d91e587 100644 --- a/app/models/miq_ui_worker.rb +++ b/app/models/miq_ui_worker.rb @@ -31,7 +31,7 @@ def self.bundler_groups end def self.kill_priority - 140 + MiqWorkerType::KILL_PRIORITY_UI_WORKERS end def container_port diff --git a/app/models/miq_web_service_worker.rb b/app/models/miq_web_service_worker.rb index d492489cfb6..c2d19ba61c8 100644 --- a/app/models/miq_web_service_worker.rb +++ b/app/models/miq_web_service_worker.rb @@ -21,6 +21,6 @@ def self.bundler_groups end def self.kill_priority - 100 + MiqWorkerType::KILL_PRIORITY_WEB_SERVICE_WORKERS end end diff --git a/app/models/miq_worker_type.rb b/app/models/miq_worker_type.rb index 473ab55cee5..148531600e4 100644 --- a/app/models/miq_worker_type.rb +++ b/app/models/miq_worker_type.rb @@ -1,6 +1,23 @@ class MiqWorkerType < ApplicationRecord EXCLUDED_CLASS_NAMES = %w[ManageIQ::Providers::BaseManager::OperationsWorker].freeze + KILL_PRIORITY_METRICS_PROCESSOR_WORKERS = 10 + KILL_PRIORITY_METRICS_COLLECTOR_WORKERS = 20 + KILL_PRIORITY_REPORTING_WORKERS = 30 + KILL_PRIORITY_SMART_PROXY_WORKERS = 40 + KILL_PRIORITY_GENERIC_WORKERS = 50 + KILL_PRIORITY_EVENT_HANDLERS = 60 + KILL_PRIORITY_REFRESH_WORKERS = 70 + KILL_PRIORITY_SCHEDULE_WORKERS = 80 + KILL_PRIORITY_PRIORITY_WORKERS = 90 + KILL_PRIORITY_WEB_SERVICE_WORKERS = 100 + KILL_PRIORITY_REFRESH_CORE_WORKERS = 110 + KILL_PRIORITY_VIM_BROKER_WORKERS = 120 + KILL_PRIORITY_EVENT_CATCHERS = 130 + KILL_PRIORITY_UI_WORKERS = 140 + KILL_PRIORITY_REMOTE_CONSOLE_WORKERS = 150 + KILL_PRIORITY_COCKPIT_WS_WORKERS = 160 + scope :in_kill_order, -> { order(:kill_priority => :asc) } def self.seed From b92954a34b3aa56f865acc1dbb9bfea6b66355eb Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Wed, 20 Nov 2019 15:49:42 -0500 Subject: [PATCH 4/6] Remove old worker types in MiqWorkerType.seed --- app/models/miq_worker_type.rb | 7 ++++++- spec/models/miq_worker_type_spec.rb | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/models/miq_worker_type.rb b/app/models/miq_worker_type.rb index 148531600e4..953ade60b81 100644 --- a/app/models/miq_worker_type.rb +++ b/app/models/miq_worker_type.rb @@ -22,12 +22,13 @@ class MiqWorkerType < ApplicationRecord def self.seed transaction do + clean_worker_types classes_for_seed.each { |klass| seed_worker(klass) } end end private_class_method def self.classes_for_seed - MiqWorker.descendants.select { |w| w.subclasses.empty? } - EXCLUDED_CLASS_NAMES.map(&:constantize) + @classes_for_seed ||= MiqWorker.descendants.select { |w| w.subclasses.empty? } - EXCLUDED_CLASS_NAMES.map(&:constantize) end private_class_method def self.seed_worker(klass) @@ -38,4 +39,8 @@ def self.seed :kill_priority => klass.kill_priority ) end + + private_class_method def self.clean_worker_types + where.not(:worker_type => classes_for_seed.map(&:to_s)).destroy_all + end end diff --git a/spec/models/miq_worker_type_spec.rb b/spec/models/miq_worker_type_spec.rb index 56cd8262698..43313336bc8 100644 --- a/spec/models/miq_worker_type_spec.rb +++ b/spec/models/miq_worker_type_spec.rb @@ -17,5 +17,18 @@ expect(ui_worker.bundler_groups).to match_array(MiqUiWorker.bundler_groups) expect(ui_worker.kill_priority).to eq(MiqUiWorker.kill_priority) end + + it "removes worker records which no longer exist" do + old_worker_name = "MiqReplicationWorker" + described_class.create!( + :worker_type => old_worker_name, + :bundler_groups => [], + :kill_priority => 123 + ) + + described_class.seed + + expect(described_class.find_by(:worker_type => old_worker_name)).to be_nil + end end end From 3fbf68af428b835a1615c4468d5a69011eb136cc Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Wed, 20 Nov 2019 16:04:58 -0500 Subject: [PATCH 5/6] Remove MiqServer::WorkerManagement::Monitor::ClassNames in favor of using MiqWorkerType directly --- app/models/miq_server/worker_management/monitor.rb | 7 +++---- .../worker_management/monitor/class_names.rb | 13 ------------- .../miq_server/worker_management/monitor/quiesce.rb | 2 +- .../worker_management/monitor/settings.rb | 2 +- .../miq_server/worker_management/monitor/start.rb | 2 +- app/models/miq_worker_type.rb | 8 ++++++++ .../miq_server/worker_management/monitor_spec.rb | 4 ++-- spec/models/miq_server/worker_monitor_spec.rb | 6 +++--- 8 files changed, 19 insertions(+), 25 deletions(-) delete mode 100644 app/models/miq_server/worker_management/monitor/class_names.rb diff --git a/app/models/miq_server/worker_management/monitor.rb b/app/models/miq_server/worker_management/monitor.rb index e24ce11189b..d8423ae936c 100644 --- a/app/models/miq_server/worker_management/monitor.rb +++ b/app/models/miq_server/worker_management/monitor.rb @@ -1,7 +1,6 @@ module MiqServer::WorkerManagement::Monitor extend ActiveSupport::Concern - include_concern 'ClassNames' include_concern 'Kill' include_concern 'Quiesce' include_concern 'Reason' @@ -25,7 +24,7 @@ def monitor_workers processed_worker_ids = [] - self.class.monitor_class_names.each do |class_name| + MiqWorkerType.worker_class_names.each do |class_name| processed_worker_ids += check_not_responding(class_name) processed_worker_ids += check_pending_stop(class_name) processed_worker_ids += clean_worker_records(class_name) @@ -47,7 +46,7 @@ def worker_not_responding(w) def sync_workers result = {} - self.class.monitor_class_names.each do |class_name| + MiqWorkerType.worker_class_names.each do |class_name| begin c = class_name.constantize raise NameError, "Constant problem: expected: #{class_name}, constantized: #{c.name}" unless c.name == class_name @@ -115,7 +114,7 @@ def monitor_reason_not_responding?(w) end def do_system_limit_exceeded - self.class.monitor_class_names_in_kill_order.each do |class_name| + MiqWorkerType.worker_class_names_in_kill_order.each do |class_name| workers = class_name.constantize.find_current.to_a next if workers.empty? diff --git a/app/models/miq_server/worker_management/monitor/class_names.rb b/app/models/miq_server/worker_management/monitor/class_names.rb deleted file mode 100644 index 4de09fa0de7..00000000000 --- a/app/models/miq_server/worker_management/monitor/class_names.rb +++ /dev/null @@ -1,13 +0,0 @@ -module MiqServer::WorkerManagement::Monitor::ClassNames - extend ActiveSupport::Concern - - module ClassMethods - def monitor_class_names - MiqWorkerType.pluck(:worker_type) - end - - def monitor_class_names_in_kill_order - MiqWorkerType.in_kill_order.pluck(:worker_type) - end - end -end diff --git a/app/models/miq_server/worker_management/monitor/quiesce.rb b/app/models/miq_server/worker_management/monitor/quiesce.rb index dc47e23615d..80f4c7ffc2c 100644 --- a/app/models/miq_server/worker_management/monitor/quiesce.rb +++ b/app/models/miq_server/worker_management/monitor/quiesce.rb @@ -5,7 +5,7 @@ def workers_quiesced? # do a subset of the monitor_workers loop to allow for graceful exit heartbeat - self.class.monitor_class_names.each do |class_name| + MiqWorkerType.worker_class_names.each do |class_name| check_not_responding(class_name) check_pending_stop(class_name) clean_worker_records(class_name) diff --git a/app/models/miq_server/worker_management/monitor/settings.rb b/app/models/miq_server/worker_management/monitor/settings.rb index 1b61c740fc4..77f32d88524 100644 --- a/app/models/miq_server/worker_management/monitor/settings.rb +++ b/app/models/miq_server/worker_management/monitor/settings.rb @@ -8,7 +8,7 @@ module MiqServer::WorkerManagement::Monitor::Settings def sync_child_worker_settings @child_worker_settings = {} - self.class.monitor_class_names.each do |class_name| + MiqWorkerType.worker_class_names.each do |class_name| c = class_name.constantize @child_worker_settings[c.settings_name] = c.worker_settings end diff --git a/app/models/miq_server/worker_management/monitor/start.rb b/app/models/miq_server/worker_management/monitor/start.rb index 79b8d09fc7f..4ce82957564 100644 --- a/app/models/miq_server/worker_management/monitor/start.rb +++ b/app/models/miq_server/worker_management/monitor/start.rb @@ -10,7 +10,7 @@ def wait_for_started_workers entered_wait_for_started_loop = Time.now.utc wait_for_started_timeout = @worker_monitor_settings[:wait_for_started_timeout] || 10.minutes loop do - starting = MiqWorker.find_starting.find_all { |w| self.class.monitor_class_names.include?(w.class.name) } + starting = MiqWorker.find_starting.find_all { |w| MiqWorkerType.worker_class_names.include?(w.class.name) } if starting.empty? _log.info("All workers have been started") break diff --git a/app/models/miq_worker_type.rb b/app/models/miq_worker_type.rb index 953ade60b81..61e31cf30b6 100644 --- a/app/models/miq_worker_type.rb +++ b/app/models/miq_worker_type.rb @@ -27,6 +27,14 @@ def self.seed end end + def self.worker_class_names + pluck(:worker_type) + end + + def self.worker_class_names_in_kill_order + in_kill_order.pluck(:worker_type) + end + private_class_method def self.classes_for_seed @classes_for_seed ||= MiqWorker.descendants.select { |w| w.subclasses.empty? } - EXCLUDED_CLASS_NAMES.map(&:constantize) end diff --git a/spec/models/miq_server/worker_management/monitor_spec.rb b/spec/models/miq_server/worker_management/monitor_spec.rb index e09cd0e2368..8e1472d5ff1 100644 --- a/spec/models/miq_server/worker_management/monitor_spec.rb +++ b/spec/models/miq_server/worker_management/monitor_spec.rb @@ -40,7 +40,7 @@ # We'll try to constantize a non-existing EventCatcher class in an existing namespace, # which incorrectly resolves to the base manager event catcher. - allow(MiqServer).to receive(:monitor_class_names).and_return(%w[ManageIQ::Providers::Foreman::ProvisioningManager::EventCatcher MiqGenericWorker]) + allow(MiqWorkerType).to receive(:worker_class_names).and_return(%w[ManageIQ::Providers::Foreman::ProvisioningManager::EventCatcher MiqGenericWorker]) expect(ManageIQ::Providers::BaseManager::EventCatcher).not_to receive(:sync_workers) expect(MiqGenericWorker).to receive(:sync_workers).and_return(:adds => [111]) @@ -48,7 +48,7 @@ end it "rescues exceptions and moves on" do - allow(MiqServer).to receive(:monitor_class_names).and_return(%w(MiqGenericWorker MiqPriorityWorker)) + allow(MiqWorkerType).to receive(:worker_class_names).and_return(%w(MiqGenericWorker MiqPriorityWorker)) allow(MiqGenericWorker).to receive(:sync_workers).and_raise expect(MiqPriorityWorker).to receive(:sync_workers).and_return(:adds => [123]) expect(server.sync_workers).to eq("MiqPriorityWorker"=>{:adds=>[123]}) diff --git a/spec/models/miq_server/worker_monitor_spec.rb b/spec/models/miq_server/worker_monitor_spec.rb index 923f1021179..2ab822ea795 100644 --- a/spec/models/miq_server/worker_monitor_spec.rb +++ b/spec/models/miq_server/worker_monitor_spec.rb @@ -219,7 +219,7 @@ it "should delete worker row after clean_worker_records" do expect(MiqWorker.count).to eq(1) - MiqServer.monitor_class_names.each { |c| @miq_server.clean_worker_records(c) } + MiqWorkerType.worker_class_names.each { |c| @miq_server.clean_worker_records(c) } expect(MiqWorker.count).to eq(0) end end @@ -231,7 +231,7 @@ it "should delete worker row after clean_worker_records" do expect(MiqWorker.count).to eq(1) - MiqServer.monitor_class_names.each { |c| @miq_server.clean_worker_records(c) } + MiqWorkerType.worker_class_names.each { |c| @miq_server.clean_worker_records(c) } expect(MiqWorker.count).to eq(0) end end @@ -243,7 +243,7 @@ it "should delete worker row after clean_worker_records" do expect(MiqWorker.count).to eq(1) - MiqServer.monitor_class_names.each { |c| @miq_server.clean_worker_records(c) } + MiqWorkerType.worker_class_names.each { |c| @miq_server.clean_worker_records(c) } expect(MiqWorker.count).to eq(0) end end From 5c9469ffe7a1168474bda634e02ed0f0348f8115 Mon Sep 17 00:00:00 2001 From: Nick Carboni Date: Wed, 20 Nov 2019 17:56:24 -0500 Subject: [PATCH 6/6] Move bundler groups out of run_single_worker.rb The caller will set the bundler groups outside of the worker entrypoint. This also prevents the chicken-and-egg problem of needing to get the bundler groups from the database before loading the environment --- app/models/miq_worker.rb | 6 +++++- app/models/miq_worker/container_common.rb | 1 + app/models/miq_worker/systemd_common.rb | 1 + lib/workers/bin/run_single_worker.rb | 5 +---- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/app/models/miq_worker.rb b/app/models/miq_worker.rb index 9e3190e4b39..92e6c1ae7c3 100644 --- a/app/models/miq_worker.rb +++ b/app/models/miq_worker.rb @@ -435,7 +435,11 @@ def command_line end def start_runner_via_spawn - pid = Kernel.spawn(command_line, [:out, :err] => [Rails.root.join("log", "evm.log"), "a"]) + pid = Kernel.spawn( + {"BUNDLER_GROUPS" => self.class.bundler_groups.join(",")}, + command_line, + [:out, :err] => [Rails.root.join("log", "evm.log"), "a"] + ) Process.detach(pid) pid end diff --git a/app/models/miq_worker/container_common.rb b/app/models/miq_worker/container_common.rb index 8ed71b99e75..0c57b7f9ae6 100644 --- a/app/models/miq_worker/container_common.rb +++ b/app/models/miq_worker/container_common.rb @@ -11,6 +11,7 @@ def configure_worker_deployment(definition, replicas = 0) container = definition[:spec][:template][:spec][:containers].first container[:image] = "#{container_image_namespace}/#{container_image_name}:#{container_image_tag}" container[:env] << {:name => "WORKER_CLASS_NAME", :value => self.class.name} + container[:env] << {:name => "BUNDLER_GROUPS", :value => self.class.bundler_groups.join(",")} end def scale_deployment diff --git a/app/models/miq_worker/systemd_common.rb b/app/models/miq_worker/systemd_common.rb index 1f8d11caee9..6ddf861fa25 100644 --- a/app/models/miq_worker/systemd_common.rb +++ b/app/models/miq_worker/systemd_common.rb @@ -67,6 +67,7 @@ def unit_file WantedBy=#{target_file_name} [Service] WorkingDirectory=#{working_directory} + Environment=BUNDLER_GROUPS=#{bundler_groups.join(",")} ExecStart=/bin/bash -lc '#{exec_start}' Restart=always Slice=#{slice_name} diff --git a/lib/workers/bin/run_single_worker.rb b/lib/workers/bin/run_single_worker.rb index e5c54a61c38..fae9eeb31ed 100755 --- a/lib/workers/bin/run_single_worker.rb +++ b/lib/workers/bin/run_single_worker.rb @@ -68,16 +68,13 @@ def all_role_names end opt_parser.abort(opt_parser.help) unless worker_class -worker_type = MiqWorkerType.find_by(:worker_type => worker_class) - -unless worker_type +unless MiqWorkerType.find_by(:worker_type => worker_class) STDERR.puts "ERR: `#{worker_class}` WORKER CLASS NOT FOUND! Please run with `-l` to see possible worker class names." exit 1 end # Skip heartbeating with single worker ENV["DISABLE_MIQ_WORKER_HEARTBEAT"] ||= options[:heartbeat] ? nil : '1' -ENV["BUNDLER_GROUPS"] = worker_type.bundler_groups.join(',') options[:ems_id] ||= ENV["EMS_ID"]