notify users of killing workers when exceed memory

- Provide more complete information for worker out of memory errors
- Provide notification messages for out of memory errors
- Introduce ability to disable notification
- Include notification.message in alert emails

https://bugzilla.redhat.com/show_bug.cgi?id=1535177

app/models/miq_action.rb

@@ -309,6 +309,7 @@ def action_email(action, rec, inputs)
  :header => "Alert Triggered",
  :policy_detail => "Alert '#{inputs[:policy].description}', triggered",
  :event_description => inputs[:event].description,
+ :event_details => Notification.notification_text(inputs[:triggering_type], inputs[:triggering_data]),
  :entity_type => rec.class.to_s,
  :entity_name => rec.name
  }

app/models/miq_event.rb

@@ -40,7 +40,7 @@ def self.raise_evm_event(target, raw_event, inputs = {}, options = {})
  return
  end
 
- event_obj = build_evm_event(event, target)
+ event_obj = build_evm_event(event, target, inputs[:full_data])
  inputs.merge!('MiqEvent::miq_event' => event_obj.id, :miq_event_id => event_obj.id)
  inputs.merge!('EventStream::event_stream' => event_obj.id, :event_stream_id => event_obj.id)
 
@@ -61,6 +61,8 @@ def process_evm_event(inputs = {})
  results = {}
  inputs[:type] ||= target.class.name
  inputs[:source_event] = source_event if source_event
+ inputs[:triggering_type] = event_type
+ inputs[:triggering_data] = full_data
 
  _log.info("Event Raised [#{event_type}]")
  begin
@@ -79,10 +81,11 @@ def process_evm_event(inputs = {})
  results
  end
 
- def self.build_evm_event(event, target)
+ def self.build_evm_event(event, target, full_data = nil)
  options = {
  :event_type => event,
  :target => target,
+ :full_data => full_data,
  :source => 'POLICY',
  :timestamp => Time.now.utc
  }

app/models/miq_server/worker_management/monitor/validation.rb

@@ -25,7 +25,16 @@ def validate_worker(w)
  if MiqWorker::STATUSES_CURRENT.include?(w.status) && usage_exceeds_threshold?(usage, memory_threshold)
  msg = "#{w.format_full_log_msg} process memory usage [#{usage}] exceeded limit [#{memory_threshold}], requesting worker to exit"
  _log.warn(msg)
- MiqEvent.raise_evm_event_queue(w.miq_server, "evm_worker_memory_exceeded", :event_details => msg, :type => w.class.name)
+ helper = ApplicationController.helpers
+ full_data = {
+ :name => w.type,
+ :memory_usage => helper.number_to_human_size(usage),
+ :memory_threshold => helper.number_to_human_size(memory_threshold),
+ }
+ MiqEvent.raise_evm_event_queue(w.miq_server, "evm_worker_memory_exceeded",
+ :event_details => msg,
+ :type => w.class.name,
+ :full_data => full_data)
  restart_worker(w)
  return false
  end

app/models/notification.rb

@@ -25,7 +25,11 @@ def type=(typ)
  def self.emit_for_event(event)
  return unless NotificationType.names.include?(event.event_type)
  type = NotificationType.find_by(:name => event.event_type)
- Notification.create(:notification_type => type, :subject => event.target)
+ return if type.none?
+ Notification.create(:notification_type => type,
+ :options => event.full_data,
+ :subject_id => event.target_id,
+ :subject_type => event.target_type)
  end
 
  def to_h
@@ -41,6 +45,12 @@ def seen_by_all_recipients?
  notification_recipients.unseen.empty?
  end
 
+ def self.notification_text(event_type, full_data)
+ return unless NotificationType.names.include?(event_type) && full_data
+ type = NotificationType.find_by(:name => event_type)
+ type.message % full_data
+ end
+
  private
 
  def emit_message

app/models/notification_type.rb

@@ -4,6 +4,8 @@ class NotificationType < ApplicationRecord
  AUDIENCE_TENANT = 'tenant'.freeze
  AUDIENCE_GLOBAL = 'global'.freeze
  AUDIENCE_SUPERADMIN = 'superadmin'.freeze
+ # don't send out notifications, but keep the template around
+ AUDIENCE_NONE = 'none'.freeze
  has_many :notifications
  validates :message, :presence => true
  validates :level, :inclusion => { :in => %w(success error warning info) }
@@ -27,9 +29,17 @@ def subscriber_ids(subject, initiator)
  end.try(:user_ids)
  when AUDIENCE_SUPERADMIN
  User.superadmins.pluck(:id)
+ when AUDIENCE_NONE
+ []
  end
  end
 
+ # this disables notifications, but allows the notification to still exist
+ # this notification template can be used for emails
+ def none?
+ audience == AUDIENCE_NONE
+ end
+
  def self.names
  @names ||= Set.new(pluck(:name))
  end

db/fixtures/notification_types.yml

@@ -264,3 +264,8 @@
  :expires_in: 24.hours
  :level: :error
  :audience: superadmin
+- :name: evm_worker_memory_exceeded
+ :message: 'Killing worker %{name} due to excessive memory usage. %{memory_usage} used memory exceeds limit of %{memory_threshold}.'
+ :expires_in: 24.hours
+ :level: :error
+ :audience: none

spec/models/miq_action_spec.rb

@@ -420,6 +420,52 @@ def stub_csv(data)
  expect(MiqQueue).to receive(:put).with(q_options).once
  action.action_email(action, nil, inputs)
  end
+
+ context 'when alerting' do
+ before do
+ NotificationType.instance_variable_set(:@names, nil)
+ NotificationType.seed if NotificationType.all.empty?
+ end
+
+ let(:description) { 'my alert' }
+ let(:alert) { MiqAlert.new(:description => description) }
+ let(:event) { MiqEventDefinition.new(:name => "AlertEvent", :description => "Alert condition met") }
+ let(:vm) { Vm.new(:name => "vm1") }
+ let(:inputs) do
+ {
+ :policy => alert,
+ :event => event,
+ :synchronous => false,
+ :results => true,
+ :sequence => 1,
+ :triggering_type => "vm_retired",
+ :triggering_data => {:subject => 'vm1'},
+ }
+ end
+
+ let(:args) do
+ [{
+ :to => nil,
+ :from => "cfadmin@cfserver.com",
+ :subject => "Alert Triggered: my alert, for (VM) vm1",
+ :miq_action_hash => {
+ :header => "Alert Triggered",
+ :policy_detail => "Alert 'my alert', triggered",
+ :event_description => "Alert condition met",
+ :event_details => "Virtual Machine vm1 has been retired.",
+ :entity_type => "Vm",
+ :entity_name => "vm1",
+ }
+ }]
+ end
+
+ it 'generates an MiqAlert email' do
+ q_options[:args] = args
+
+ expect(MiqQueue).to receive(:put).with(q_options).once
+ action.action_email(action, vm, inputs)
+ end
+ end
  end
  end
 

spec/models/miq_event_spec.rb

@@ -129,11 +129,10 @@
  event = 'vm_start'
  FactoryGirl.create(:miq_event_definition, :name => event)
  FactoryGirl.create(:miq_event, :event_type => event, :target => @cluster)
- target_class = @cluster.class.name
-
- expect(MiqPolicy).to receive(:enforce_policy).with(@cluster, event, :type => target_class)
- expect(MiqAlert).to receive(:evaluate_alerts).with(@cluster, event, :type => target_class)
- expect(MiqEvent).to receive(:raise_event_for_children).with(@cluster, event, :type => target_class)
+ inputs = {:type => @cluster.class.name, :triggering_type => event, :triggering_data => nil}
+ expect(MiqPolicy).to receive(:enforce_policy).with(@cluster, event, inputs)
+ expect(MiqAlert).to receive(:evaluate_alerts).with(@cluster, event, inputs)
+ expect(MiqEvent).to receive(:raise_event_for_children).with(@cluster, event, inputs)
 
  results = MiqEvent.first.process_evm_event
  expect(results.keys).to match_array([:policy, :alert, :children_events])
@@ -163,8 +162,9 @@
  ems = FactoryGirl.create(:ext_management_system)
  FactoryGirl.create(:miq_event_definition, :name => event)
  FactoryGirl.create(:miq_event, :event_type => event, :target => ems)
+ inputs = {:type => ems.class.name, :triggering_type => event, :triggering_data => nil}
 
- expect(MiqPolicy).to receive(:enforce_policy).with(ems, event, :type => ems.class.name)
+ expect(MiqPolicy).to receive(:enforce_policy).with(ems, event, inputs)
  MiqEvent.first.process_evm_event
  end
 
@@ -181,12 +181,18 @@
  :event_type => event,
  :target => vm,
  :full_data => {:source_event_id => ems_event.id})
+ inputs = {
+ :type => vm.class.name,
+ :source_event => ems_event,
+ :triggering_type => event,
+ :triggering_data => {:source_event_id => ems_event.id}
+ }
 
  expect(MiqPolicy).to receive(:enforce_policy).with(
  vm,
  event,
- :type => vm.class.name,
-  :source_event => ems_event)
+ inputs
+ )
  MiqEvent.first.process_evm_event
  end
  end

spec/models/notification_spec.rb

@@ -131,4 +131,22 @@
  expect(notification.seen_by_all_recipients?).to be_truthy
  end
  end
+
+ describe '.notification_text' do
+ before { NotificationType.seed }
+
+ it 'does not lookup notificaiton type without event or full_data' do
+ NotificationType.names
+ expect do
+ Notification.notification_text(nil, nil)
+ Notification.notification_text('abc', nil)
+ Notification.notification_text(nil, {})
+ end.to match_query_limit_of(0)
+ end
+
+ it 'applies message' do
+ full_data = {:subject => 'vm1'}
+ expect(Notification.notification_text('vm_retired', full_data)).to eq("Virtual Machine vm1 has been retired.")
+ end
+ end
 end