notify users of killing workers when exceed memory

- Provide more complete information for worker out of memory errors
- Provide notification messages for out of memory errors
- Introduce ability to disable notification
- Include notification.message in alert emails

https://bugzilla.redhat.com/show_bug.cgi?id=1535177

app/models/miq_action.rb

@@ -309,6 +309,7 @@ def action_email(action, rec, inputs)
         :header            => "Alert Triggered",
         :policy_detail     => "Alert '#{inputs[:policy].description}', triggered",
         :event_description => inputs[:event].description,
+        :event_details     => Notification.notification_text(inputs[:triggering_type], inputs[:triggering_data]),
         :entity_type       => rec.class.to_s,
         :entity_name       => rec.name
       }

app/models/miq_event.rb

@@ -40,7 +40,7 @@ def self.raise_evm_event(target, raw_event, inputs = {}, options = {})
       return
     end
 
-    event_obj = build_evm_event(event, target)
+    event_obj = build_evm_event(event, target, inputs[:full_data])
     inputs.merge!('MiqEvent::miq_event' => event_obj.id, :miq_event_id => event_obj.id)
     inputs.merge!('EventStream::event_stream' => event_obj.id, :event_stream_id => event_obj.id)
 
@@ -61,6 +61,8 @@ def process_evm_event(inputs = {})
     results = {}
     inputs[:type] ||= target.class.name
     inputs[:source_event] = source_event if source_event
+    inputs[:triggering_type] = event_type
+    inputs[:triggering_data] = full_data
 
     _log.info("Event Raised [#{event_type}]")
     begin
@@ -79,10 +81,11 @@ def process_evm_event(inputs = {})
     results
   end
 
-  def self.build_evm_event(event, target)
+  def self.build_evm_event(event, target, full_data = nil)
     options = {
       :event_type => event,
       :target     => target,
+      :full_data  => full_data,
       :source     => 'POLICY',
       :timestamp  => Time.now.utc
     }

app/models/miq_server/worker_management/monitor/validation.rb

@@ -25,7 +25,16 @@ def validate_worker(w)
     if MiqWorker::STATUSES_CURRENT.include?(w.status) && usage_exceeds_threshold?(usage, memory_threshold)
       msg = "#{w.format_full_log_msg} process memory usage [#{usage}] exceeded limit [#{memory_threshold}], requesting worker to exit"
       _log.warn(msg)
-      MiqEvent.raise_evm_event_queue(w.miq_server, "evm_worker_memory_exceeded", :event_details => msg, :type => w.class.name)
+      helper = ApplicationController.helpers
+      full_data = {
+        :name             => w.type,
+        :memory_usage     => helper.number_to_human_size(usage),
+        :memory_threshold => helper.number_to_human_size(memory_threshold),
+      }
+      MiqEvent.raise_evm_event_queue(w.miq_server, "evm_worker_memory_exceeded",
+                                     :event_details => msg,
+                                     :type          => w.class.name,
+                                     :full_data     => full_data)
       restart_worker(w)
       return false
     end

app/models/notification.rb

@@ -25,7 +25,10 @@ def type=(typ)
   def self.emit_for_event(event)
     return unless NotificationType.names.include?(event.event_type)
     type = NotificationType.find_by(:name => event.event_type)
-    Notification.create(:notification_type => type, :subject => event.target)
+    return unless type.has_audience?
+    Notification.create(:notification_type => type,
+                        :options           => event.full_data,
+                        :subject           => event.target)
   end
 
   def to_h
@@ -41,6 +44,12 @@ def seen_by_all_recipients?
     notification_recipients.unseen.empty?
   end
 
+  def self.notification_text(event_type, full_data)
+    return unless full_data && NotificationType.names.include?(event_type)
+    type = NotificationType.find_by(:name => event_type)
+    type.message % full_data
+  end
+
   private
 
   def emit_message

app/models/notification_type.rb

@@ -4,6 +4,8 @@ class NotificationType < ApplicationRecord
   AUDIENCE_TENANT = 'tenant'.freeze
   AUDIENCE_GLOBAL = 'global'.freeze
   AUDIENCE_SUPERADMIN = 'superadmin'.freeze
+  # don't send out notifications, but keep the template around
+  AUDIENCE_NONE = 'none'.freeze
   has_many :notifications
   validates :message, :presence => true
   validates :level, :inclusion => { :in => %w(success error warning info) }
@@ -27,9 +29,17 @@ def subscriber_ids(subject, initiator)
       end.try(:user_ids)
     when AUDIENCE_SUPERADMIN
       User.superadmins.pluck(:id)
+    when AUDIENCE_NONE
+      []
     end
   end
 
+  # this disables notifications, but allows the notification to still exist
+  # this notification template can be used for emails
+  def has_audience?
+    audience != AUDIENCE_NONE
+  end
+
   def self.names
     @names ||= Set.new(pluck(:name))
   end

db/fixtures/notification_types.yml

@@ -264,3 +264,8 @@
   :expires_in: 24.hours
   :level: :error
   :audience: superadmin
+- :name: evm_worker_memory_exceeded
+  :message: 'Killing worker %{name} due to excessive memory usage. %{memory_usage} used memory exceeds limit of %{memory_threshold}.'
+  :expires_in: 24.hours
+  :level: :error
+  :audience: none

spec/models/miq_action_spec.rb

@@ -420,6 +420,52 @@ def stub_csv(data)
         expect(MiqQueue).to receive(:put).with(q_options).once
         action.action_email(action, nil, inputs)
       end
+
+      context 'when alerting' do
+        before do
+          NotificationType.instance_variable_set(:@names, nil)
+          NotificationType.seed if NotificationType.all.empty?
+        end
+
+        let(:description) { 'my alert' }
+        let(:alert) { MiqAlert.new(:description => description) }
+        let(:event) { MiqEventDefinition.new(:name => "AlertEvent", :description => "Alert condition met") }
+        let(:vm) { Vm.new(:name => "vm1") }
+        let(:inputs) do
+          {
+            :policy          => alert,
+            :event           => event,
+            :synchronous     => false,
+            :results         => true,
+            :sequence        => 1,
+            :triggering_type => "vm_retired",
+            :triggering_data => {:subject => 'vm1'},
+          }
+        end
+
+        let(:args) do
+          [{
+            :to              => nil,
+            :from            => "cfadmin@cfserver.com",
+            :subject         => "Alert Triggered: my alert, for (VM) vm1",
+            :miq_action_hash => {
+              :header            => "Alert Triggered",
+              :policy_detail     => "Alert 'my alert', triggered",
+              :event_description => "Alert condition met",
+              :event_details     => "Virtual Machine vm1 has been retired.",
+              :entity_type       => "Vm",
+              :entity_name       => "vm1",
+            }
+          }]
+        end
+
+        it 'generates an MiqAlert email' do
+          q_options[:args] = args
+
+          expect(MiqQueue).to receive(:put).with(q_options).once
+          action.action_email(action, vm, inputs)
+        end
+      end
     end
   end
 

spec/models/miq_event_spec.rb

@@ -129,11 +129,10 @@
         event = 'vm_start'
         FactoryGirl.create(:miq_event_definition, :name => event)
         FactoryGirl.create(:miq_event, :event_type => event, :target => @cluster)
-        target_class = @cluster.class.name
-
-        expect(MiqPolicy).to receive(:enforce_policy).with(@cluster, event, :type => target_class)
-        expect(MiqAlert).to receive(:evaluate_alerts).with(@cluster, event, :type => target_class)
-        expect(MiqEvent).to receive(:raise_event_for_children).with(@cluster, event, :type => target_class)
+        inputs = {:type => @cluster.class.name, :triggering_type => event, :triggering_data => nil}
+        expect(MiqPolicy).to receive(:enforce_policy).with(@cluster, event, inputs)
+        expect(MiqAlert).to receive(:evaluate_alerts).with(@cluster, event, inputs)
+        expect(MiqEvent).to receive(:raise_event_for_children).with(@cluster, event, inputs)
 
         results = MiqEvent.first.process_evm_event
         expect(results.keys).to match_array([:policy, :alert, :children_events])
@@ -163,8 +162,9 @@
         ems = FactoryGirl.create(:ext_management_system)
         FactoryGirl.create(:miq_event_definition, :name => event)
         FactoryGirl.create(:miq_event, :event_type => event, :target => ems)
+        inputs = {:type => ems.class.name, :triggering_type => event, :triggering_data => nil}
 
-        expect(MiqPolicy).to receive(:enforce_policy).with(ems, event, :type => ems.class.name)
+        expect(MiqPolicy).to receive(:enforce_policy).with(ems, event, inputs)
         MiqEvent.first.process_evm_event
       end
 
@@ -181,12 +181,18 @@
           :event_type => event,
           :target     => vm,
           :full_data  => {:source_event_id => ems_event.id})
+        inputs = {
+          :type            => vm.class.name,
+          :source_event    => ems_event,
+          :triggering_type => event,
+          :triggering_data => {:source_event_id => ems_event.id}
+        }
 
         expect(MiqPolicy).to receive(:enforce_policy).with(
           vm,
           event,
-          :type         => vm.class.name,
-          :source_event => ems_event)
+          inputs
+        )
         MiqEvent.first.process_evm_event
       end
     end

spec/models/notification_spec.rb

@@ -131,4 +131,25 @@
       expect(notification.seen_by_all_recipients?).to be_truthy
     end
   end
+
+  describe '.notification_text' do
+    before do
+      NotificationType.instance_variable_set(:@names, nil)
+      NotificationType.seed if NotificationType.all.empty?
+    end
+
+    it 'does not lookup notificaiton type without event or full_data' do
+      NotificationType.names
+      expect do
+        Notification.notification_text(nil, nil)
+        Notification.notification_text('abc', nil)
+        Notification.notification_text(nil, {})
+      end.to match_query_limit_of(0)
+    end
+
+    it 'applies message' do
+      full_data = {:subject => 'vm1'}
+      expect(Notification.notification_text('vm_retired', full_data)).to eq("Virtual Machine vm1 has been retired.")
+    end
+  end
 end