Add AWS SNS notification support for webhook (ansible#15184)

Support for AWS SNS notifications. SNS is a widespread service that is used to integrate with other AWS services(EG lambdas). This support would unlock use cases like triggering lambda functions, especially when AWX is deployed on EKS.

Decisions:

Data Structure
- I preferred using the same structure as Webhook for message body data because it contains all job details. For now, I directly linked to Webhook to avoid duplication, but I am open to suggestions.

AWS authentication
- To support non-AWS native environments, I added configuration options for AWS secret key, ID, and session tokens. When entered, these values are supplied to the underlining boto3 SNS client. If not entered, it falls back to the default authentication chain to support the native AWS environment. Properly configured EKS pods are created with temporary credentials that the default authentication chain can pick automatically.

---------

Signed-off-by: Ethem Cem Ozkan <ethemcem.ozkan@gmail.com>

awx/api/serializers.py

@@ -5381,7 +5381,7 @@ class Meta:
         )
 
     def get_body(self, obj):
-        if obj.notification_type in ('webhook', 'pagerduty'):
+        if obj.notification_type in ('webhook', 'pagerduty', 'awssns'):
             if isinstance(obj.body, dict):
                 if 'body' in obj.body:
                     return obj.body['body']
@@ -5403,9 +5403,9 @@ def get_related(self, obj):
     def to_representation(self, obj):
         ret = super(NotificationSerializer, self).to_representation(obj)
 
-        if obj.notification_type == 'webhook':
+        if obj.notification_type in ('webhook', 'awssns'):
             ret.pop('subject')
-        if obj.notification_type not in ('email', 'webhook', 'pagerduty'):
+        if obj.notification_type not in ('email', 'webhook', 'pagerduty', 'awssns'):
             ret.pop('body')
         return ret
 

awx/main/migrations/0193_alter_notification_notification_type_and_more.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.2.6 on 2024-05-08 07:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0192_custom_roles'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='notification',
+            name='notification_type',
+            field=models.CharField(
+                choices=[
+                    ('awssns', 'AWS SNS'),
+                    ('email', 'Email'),
+                    ('grafana', 'Grafana'),
+                    ('irc', 'IRC'),
+                    ('mattermost', 'Mattermost'),
+                    ('pagerduty', 'Pagerduty'),
+                    ('rocketchat', 'Rocket.Chat'),
+                    ('slack', 'Slack'),
+                    ('twilio', 'Twilio'),
+                    ('webhook', 'Webhook'),
+                ],
+                max_length=32,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='notificationtemplate',
+            name='notification_type',
+            field=models.CharField(
+                choices=[
+                    ('awssns', 'AWS SNS'),
+                    ('email', 'Email'),
+                    ('grafana', 'Grafana'),
+                    ('irc', 'IRC'),
+                    ('mattermost', 'Mattermost'),
+                    ('pagerduty', 'Pagerduty'),
+                    ('rocketchat', 'Rocket.Chat'),
+                    ('slack', 'Slack'),
+                    ('twilio', 'Twilio'),
+                    ('webhook', 'Webhook'),
+                ],
+                max_length=32,
+            ),
+        ),
+    ]

awx/main/models/notifications.py

@@ -31,6 +31,7 @@
 from awx.main.notifications.grafana_backend import GrafanaBackend
 from awx.main.notifications.rocketchat_backend import RocketChatBackend
 from awx.main.notifications.irc_backend import IrcBackend
+from awx.main.notifications.awssns_backend import AWSSNSBackend
 
 
 logger = logging.getLogger('awx.main.models.notifications')
@@ -40,6 +41,7 @@
 
 class NotificationTemplate(CommonModelNameNotUnique):
     NOTIFICATION_TYPES = [
+        ('awssns', _('AWS SNS'), AWSSNSBackend),
         ('email', _('Email'), CustomEmailBackend),
         ('slack', _('Slack'), SlackBackend),
         ('twilio', _('Twilio'), TwilioBackend),

awx/main/notifications/awssns_backend.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2016 Ansible, Inc.
+# All Rights Reserved.
+import json
+import logging
+
+import boto3
+from botocore.exceptions import ClientError
+
+from awx.main.notifications.base import AWXBaseEmailBackend
+from awx.main.notifications.custom_notification_base import CustomNotificationBase
+
+logger = logging.getLogger('awx.main.notifications.awssns_backend')
+WEBSOCKET_TIMEOUT = 30
+
+
+class AWSSNSBackend(AWXBaseEmailBackend, CustomNotificationBase):
+    init_parameters = {
+        "aws_region": {"label": "AWS Region", "type": "string", "default": ""},
+        "aws_access_key_id": {"label": "Access Key ID", "type": "string", "default": ""},
+        "aws_secret_access_key": {"label": "Secret Access Key", "type": "password", "default": ""},
+        "aws_session_token": {"label": "Session Token", "type": "password", "default": ""},
+        "sns_topic_arn": {"label": "SNS Topic ARN", "type": "string", "default": ""},
+    }
+    recipient_parameter = "sns_topic_arn"
+    sender_parameter = None
+
+    DEFAULT_BODY = "{{ job_metadata }}"
+    default_messages = CustomNotificationBase.job_metadata_messages
+
+    def __init__(self, aws_region, aws_access_key_id, aws_secret_access_key, aws_session_token, fail_silently=False, **kwargs):
+        session = boto3.session.Session()
+        client_config = {"service_name": 'sns'}
+        if aws_region:
+            client_config["region_name"] = aws_region
+        if aws_secret_access_key:
+            client_config["aws_secret_access_key"] = aws_secret_access_key
+        if aws_access_key_id:
+            client_config["aws_access_key_id"] = aws_access_key_id
+        if aws_session_token:
+            client_config["aws_session_token"] = aws_session_token
+        self.client = session.client(**client_config)
+        super(AWSSNSBackend, self).__init__(fail_silently=fail_silently)
+
+    def _sns_publish(self, topic_arn, message):
+        self.client.publish(TopicArn=topic_arn, Message=message, MessageAttributes={})
+
+    def format_body(self, body):
+        if isinstance(body, str):
+            try:
+                body = json.loads(body)
+            except json.JSONDecodeError:
+                pass
+
+        if isinstance(body, dict):
+            body = json.dumps(body)
+        # convert dict body to json string
+        return body
+
+    def send_messages(self, messages):
+        sent_messages = 0
+        for message in messages:
+            sns_topic_arn = str(message.recipients()[0])
+            try:
+                self._sns_publish(topic_arn=sns_topic_arn, message=message.body)
+                sent_messages += 1
+            except ClientError as error:
+                if not self.fail_silently:
+                    raise error
+
+        return sent_messages

awx/main/notifications/custom_notification_base.py

@@ -32,3 +32,15 @@ class CustomNotificationBase(object):
             "denied": {"message": DEFAULT_APPROVAL_DENIED_MSG, "body": None},
         },
     }
+
+    job_metadata_messages = {
+        "started": {"body": "{{ job_metadata }}"},
+        "success": {"body": "{{ job_metadata }}"},
+        "error": {"body": "{{ job_metadata }}"},
+        "workflow_approval": {
+            "running": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" needs review. This node can be viewed at: {{ workflow_url }}"}'},
+            "approved": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was approved. {{ workflow_url }}"}'},
+            "timed_out": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" has timed out. {{ workflow_url }}"}'},
+            "denied": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was denied. {{ workflow_url }}"}'},
+        },
+    }

awx/main/notifications/webhook_backend.py

@@ -27,17 +27,7 @@ class WebhookBackend(AWXBaseEmailBackend, CustomNotificationBase):
     sender_parameter = None
 
     DEFAULT_BODY = "{{ job_metadata }}"
-    default_messages = {
-        "started": {"body": DEFAULT_BODY},
-        "success": {"body": DEFAULT_BODY},
-        "error": {"body": DEFAULT_BODY},
-        "workflow_approval": {
-            "running": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" needs review. This node can be viewed at: {{ workflow_url }}"}'},
-            "approved": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was approved. {{ workflow_url }}"}'},
-            "timed_out": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" has timed out. {{ workflow_url }}"}'},
-            "denied": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was denied. {{ workflow_url }}"}'},
-        },
-    }
+    default_messages = CustomNotificationBase.job_metadata_messages
 
     def __init__(self, http_method, headers, disable_ssl_verification=False, fail_silently=False, username=None, password=None, **kwargs):
         self.http_method = http_method

awx/main/tests/unit/notifications/test_awssns.py

@@ -0,0 +1,26 @@
+from unittest import mock
+from django.core.mail.message import EmailMessage
+
+import awx.main.notifications.awssns_backend as awssns_backend
+
+
+def test_send_messages():
+    with mock.patch('awx.main.notifications.awssns_backend.AWSSNSBackend._sns_publish') as sns_publish_mock:
+        aws_region = 'us-east-1'
+        sns_topic = f"arn:aws:sns:{aws_region}:111111111111:topic-mock"
+        backend = awssns_backend.AWSSNSBackend(aws_region=aws_region, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None)
+        message = EmailMessage(
+            'test subject',
+            {'body': 'test body'},
+            [],
+            [
+                sns_topic,
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        sns_publish_mock.assert_called_once_with(topic_arn=sns_topic, message=message.body)
+        assert sent_messages == 1

awx/ui/src/components/NotificationList/NotificationList.js

@@ -190,6 +190,7 @@ function NotificationList({
             name: t`Notification type`,
             key: 'or__notification_type',
             options: [
+              ['awssns', t`AWS SNS`],
               ['email', t`Email`],
               ['grafana', t`Grafana`],
               ['hipchat', t`Hipchat`],

awx/ui/src/screens/NotificationTemplate/NotificationTemplateDetail/NotificationTemplateDetail.js

@@ -138,6 +138,25 @@ function NotificationTemplateDetail({ template, defaultMessages }) {
           }
           dataCy="nt-detail-type"
         />
+        {template.notification_type === 'awssns' && (
+          <>
+            <Detail
+              label={t`AWS Region`}
+              value={configuration.aws_region}
+              dataCy="nt-detail-aws-region"
+            />
+            <Detail
+              label={t`Access Key ID`}
+              value={configuration.aws_access_key_id}
+              dataCy="nt-detail-aws-access-key-id"
+            />
+            <Detail
+              label={t`SNS Topic ARN`}
+              value={configuration.sns_topic_arn}
+              dataCy="nt-detail-sns-topic-arn"
+            />
+          </>
+        )}
         {template.notification_type === 'email' && (
           <>
             <Detail
@@ -455,8 +474,8 @@ function NotificationTemplateDetail({ template, defaultMessages }) {
 }
 
 function CustomMessageDetails({ messages, defaults, type }) {
-  const showMessages = type !== 'webhook';
-  const showBodies = ['email', 'pagerduty', 'webhook'].includes(type);
+  const showMessages = !['awssns', 'webhook'].includes(type);
+  const showBodies = ['email', 'pagerduty', 'webhook', 'awssns'].includes(type);
 
   return (
     <>

awx/ui/src/screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js

@@ -131,6 +131,7 @@ function NotificationTemplatesList() {
                 name: t`Notification type`,
                 key: 'or__notification_type',
                 options: [
+                  ['awssns', t`AWS SNS`],
                   ['email', t`Email`],
                   ['grafana', t`Grafana`],
                   ['hipchat', t`Hipchat`],

awx/ui/src/screens/NotificationTemplate/constants.js

@@ -1,5 +1,6 @@
 /* eslint-disable-next-line import/prefer-default-export */
 export const NOTIFICATION_TYPES = {
+  awssns: 'AWS SNS',
   email: 'Email',
   grafana: 'Grafana',
   irc: 'IRC',

awx/ui/src/screens/NotificationTemplate/shared/CustomMessagesSubForm.js

@@ -11,8 +11,8 @@ import getDocsBaseUrl from 'util/getDocsBaseUrl';
 
 function CustomMessagesSubForm({ defaultMessages, type }) {
   const [useCustomField, , useCustomHelpers] = useField('useCustomMessages');
-  const showMessages = type !== 'webhook';
-  const showBodies = ['email', 'pagerduty', 'webhook'].includes(type);
+  const showMessages = !['webhook', 'awssns'].includes(type);
+  const showBodies = ['email', 'pagerduty', 'webhook', 'awssns'].includes(type);
 
   const { setFieldValue } = useFormikContext();
   const config = useConfig();

awx/ui/src/screens/NotificationTemplate/shared/NotificationTemplateForm.js

@@ -78,6 +78,7 @@ function NotificationTemplateFormFields({ defaultMessages, template }) {
               label: t`Choose a Notification Type`,
               isDisabled: true,
             },
+            { value: 'awssns', key: 'awssns', label: t`AWS SNS` },
             { value: 'email', key: 'email', label: t`E-mail` },
             { value: 'grafana', key: 'grafana', label: 'Grafana' },
             { value: 'irc', key: 'irc', label: 'IRC' },

awx/ui/src/screens/NotificationTemplate/shared/TypeInputsSubForm.js

@@ -29,6 +29,7 @@ import Popover from '../../../components/Popover/Popover';
 import getHelpText from './Notifications.helptext';
 
 const TypeFields = {
+  awssns: AWSSNSFields,
   email: EmailFields,
   grafana: GrafanaFields,
   irc: IRCFields,
@@ -58,6 +59,44 @@ TypeInputsSubForm.propTypes = {
 
 export default TypeInputsSubForm;
 
+function AWSSNSFields() {
+  return (
+    <>
+      <FormField
+        id="awssns-aws-region"
+        label={t`AWS Region`}
+        name="notification_configuration.aws_region"
+        type="text"
+        isRequired
+      />
+      <FormField
+        id="awssns-aws-access-key-id"
+        label={t`Access Key ID`}
+        name="notification_configuration.aws_access_key_id"
+        type="text"
+      />
+      <PasswordField
+        id="awssns-aws-secret-access-key"
+        label={t`Secret Access Key`}
+        name="notification_configuration.aws_secret_access_key"
+      />
+      <PasswordField
+        id="awssns-aws-session-token"
+        label={t`Session Token`}
+        name="notification_configuration.aws_session_token"
+      />
+      <FormField
+        id="awssns-sns-topic-arn"
+        label={t`SNS Topic ARN`}
+        name="notification_configuration.sns_topic_arn"
+        type="text"
+        validate={required(null)}
+        isRequired
+      />
+    </>
+  );
+}
+
 function EmailFields() {
   const helpText = getHelpText();
   return (

awx/ui/src/screens/NotificationTemplate/shared/notification-template-default-messages.json

@@ -203,6 +203,39 @@
       }
     }
   },
+  "awssns": {
+    "started": {
+      "body": "{{ job_metadata }}"
+    },
+    "success": {
+      "body": "{{ job_metadata }}"
+    },
+    "error": {
+      "body": "{{ job_metadata }}"
+    },
+    "workflow_approval": {
+      "running": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" needs review. This node can be viewed at: {{ workflow_url }}"
+        }
+      },
+      "approved": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" was approved. {{ workflow_url }}"
+        }
+      },
+      "timed_out": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" has timed out. {{ workflow_url }}"
+        }
+      },
+      "denied": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" was denied. {{ workflow_url }}"
+        }
+      }
+    }
+  },
   "mattermost": {
     "started": {
       "message": "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}",