diff --git a/doc/configuration.rst b/doc/configuration.rst index ed9cc334e7..25ee073594 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -105,34 +105,6 @@ default-scheduler-url non-standard URI scheme: ``http+unix`` example: ``http+unix://%2Fvar%2Frun%2Fluigid%2Fluigid.sock/`` -email-prefix - Optional prefix to add to the subject line of all e-mails. For - example, setting this to "[LUIGI]" would change the subject line of an - e-mail from "Luigi: Framework error" to "[LUIGI] Luigi: Framework - error" - -email-sender - User name in from field of error e-mails. - Default value: luigi-client@ - -email-type - Type of e-mail to send. Valid values are "plain", "html" and "none". - When set to html, tracebacks are wrapped in
 tags to get fixed-
-  width font.
-
-  New in version 2.1.0: When set to none, no e-mails will be sent.
-
-  Default value is plain.
-
-error-email
-  Recipient of all error e-mails. If this is not set, no error e-mails
-  are sent when Luigi crashes unless the crashed job has owners set. If
-  Luigi is run from the command line, no e-mails will be sent unless
-  output is redirected to a file.
-
-  Set it to SNS Topic ARN if you want to receive notifications through
-  Amazon SNS. See also section `[email]`_.
-
 hdfs-tmp-dir
   Base directory in which to store temporary files on hdfs. Defaults to
   tempfile.gettempdir()
@@ -182,32 +154,6 @@ rpc-connect-timeout
   Number of seconds to wait before timing out when making an API call.
   Defaults to 10.0
 
-smtp_host
-  Hostname for sending mail throug smtp. Defaults to localhost.
-
-smtp_local_hostname
-  If specified, overrides the FQDN of localhost in the HELO/EHLO
-  command.
-
-smtp_login
-  Username to log in to your smtp server, if necessary.
-
-smtp_password
-  Password to log in to your smtp server. Must be specified for
-  smtp_login to have an effect.
-
-smtp_port
-  Port number for smtp on smtp_host. Defaults to 0.
-
-smtp_ssl
-  If true, connects to smtp through SSL. Defaults to false.
-
-smtp_without_tls
-  If true, connects to smtp without TLS. Defaults to false.
-
-smtp_timeout
-  Optionally sets the number of seconds after which smtp attempts should
-  time out.
 
 .. _worker-config:
 
@@ -310,19 +256,46 @@ force-send
   If true, e-mails are sent in all run configurations (even if stdout is
   connected to a tty device).  Defaults to False.
 
-type
+format
+  Type of e-mail to send. Valid values are "plain", "html" and "none".
+  When set to html, tracebacks are wrapped in 
 tags to get fixed-
+  width font. When set to none, no e-mails will be sent.
+
+  Default value is plain.
+
+method
   Valid values are "smtp", "sendgrid", "ses" and "sns". SES and SNS are
   services of Amazon web services. SendGrid is an email delivery service.
   The default value is "smtp".
 
-In order to send messages through Amazon SNS or SES set up your AWS config
-files or run Luigi on an EC2 instance with proper instance profile.
+  In order to send messages through Amazon SNS or SES set up your AWS
+  config files or run Luigi on an EC2 instance with proper instance
+  profile.
 
-These parameters control sending error e-mails through SendGrid.
+  In order to use sendgrid, fill in your sendgrid username and password
+  in the `[sendgrid]`_ section.
 
-SENDGRID_USERNAME
+  In order to use smtp, fill in the appropriate fields in the `[smtp]`_
+  section.
 
-SENDGRID_PASSWORD
+prefix
+  Optional prefix to add to the subject line of all e-mails. For
+  example, setting this to "[LUIGI]" would change the subject line of an
+  e-mail from "Luigi: Framework error" to "[LUIGI] Luigi: Framework
+  error"
+
+receiver
+  Recipient of all error e-mails. If this is not set, no error e-mails
+  are sent when Luigi crashes unless the crashed job has owners set. If
+  Luigi is run from the command line, no e-mails will be sent unless
+  output is redirected to a file.
+
+  Set it to SNS Topic ARN if you want to receive notifications through
+  Amazon SNS. Make sure to set method to sns in this case too.
+
+sender
+  User name in from field of error e-mails.
+  Default value: luigi-client@
 
 
 [hadoop]
@@ -605,6 +578,51 @@ worker-disconnect-delay
   failed. Defaults to 60.
 
 
+[sendgrid]
+----------
+
+These parameters control sending error e-mails through SendGrid.
+
+password
+  Password used for sendgrid login
+
+username
+  Name of the user for the sendgrid login
+
+
+[smtp]
+------
+
+These parameters control the smtp server setup.
+
+host
+  Hostname for sending mail throug smtp. Defaults to localhost.
+
+local_hostname
+  If specified, overrides the FQDN of localhost in the HELO/EHLO
+  command.
+
+no_tls
+  If true, connects to smtp without TLS. Defaults to false.
+
+password
+  Password to log in to your smtp server. Must be specified for
+  username to have an effect.
+
+port
+  Port number for smtp on smtp_host. Defaults to 0.
+
+ssl
+  If true, connects to smtp through SSL. Defaults to false.
+
+timeout
+  Sets the number of seconds after which smtp attempts should time out.
+  Defaults to 10.
+
+username
+  Username to log in to your smtp server, if necessary.
+
+
 [spark]
 -------
 
diff --git a/luigi/notifications.py b/luigi/notifications.py
index 9ced42e8dc..5c4a3f5855 100644
--- a/luigi/notifications.py
+++ b/luigi/notifications.py
@@ -19,14 +19,12 @@
 
 This needs some more documentation.
 See :doc:`/configuration` for configuration options.
-In particular using the config `error-email` should set up Luigi so that it will send emails when tasks fail.
+In particular using the config `receiver` should set up Luigi so that it will send emails when tasks fail.
 
 .. code-block:: ini
 
-    [core]
-    error-email=foo@bar.baz
-
-TODO: Eventually, all email configuration should move into the [email] section.
+    [email]
+    receiver=foo@bar.baz
 '''
 
 import logging
@@ -34,7 +32,6 @@
 import sys
 import textwrap
 
-from luigi import configuration
 import luigi.task
 import luigi.parameter
 
@@ -68,26 +65,92 @@ def complete(self):
         return False
 
 
-def email_type():
-    return configuration.get_config().get('core', 'email-type', 'plain')
+class email(luigi.Config):
+    force_send = luigi.parameter.BoolParameter(
+        default=False,
+        description='Send e-mail even from a tty or with DEBUG set')
+    format = luigi.parameter.ChoiceParameter(
+        default='plain',
+        config_path=dict(section='core', name='email-type'),
+        choices=('plain', 'html', 'none'),
+        description='Format type for sent e-mails')
+    method = luigi.parameter.ChoiceParameter(
+        default='smtp',
+        config_path=dict(section='email', name='type'),
+        choices=('smtp', 'sendgrid', 'ses', 'sns'),
+        description='Method for sending e-mail')
+    prefix = luigi.parameter.Parameter(
+        default=None,
+        config_path=dict(section='core', name='email-prefix'),
+        description='Prefix for subject lines of all e-mails')
+    receiver = luigi.parameter.Parameter(
+        default=None,
+        config_path=dict(section='core', name='error-email'),
+        description='Address to send error e-mails to')
+    sender = luigi.parameter.Parameter(
+        default=DEFAULT_CLIENT_EMAIL,
+        config_path=dict(section='core', name='email-sender'),
+        description='Address to send e-mails from')
+
+
+class smtp(luigi.Config):
+    host = luigi.parameter.Parameter(
+        default='localhost',
+        config_path=dict(section='core', name='smtp_host'),
+        description='Hostname of smtp server')
+    local_hostname = luigi.parameter.Parameter(
+        default=None,
+        config_path=dict(section='core', name='smtp_local_hostname'),
+        description='If specified, local_hostname is used as the FQDN of the local host in the HELO/EHLO command')
+    no_tls = luigi.parameter.BoolParameter(
+        default=False,
+        config_path=dict(section='core', name='smtp_without_tls'),
+        description='Do not use TLS in SMTP connections')
+    password = luigi.parameter.Parameter(
+        default=None,
+        config_path=dict(section='core', name='smtp_password'),
+        description='Password for the SMTP server login')
+    port = luigi.parameter.IntParameter(
+        default=0,
+        config_path=dict(section='core', name='smtp_port'),
+        description='Port number for smtp server')
+    ssl = luigi.parameter.BoolParameter(
+        default=False,
+        config_path=dict(section='core', name='smtp_ssl'),
+        description='Use SSL for the SMTP connection.')
+    timeout = luigi.parameter.FloatParameter(
+        default=10.0,
+        config_path=dict(section='core', name='smtp_timeout'),
+        description='Number of seconds before timing out the smtp connection')
+    username = luigi.parameter.Parameter(
+        default=None,
+        config_path=dict(section='core', name='smtp_login'),
+        description='Username used to log in to the SMTP host')
+
+
+class sendgrid(luigi.Config):
+    username = luigi.parameter.Parameter(
+        config_path=dict(section='email', name='SENDGRID_USERNAME'),
+        description='Username for sendgrid login')
+    password = luigi.parameter.Parameter(
+        config_path=dict(section='email', name='SENDGRID_PASSWORD'),
+        description='Username for sendgrid login')
 
 
 def generate_email(sender, subject, message, recipients, image_png):
-    import email
-    import email.mime
-    import email.mime.multipart
-    import email.mime.text
-    import email.mime.image
+    from email.mime.multipart import MIMEMultipart
+    from email.mime.text import MIMEText
+    from email.mime.image import MIMEImage
 
-    msg_root = email.mime.multipart.MIMEMultipart('related')
+    msg_root = MIMEMultipart('related')
 
-    msg_text = email.mime.text.MIMEText(message, email_type())
+    msg_text = MIMEText(message, email().format)
     msg_text.set_charset('utf-8')
     msg_root.attach(msg_text)
 
     if image_png:
         with open(image_png, 'rb') as fp:
-            msg_image = email.mime.image.MIMEImage(fp.read(), 'png')
+            msg_image = MIMEImage(fp.read(), 'png')
         msg_root.attach(msg_image)
 
     msg_root['Subject'] = subject
@@ -101,7 +164,7 @@ def wrap_traceback(traceback):
     """
     For internal use only (until further notice)
     """
-    if email_type() == 'html':
+    if email().format == 'html':
         try:
             from pygments import highlight
             from pygments.lexers import PythonTracebackLexer
@@ -121,38 +184,34 @@ def wrap_traceback(traceback):
     return wrapped
 
 
-def send_email_smtp(config, sender, subject, message, recipients, image_png):
+def send_email_smtp(sender, subject, message, recipients, image_png):
     import smtplib
 
-    smtp_ssl = config.getboolean('core', 'smtp_ssl', False)
-    smtp_without_tls = config.getboolean('core', 'smtp_without_tls', False)
-    smtp_host = config.get('core', 'smtp_host', 'localhost')
-    smtp_port = config.getint('core', 'smtp_port', 0)
-    smtp_local_hostname = config.get('core', 'smtp_local_hostname', None)
-    smtp_timeout = config.getfloat('core', 'smtp_timeout', None)
-    kwargs = dict(host=smtp_host, port=smtp_port, local_hostname=smtp_local_hostname)
-    if smtp_timeout:
-        kwargs['timeout'] = smtp_timeout
-
-    smtp_login = config.get('core', 'smtp_login', None)
-    smtp_password = config.get('core', 'smtp_password', None)
+    smtp_config = smtp()
+    kwargs = dict(
+        host=smtp_config.host,
+        port=smtp_config.port,
+        local_hostname=smtp_config.local_hostname,
+    )
+    if smtp_config.timeout:
+        kwargs['timeout'] = smtp_config.timeout
 
     try:
-        smtp = smtplib.SMTP(**kwargs) if not smtp_ssl else smtplib.SMTP_SSL(**kwargs)
-        smtp.ehlo_or_helo_if_needed()
-        if smtp.has_extn('starttls') and not smtp_without_tls:
-            smtp.starttls()
-        if smtp_login and smtp_password:
-            smtp.login(smtp_login, smtp_password)
+        smtp_conn = smtplib.SMTP_SSL(**kwargs) if smtp_config.ssl else smtplib.SMTP(**kwargs)
+        smtp_conn.ehlo_or_helo_if_needed()
+        if smtp_conn.has_extn('starttls') and not smtp_config.no_tls:
+            smtp_conn.starttls()
+        if smtp_config.username and smtp_config.password:
+            smtp_conn.login(smtp_config.username, smtp_config.password)
 
         msg_root = generate_email(sender, subject, message, recipients, image_png)
 
-        smtp.sendmail(sender, recipients, msg_root.as_string())
+        smtp_conn.sendmail(sender, recipients, msg_root.as_string())
     except socket.error:
         logger.error("Not able to connect to smtp server")
 
 
-def send_email_ses(config, sender, subject, message, recipients, image_png):
+def send_email_ses(sender, subject, message, recipients, image_png):
     """
     Sends notification through AWS SES.
 
@@ -177,16 +236,15 @@ def send_email_ses(config, sender, subject, message, recipients, image_png):
                                                response['ResponseMetadata']['HTTPStatusCode']))
 
 
-def send_email_sendgrid(config, sender, subject, message, recipients, image_png):
-    import sendgrid
-    client = sendgrid.SendGridClient(config.get('email', 'SENDGRID_USERNAME', None),
-                                     config.get('email', 'SENDGRID_PASSWORD', None),
-                                     raise_errors=True)
-    to_send = sendgrid.Mail()
+def send_email_sendgrid(sender, subject, message, recipients, image_png):
+    import sendgrid as sendgrid_lib
+    client = sendgrid_lib.SendGridClient(
+        sendgrid().username, sendgrid().password, raise_errors=True)
+    to_send = sendgrid_lib.Mail()
     to_send.add_to(recipients)
     to_send.set_from(sender)
     to_send.set_subject(subject)
-    if email_type() == 'html':
+    if email().format == 'html':
         to_send.set_html(message)
     else:
         to_send.set_text(message)
@@ -197,10 +255,10 @@ def send_email_sendgrid(config, sender, subject, message, recipients, image_png)
 
 
 def _email_disabled():
-    if email_type() == 'none':
-        logger.info("Not sending email when email-type is none")
+    if email().format == 'none':
+        logger.info("Not sending email when email format is none")
         return True
-    elif configuration.get_config().getboolean('email', 'force-send', False):
+    elif email().force_send:
         return False
     elif sys.stdout.isatty():
         logger.info("Not sending email when running from a tty")
@@ -211,7 +269,7 @@ def _email_disabled():
         return False
 
 
-def send_email_sns(config, sender, subject, message, topic_ARN, image_png):
+def send_email_sns(sender, subject, message, topic_ARN, image_png):
     """
     Sends notification through AWS SNS. Takes Topic ARN from recipients.
 
@@ -243,18 +301,17 @@ def send_email(subject, message, sender, recipients, image_png=None):
     Decides whether to send notification. Notification is cancelled if there are
     no recipients or if stdout is onto tty or if in debug mode.
 
-    Dispatches on config value email.type.  Default is 'smtp'.
+    Dispatches on config value email.method.  Default is 'smtp'.
     """
-    config = configuration.get_config()
-    notifiers = {'ses': send_email_ses,
-                 'sendgrid': send_email_sendgrid,
-                 'smtp': send_email_smtp,
-                 'sns': send_email_sns}
+    notifiers = {
+        'ses': send_email_ses,
+        'sendgrid': send_email_sendgrid,
+        'smtp': send_email_smtp,
+        'sns': send_email_sns,
+    }
 
     subject = _prefix(subject)
-    if not recipients or recipients == (None,):
-        return
-    if _email_disabled():
+    if not recipients or recipients == (None,) or _email_disabled():
         return
 
     # Clean the recipients lists to allow multiple error-email addresses, comma
@@ -267,14 +324,12 @@ def send_email(subject, message, sender, recipients, image_png=None):
     recipients = recipients_tmp
 
     # Get appropriate sender and call it to send the notification
-    email_sender_type = config.get('email', 'type', None)
-    email_sender = notifiers.get(email_sender_type, send_email_smtp)
-    email_sender(config, sender, subject, message, recipients, image_png)
+    email_sender = notifiers[email().method]
+    email_sender(sender, subject, message, recipients, image_png)
 
 
 def _email_recipients(additional_recipients=None):
-    config = configuration.get_config()
-    receiver = config.get('core', 'error-email', None)
+    receiver = email().receiver
     recipients = [receiver] if receiver else []
     if additional_recipients:
         if isinstance(additional_recipients, str):
@@ -290,10 +345,9 @@ def send_error_email(subject, message, additional_recipients=None):
 
     If no error-email is configured, then a message is logged.
     """
-    config = configuration.get_config()
     recipients = _email_recipients(additional_recipients)
     if recipients:
-        sender = config.get('core', 'email-sender', DEFAULT_CLIENT_EMAIL)
+        sender = email().sender
         logger.info("Sending warning email to %r", recipients)
         send_email(
             subject=subject,
@@ -312,11 +366,10 @@ def _prefix(subject):
     If the config has a special prefix for emails then this function adds
     this prefix.
     """
-    config = configuration.get_config()
-    email_prefix = config.get('core', 'email-prefix', None)
-    if email_prefix is not None:
-        subject = "%s %s" % (email_prefix, subject)
-    return subject
+    if email().prefix is not None:
+        return "{} {}".format(email().prefix, subject)
+    else:
+        return subject
 
 
 def format_task_error(headline, task, command, formatted_exception=None):
@@ -330,13 +383,12 @@ def format_task_error(headline, task, command, formatted_exception=None):
     :return: message body
     """
 
-    typ = email_type()
     if formatted_exception:
         formatted_exception = wrap_traceback(formatted_exception)
     else:
         formatted_exception = ""
 
-    if typ == 'html':
+    if email().format == 'html':
         msg_template = textwrap.dedent('''
         
         
diff --git a/test/notifications_test.py b/test/notifications_test.py
index 45b089853f..0d5a144f65 100644
--- a/test/notifications_test.py
+++ b/test/notifications_test.py
@@ -22,7 +22,6 @@
 
 from helpers import with_config
 from luigi import notifications
-from luigi import configuration
 from luigi.scheduler import Scheduler
 from luigi.worker import Worker
 from luigi import six
@@ -215,8 +214,7 @@ def test_sends_smtp_email(self):
                 generate_email.return_value\
                     .as_string.return_value = self.mocked_email_msg
 
-                notifications.send_email_smtp(configuration.get_config(),
-                                              *self.notification_args)
+                notifications.send_email_smtp(*self.notification_args)
 
                 SMTP.assert_called_once_with(**smtp_kws)
                 SMTP.return_value.login.assert_called_once_with("Robin", "dooH")
@@ -249,8 +247,7 @@ def test_sends_smtp_email_without_tls(self):
                 generate_email.return_value \
                     .as_string.return_value = self.mocked_email_msg
 
-                notifications.send_email_smtp(configuration.get_config(),
-                                              *self.notification_args)
+                notifications.send_email_smtp(*self.notification_args)
 
                 SMTP.assert_called_once_with(**smtp_kws)
                 self.assertEqual(SMTP.return_value.starttls.called, False)
@@ -284,8 +281,7 @@ def test_sends_smtp_email_exceptions(self):
                     .as_string.return_value = self.mocked_email_msg
 
                 try:
-                    notifications.send_email_smtp(configuration.get_config(),
-                                                  *self.notification_args)
+                    notifications.send_email_smtp(*self.notification_args)
                 except socket.error:
                     self.fail("send_email_smtp() raised expection unexpectedly")
 
@@ -315,8 +311,7 @@ def test_sends_sendgrid_email(self):
         """
 
         with mock.patch('sendgrid.SendGridClient') as SendgridClient:
-            notifications.send_email_sendgrid(configuration.get_config(),
-                                              *self.notification_args)
+            notifications.send_email_sendgrid(*self.notification_args)
 
             SendgridClient.assert_called_once_with("Nikola", "jahuS", raise_errors=True)
             self.assertTrue(SendgridClient.return_value.send.called)
@@ -346,8 +341,7 @@ def test_sends_ses_email(self):
                 generate_email.return_value\
                     .as_string.return_value = self.mocked_email_msg
 
-                notifications.send_email_ses(configuration.get_config(),
-                                             *self.notification_args)
+                notifications.send_email_ses(*self.notification_args)
 
                 SES = boto_client.return_value
                 SES.send_raw_email.assert_called_once_with(
@@ -376,8 +370,7 @@ def test_sends_sns_email(self):
         """
 
         with mock.patch('boto3.resource') as res:
-            notifications.send_email_sns(configuration.get_config(),
-                                         *self.notification_args)
+            notifications.send_email_sns(*self.notification_args)
 
             SNS = res.return_value
             SNS.Topic.assert_called_once_with(self.recipients[0])
@@ -395,8 +388,7 @@ def test_sns_subject_is_shortened(self):
                        'mailFailure=False, mongodb=mongodb://localhost/stats) FAILED'
 
         with mock.patch('boto3.resource') as res:
-            notifications.send_email_sns(configuration.get_config(),
-                                         self.sender, long_subject, self.message,
+            notifications.send_email_sns(self.sender, long_subject, self.message,
                                          self.recipients, self.image_png)
 
             SNS = res.return_value
@@ -406,7 +398,7 @@ def test_sns_subject_is_shortened(self):
                             "Subject can be max 100 chars long! Found {}.".format(len(called_subj)))
 
 
-class Test_Notification_Dispatcher(unittest.TestCase, NotificationFixture):
+class TestNotificationDispatcher(unittest.TestCase, NotificationFixture):
     """
     Test dispatching of notifications on configuration values.
     """
@@ -425,7 +417,7 @@ def check_dispatcher(self, target):
 
             self.assertTrue(sender.called)
 
-            call_args = sender.call_args[0][1:]
+            call_args = sender.call_args[0]
 
             self.assertEqual(tuple(expected_args), call_args)