add email template and alert functions

coldfront/core/resource/management/commands/add_resource_defaults.py

@@ -114,7 +114,7 @@ def handle(self, *args, **options):
  defaults={'value': default_value}
  )
 
- quantity_label = "Quantity in TB"
+ quantity_label = "TB"
  if default_value == 20:
  quantity_label += " in 20T increments"
 

coldfront/core/utils/fasrc.py

@@ -2,22 +2,27 @@
 """
 import os
 import json
+import logging
 import operator
 from functools import reduce
 from datetime import datetime
 
 import pandas as pd
+from django.conf import settings
 from django.db.models import Q
 from django.contrib.auth import get_user_model
+from django.urls import reverse
 from ifxbilling.models import Product
 
 from coldfront.core.utils.common import import_from_settings
 from coldfront.core.project.models import Project
 from coldfront.core.resource.models import Resource
+from coldfront.core.utils.mail import send_allocation_manager_email
 
+logger = logging.getLogger(__name__)
 
-MISSING_DATA_DIR = './local_data/missing/'
-
+MISSING_DATA_DIR = import_from_settings('MISSING_DATA_DIR', './local_data/missing/')
+DATA_MANAGERS = import_from_settings('DATA_MANAGERS', ['General Manager, Data Manager'])
 username_ignore_list = import_from_settings('username_ignore_list', [])
 groupname_ignore_list = import_from_settings('groupname_ignore_list', [])
 
@@ -29,6 +34,7 @@ def get_quarter_start_end():
  quarter = (datetime.today().month-1)//3
  return (quarter_starts[quarter], quarter_ends[quarter])
 
+
 def sort_by(list1, sorter, how='attr'):
  """split one list into two on basis of each item's ability to meet a condition
  Parameters
@@ -50,6 +56,7 @@ def sort_by(list1, sorter, how='attr'):
  raise Exception('unclear sorting method')
  return is_true, is_false
 
+
 def select_one_project_allocation(project_obj, resource_obj, dirpath=None):
  """
  Get one allocation for a given project/resource pairing; handle return of
@@ -69,13 +76,15 @@ def select_one_project_allocation(project_obj, resource_obj, dirpath=None):
  allocation_query = project_obj.allocation_set.filter(**filter_vals)
  if allocation_query.count() == 1:
  allocation_obj = allocation_query.first()
- if allocation_obj.path and dirpath and allocation_obj.path not in dirpath and dirpath not in allocation_obj.path:
- return None
+ if allocation_obj.subdirectory and allocation_obj.subdirectory != dirpath:
+ logger.error("directory path mismatch:", allocation_obj.subdirectory, dirpath)
  elif allocation_query.count() < 1:
  allocation_obj = None
  elif allocation_query.count() > 1:
- allocation_obj = next((a for a in allocation_query if a.path.lower() in dirpath.lower()),
- None)
+ allocation_obj = next(
+ (a for a in allocation_query if a.path.lower() in dirpath.lower()),
+ 'MultiAllocationError'
+ )
  return allocation_obj
 
 
@@ -90,6 +99,7 @@ def determine_size_fmt(byte_num):
  byte_num /= 1024.0
  return(round(byte_num, 3), unit)
 
+
 def convert_size_fmt(num, target_unit, source_unit='B'):
  units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB']
  diff = units.index(target_unit) - units.index(source_unit)
@@ -103,6 +113,7 @@ def convert_size_fmt(num, target_unit, source_unit='B'):
  num*=1024.0
  return round(num, 3)
 
+
 def get_resource_rate(resource):
  """find Product with the name provided and return the associated rate"""
  try:
@@ -121,6 +132,53 @@ def get_resource_rate(resource):
  price = convert_size_fmt(rate_obj.price, 'TB', source_unit=rate_obj.units)
  return round(price/100, 2)
 
+
+def allocation_reaching_capacity_operations(allocation_obj, new_usage):
+ """if allocation_obj.usage is <80% of allocation_obj.limit and new_usage
+ >80% of allocation_obj.limit, send email to pi and data manager.
+ """
+ threshold = 80
+ if (
+ allocation_obj.usage_exact/allocation_obj.size_exact < 0.8
+ and new_usage/allocation_obj.size > 0.8
+ ):
+ resource = allocation_obj.get_parent_resource
+ # send email to pi and data manager
+ # define: center_name, threshold, resource, allocation_url, request_allocation_url, starfish_url, starfish_docs_url, signature
+ allocation_pk_dict = {'pk': allocation_obj.pk}
+ other_vars = {
+ 'threshold': threshold,
+ 'project_title': allocation_obj.project.title,
+ 'allocation_quota': f'{allocation_obj.size} {allocation_obj.get_parent_resource.quantity_label}',
+ 'resource': resource.name,
+ 'change_request_url': reverse('allocation-change', kwargs=allocation_pk_dict),
+ }
+ if (
+ 'coldfront.plugins.sftocf' in settings.INSTALLED_APPS
+ and 'tape' not in resource.name
+ and 'tier' in resource.name
+ and allocation_obj.project.sf_zone
+ ):
+ other_vars['starfish'] = True
+ STARFISH_SERVER = import_from_settings('STARFISH_SERVER')
+ starfish_url = f'https://{STARFISH_SERVER}.rc.fas.harvard.edu/api/'
+ other_vars['starfish_url'] = starfish_url
+ other_vars['starfish_docs_url'] = 'https://docs.rc.fas.harvard.edu/kb/starfish-data-management/'
+ subject = f'Allocation Usage Warning for {allocation_obj.project.title} on {other_vars["resource"]}'
+ send_allocation_manager_email(
+ allocation_obj,
+ subject,
+ 'email/allocation_usage_high.txt',
+ url_path=reverse('allocation-detail', kwargs=allocation_pk_dict),
+ manager_types=DATA_MANAGERS,
+ other_vars=other_vars
+ )
+ logger.info('allocation %s %s reaching capacity; warning email sent',
+ allocation_obj.allocation.pk, allocation_obj.allocation)
+ return True
+ return False
+
+
 def id_present_missing_resources(resourceserver_list):
  """
  Collect all Resource entries with resources in param resourceserver_list;
@@ -202,6 +260,7 @@ def log_missing(modelname, missing):
  """
  update_csv(missing, MISSING_DATA_DIR, f'missing_{modelname}s.csv')
 
+
 def slate_for_check(log_entries):
  """Add an issue encountered during runtime to a CSV for administrative review.
 
@@ -214,6 +273,7 @@ def slate_for_check(log_entries):
  """
  update_csv(log_entries, 'local_data/', 'program_error_checks.csv')
 
+
 def update_csv(new_entries, dirpath, csv_name, date_update='date'):
  """Add or update entries in CSV, order CSV by descending date and save.
 

coldfront/core/utils/mail.py

@@ -49,6 +49,8 @@ def send_email(subject, body, sender, receiver_list, cc=[]):
 
  try:
  email = EmailMessage(subject, body, sender, receiver_list, cc=cc)
+ logger.info('Email sent to %s from %s with subject %s',
+ ','.join(receiver_list+cc), sender, subject)
  email.send(fail_silently=False)
  except SMTPException:
  logger.error('Failed to send email to %s from %s with subject %s',
@@ -84,9 +86,9 @@ def build_link(url_path, domain_url=''):
 
 def send_allocation_admin_email(
  allocation_obj, subject, template_name,
- url_path='', domain_url='', other_vars=None
+ url_path='', domain_url='', cc=[], other_vars=None
 ):
- """Send allocation admin emails
+ """Send allocation-related email to system admins
  """
  url_path = url_path or reverse('allocation-request-list')
 
@@ -101,7 +103,6 @@ def send_allocation_admin_email(
  ctx['resource'] = resource_name
  ctx['url'] = url
 
- cc = []
  if ctx.get('user'):
  cc.append(ctx.get('user').email)
  send_email_template(
@@ -113,6 +114,40 @@ def send_allocation_admin_email(
  cc=cc
  )
 
+def send_allocation_manager_email(
+ allocation_obj, subject, template_name,
+ url_path='', manager_types=[], domain_url='', other_vars=None
+):
+ """Send allocation-related email to allocation pi, cc'ing managers
+ """
+ url_path = url_path or reverse('allocation-request-list')
+
+ url = build_link(url_path, domain_url=domain_url)
+ pi = allocation_obj.project.pi
+ pi_name = f'{pi.first_name} {pi.last_name}'
+ resource_name = allocation_obj.get_parent_resource
+
+ ctx = email_template_context(other_vars)
+ ctx['pi_name'] = pi_name
+ ctx['pi_username'] = f'{pi.username}'
+ ctx['resource'] = resource_name
+ ctx['url'] = url
+
+ cc = []
+ if manager_types:
+ managers = allocation_obj.project.projectuser_set.filter(
+ role__name__in=manager_types)
+ cc.extend([manager.user.email for manager in managers])
+ send_email_template(
+ f'{subject}: {pi_name} - {resource_name}',
+ template_name,
+ ctx,
+ EMAIL_SENDER,
+ [pi.email],
+ cc=cc
+ )
+
+
 def send_allocation_customer_email(
  allocation_obj, subject, template_name,
  url_path='', domain_url=''

coldfront/core/utils/tests.py

@@ -14,11 +14,15 @@
  build_link,
  logger
 )
+from coldfront.core.utils.fasrc import allocation_reaching_capacity_operations
 
 @patch('coldfront.core.utils.mail.EMAIL_ENABLED', True)
 @patch('coldfront.config.email.EMAIL_BACKEND', 'django.core.mail.backends.locmem.EmailBackend')
 @patch('coldfront.core.utils.mail.EMAIL_SENDER', 'test-admin@coldfront.org')
 @patch('coldfront.core.utils.mail.EMAIL_TICKET_SYSTEM_ADDRESS', 'tickets@example.org')
+@patch('coldfront.core.utils.mail.EMAIL_CENTER_NAME', 'HPC Center')
+@patch('coldfront.core.utils.mail.EMAIL_SIGNATURE', 'HPC Center Team')
+# @patch('coldfront.config.base.INSTALLED_APPS', 'coldfront.plugins')
 class EmailFunctionsTestCase(TestCase):
 
  def setUp(self):
@@ -107,12 +111,29 @@ def test_build_link(self):
  self.assertEqual(build_link(url_path, domain_url), expected_url)
  self.assertEqual(build_link(url_path), f'{CENTER_BASE_URL}{url_path}')
 
+ def test_allocation_reaching_capacity_operations(self):
+ allocation_obj = MagicMock()
+ allocation_obj.pk = 1
+ allocation_obj.project.title = 'Test Project'
+ allocation_obj.project.pi.first_name = 'John'
+ allocation_obj.project.pi.last_name = 'Doe'
+ allocation_obj.project.pi.username = 'jdoe'
+ allocation_obj.project.pi.email = 'jdoe@test_project.edu'
+ allocation_obj.get_parent_resource.name = 'Test Resource'
+ allocation_obj.size = 140140566725200
+ allocation_obj.usage = 100
+ new_usage = 140140566625100
+ result = allocation_reaching_capacity_operations(allocation_obj, new_usage)
+ self.assertEqual(result, True)
+ self.assertEqual(len(mail.outbox), 1)
+ print(mail.outbox[0].__dict__)
+
  def test_send_allocation_admin_email(self):
  allocation_obj = MagicMock()
  allocation_obj.project.pi.first_name = 'John'
  allocation_obj.project.pi.last_name = 'Doe'
  allocation_obj.project.pi.username = 'jdoe'
- allocation_obj.get_parent_resource = 'Test Resource'
+ allocation_obj.get_parent_resource.name = 'Test Resource'
  send_allocation_admin_email(allocation_obj, self.subject, self.template_name)
  self.assertEqual(len(mail.outbox), 1)
 

coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py

@@ -76,7 +76,8 @@ def handle(self, *args, **options):
 
  resource = Resource.objects.get(name__contains=entry['server'])
  alloc_obj = select_one_project_allocation(project, resource, dirpath=lab_path)
- if alloc_obj is not None:
+ if alloc_obj == 'MultiAllocationError':
+ logger.warning('Multiple allocations found for %s %s %s', lab_name, lab_server, lab_path)
  continue
  lab_usage_entries = [
  i for i in allocation_usages if i['vol_name'] == lab_server

coldfront/templates/email/allocation_usage_high.txt

@@ -0,0 +1,14 @@
+This is a notice that usage of {{project_title}}’s allocation on {{resource}} has exceeded {{threshold}}% of its quota of {{allocation_quota}}.
+You can view more information about the allocation here: {{url}}.
+
+To avoid reaching quota capacity, we strongly encourage either reducing allocation usage or requesting an allocation quota increase.
+You can request an allocation quota increase here: {{change_request_url}}.
+{% if starfish %}
+You can view files and directories and identify those that have not been used recently for this allocation via the starfish platform. 
+You can access the starfish platform here: {{starfish_url}}.
+
+Some tips for using starfish to identify files that have not been accessed recently or may no longer be in use are available here: {{starfish_docs_url}}
+{% endif %}
+
+Thank you,
+{{signature}}