DRAFT: allocation fullness email alerts

coldfront/core/allocation/views.py

@@ -68,7 +68,7 @@
 from coldfront.core.project.models import (Project, ProjectPermission,
  ProjectUserStatusChoice)
 from coldfront.core.resource.models import Resource
-from coldfront.core.utils.common import get_domain_url, import_from_settings
+from coldfront.core.utils.common import import_from_settings
 from coldfront.core.utils.mail import send_allocation_admin_email, send_allocation_customer_email
 
 
@@ -393,8 +393,7 @@ def post(self, request, *args, **kwargs):
  )
 
  send_allocation_customer_email(
- allocation_obj, 'Allocation Activated',
- 'email/allocation_activated.txt', domain_url=get_domain_url(self.request)
+ allocation_obj, 'Allocation Activated', 'email/allocation_activated.txt'
  )
  if action == 'approve':
  messages.success(request, 'Allocation Activated!')
@@ -425,7 +424,6 @@ def post(self, request, *args, **kwargs):
  allocation_obj,
  f'Allocation {allocation_obj.status.name}',
  f'email/allocation_{allocation_obj.status.name.lower()}.txt',
- domain_url=get_domain_url(self.request),
  )
  messages.success(request, f'Allocation {allocation_obj.status.name}!')
  else:
@@ -748,7 +746,6 @@ def form_valid(self, form):
  allocation_obj,
  'New Allocation Request',
  'email/new_allocation_request.txt',
- domain_url=get_domain_url(self.request),
  url_path=reverse('allocation-detail', kwargs={'pk': allocation_obj.pk}),
  other_vars=other_vars,
  )
@@ -1420,7 +1417,6 @@ def post(self, request, *args, **kwargs):
  allocation_obj,
  'Allocation Renewed',
  'email/allocation_renewed.txt',
- domain_url=get_domain_url(self.request),
  )
 
  messages.success(request, 'Allocation renewed successfully')
@@ -1946,7 +1942,6 @@ def post(self, request, *args, **kwargs):
  alloc_change_obj.allocation,
  'Allocation Change Denied',
  'email/allocation_change_denied.txt',
- domain_url=get_domain_url(self.request),
  )
  save_and_redirect = True
 
@@ -2056,7 +2051,6 @@ def post(self, request, *args, **kwargs):
  alloc_change_obj.allocation,
  'Allocation Change Approved',
  'email/allocation_change_approved.txt',
- domain_url=get_domain_url(self.request),
  )
 
  message = make_allocation_change_message(alloc_change_obj, 'APPROVED')
@@ -2293,7 +2287,6 @@ def post(self, request, *args, **kwargs):
  'allocation-change-detail',
  kwargs={'pk': allocation_change_request_obj.pk},
  ),
- domain_url=get_domain_url(self.request),
  other_vars=email_vars,
  )
  return HttpResponseRedirect(reverse('allocation-detail', kwargs={'pk': pk}))

coldfront/core/resource/management/commands/add_resource_defaults.py

@@ -114,7 +114,7 @@ def handle(self, *args, **options):
  defaults={'value': default_value}
  )
 
- quantity_label = "Quantity in TB"
+ quantity_label = "TB"
  if default_value == 20:
  quantity_label += " in 20T increments"
 

coldfront/core/utils/fasrc.py

@@ -2,22 +2,27 @@
 """
 import os
 import json
+import logging
 import operator
 from functools import reduce
 from datetime import datetime
 
 import pandas as pd
+from django.conf import settings
 from django.db.models import Q
 from django.contrib.auth import get_user_model
+from django.urls import reverse
 from ifxbilling.models import Product
 
 from coldfront.core.utils.common import import_from_settings
 from coldfront.core.project.models import Project
 from coldfront.core.resource.models import Resource
+from coldfront.core.utils.mail import send_allocation_manager_email, build_link
 
+logger = logging.getLogger(__name__)
 
-MISSING_DATA_DIR = './local_data/missing/'
-
+MISSING_DATA_DIR = import_from_settings('MISSING_DATA_DIR', './local_data/missing/')
+DATA_MANAGERS = import_from_settings('DATA_MANAGERS', ['General Manager, Storage Manager'])
 username_ignore_list = import_from_settings('username_ignore_list', [])
 groupname_ignore_list = import_from_settings('groupname_ignore_list', [])
 
@@ -29,6 +34,7 @@ def get_quarter_start_end():
  quarter = (datetime.today().month-1)//3
  return (quarter_starts[quarter], quarter_ends[quarter])
 
+
 def sort_by(list1, sorter, how='attr'):
  """split one list into two on basis of each item's ability to meet a condition
  Parameters
@@ -50,6 +56,7 @@ def sort_by(list1, sorter, how='attr'):
  raise Exception('unclear sorting method')
  return is_true, is_false
 
+
 def select_one_project_allocation(project_obj, resource_obj, dirpath=None):
  """
  Get one allocation for a given project/resource pairing; handle return of
@@ -69,13 +76,15 @@ def select_one_project_allocation(project_obj, resource_obj, dirpath=None):
  allocation_query = project_obj.allocation_set.filter(**filter_vals)
  if allocation_query.count() == 1:
  allocation_obj = allocation_query.first()
- if allocation_obj.path and dirpath and allocation_obj.path not in dirpath and dirpath not in allocation_obj.path:
- return None
+ if allocation_obj.path and allocation_obj.path != dirpath:
+ logger.error("directory path mismatch:", allocation_obj.path, dirpath)
  elif allocation_query.count() < 1:
  allocation_obj = None
  elif allocation_query.count() > 1:
- allocation_obj = next((a for a in allocation_query if a.path.lower() in dirpath.lower()),
- None)
+ allocation_obj = next(
+ (a for a in allocation_query if a.path.lower() in dirpath.lower()),
+ 'MultiAllocationError'
+ )
  return allocation_obj
 
 
@@ -90,6 +99,7 @@ def determine_size_fmt(byte_num):
  byte_num /= 1024.0
  return(round(byte_num, 3), unit)
 
+
 def convert_size_fmt(num, target_unit, source_unit='B'):
  units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB']
  diff = units.index(target_unit) - units.index(source_unit)
@@ -103,6 +113,7 @@ def convert_size_fmt(num, target_unit, source_unit='B'):
  num*=1024.0
  return round(num, 3)
 
+
 def get_resource_rate(resource):
  """find Product with the name provided and return the associated rate"""
  try:
@@ -121,6 +132,58 @@ def get_resource_rate(resource):
  price = convert_size_fmt(rate_obj.price, 'TB', source_unit=rate_obj.units)
  return round(price/100, 2)
 
+
+def allocation_reaching_capacity_operations(allocation_obj, new_byte_usage):
+ """if allocation_obj.usage is <80/90% of allocation_obj.limit and new_byte_usage
+ >80/90% of allocation_obj.limit, send email to pi and data manager.
+ """
+ threshold = None
+ size_bytes = float(allocation_obj.size_exact)
+ for limit in [80, 90]:
+ if (
+ allocation_obj.usage_exact/size_bytes < limit/100
+ and new_byte_usage/size_bytes > limit/100
+ ):
+ threshold = limit
+ if threshold:
+ resource = allocation_obj.get_parent_resource
+ # send email to pi and data manager
+ # define: center_name, threshold, resource, allocation_url, request_allocation_url, starfish_url, starfish_docs_url, signature
+ allocation_pk_dict = {'pk': allocation_obj.pk}
+ other_vars = {
+ 'threshold': threshold,
+ 'project_title': allocation_obj.project.title,
+ 'allocation_quota': f'{allocation_obj.size} {allocation_obj.get_parent_resource.quantity_label}',
+ 'resource': resource.name,
+ 'change_request_url': build_link(reverse('allocation-change', kwargs=allocation_pk_dict)),
+ }
+ if (
+ 'coldfront.plugins.sftocf' in settings.INSTALLED_APPS
+ and 'tape' not in resource.name
+ and 'tier' in resource.name
+ and allocation_obj.project.sf_zone
+ ):
+ other_vars['starfish'] = True
+ STARFISH_SERVER = import_from_settings('STARFISH_SERVER')
+ starfish_url = f'https://{STARFISH_SERVER}.rc.fas.harvard.edu'
+ other_vars['starfish_url'] = starfish_url
+ other_vars['starfish_docs_url'] = 'https://docs.rc.fas.harvard.edu/kb/starfish-data-management/'
+ subject = f'Allocation Usage Warning for {allocation_obj.project.title} on {other_vars["resource"]}'
+ send_allocation_manager_email(
+ allocation_obj,
+ subject,
+ 'email/allocation_usage_high.txt',
+ url_path=reverse('allocation-detail', kwargs=allocation_pk_dict),
+ manager_types=DATA_MANAGERS,
+ other_vars=other_vars
+ )
+ msg = f"email sent for allocation {allocation_obj.pk} - old usage {allocation_obj.usage_exact} ({allocation_obj.usage_exact/size_bytes}%), new usage {new_byte_usage} ({new_byte_usage/size_bytes}%), quota {allocation_obj.size_exact}"
+ logger.info(msg)
+ print(msg)
+ return True
+ return False
+
+
 def id_present_missing_resources(resourceserver_list):
  """
  Collect all Resource entries with resources in param resourceserver_list;
@@ -202,6 +265,7 @@ def log_missing(modelname, missing):
  """
  update_csv(missing, MISSING_DATA_DIR, f'missing_{modelname}s.csv')
 
+
 def slate_for_check(log_entries):
  """Add an issue encountered during runtime to a CSV for administrative review.
 
@@ -214,6 +278,7 @@ def slate_for_check(log_entries):
  """
  update_csv(log_entries, 'local_data/', 'program_error_checks.csv')
 
+
 def update_csv(new_entries, dirpath, csv_name, date_update='date'):
  """Add or update entries in CSV, order CSV by descending date and save.
 

coldfront/core/utils/mail.py

@@ -49,6 +49,8 @@ def send_email(subject, body, sender, receiver_list, cc=[]):
 
  try:
  email = EmailMessage(subject, body, sender, receiver_list, cc=cc)
+ logger.info('Email sent to %s from %s with subject %s',
+ ','.join(receiver_list+cc), sender, subject)
  email.send(fail_silently=False)
  except SMTPException:
  logger.error('Failed to send email to %s from %s with subject %s',
@@ -84,9 +86,9 @@ def build_link(url_path, domain_url=''):
 
 def send_allocation_admin_email(
  allocation_obj, subject, template_name,
- url_path='', domain_url='', other_vars=None
+ url_path='', domain_url='', cc=[], other_vars=None
 ):
- """Send allocation admin emails
+ """Send allocation-related email to system admins
  """
  url_path = url_path or reverse('allocation-request-list')
 
@@ -101,7 +103,6 @@ def send_allocation_admin_email(
  ctx['resource'] = resource_name
  ctx['url'] = url
 
- cc = []
  if ctx.get('user'):
  cc.append(ctx.get('user').email)
  send_email_template(
@@ -113,6 +114,40 @@ def send_allocation_admin_email(
  cc=cc
  )
 
+def send_allocation_manager_email(
+ allocation_obj, subject, template_name,
+ url_path='', manager_types=[], domain_url='', other_vars=None
+):
+ """Send allocation-related email to allocation pi, cc'ing managers
+ """
+ url_path = url_path or reverse('allocation-request-list')
+
+ url = build_link(url_path, domain_url=domain_url)
+ pi = allocation_obj.project.pi
+ pi_name = f'{pi.first_name} {pi.last_name}'
+ resource_name = allocation_obj.get_parent_resource
+
+ ctx = email_template_context(other_vars)
+ ctx['pi_name'] = pi_name
+ ctx['pi_username'] = f'{pi.username}'
+ ctx['resource'] = resource_name
+ ctx['url'] = url
+
+ cc = []
+ if manager_types:
+ managers = allocation_obj.project.projectuser_set.filter(
+ role__name__in=manager_types)
+ cc.extend([manager.user.email for manager in managers])
+ send_email_template(
+ f'{subject}: {pi_name} - {resource_name}',
+ template_name,
+ ctx,
+ EMAIL_SENDER,
+ [pi.email],
+ cc=cc
+ )
+
+
 def send_allocation_customer_email(
  allocation_obj, subject, template_name,
  url_path='', domain_url=''

coldfront/core/utils/tests.py

@@ -10,15 +10,19 @@
  email_template_context,
  send_allocation_admin_email,
  send_allocation_customer_email,
- CENTER_BASE_URL,
  build_link,
  logger
 )
+from coldfront.core.utils.fasrc import allocation_reaching_capacity_operations
 
 @patch('coldfront.core.utils.mail.EMAIL_ENABLED', True)
 @patch('coldfront.config.email.EMAIL_BACKEND', 'django.core.mail.backends.locmem.EmailBackend')
 @patch('coldfront.core.utils.mail.EMAIL_SENDER', 'test-admin@coldfront.org')
 @patch('coldfront.core.utils.mail.EMAIL_TICKET_SYSTEM_ADDRESS', 'tickets@example.org')
+@patch('coldfront.core.utils.mail.EMAIL_CENTER_NAME', 'HPC Center')
+@patch('coldfront.core.utils.mail.CENTER_BASE_URL', 'https://centerbaseexampleurl.org')
+@patch('coldfront.core.utils.mail.EMAIL_SIGNATURE', 'HPC Center Team')
+# @patch('coldfront.config.base.INSTALLED_APPS', 'coldfront.plugins')
 class EmailFunctionsTestCase(TestCase):
 
  def setUp(self):
@@ -105,14 +109,31 @@ def test_build_link(self):
  domain_url = 'https://example.com'
  expected_url = f'{domain_url}{url_path}'
  self.assertEqual(build_link(url_path, domain_url), expected_url)
- self.assertEqual(build_link(url_path), f'{CENTER_BASE_URL}{url_path}')
+ self.assertEqual(build_link(url_path), f'https://centerbaseexampleurl.org{url_path}')
+
+ def test_allocation_reaching_capacity_operations(self):
+ allocation_obj = MagicMock()
+ allocation_obj.pk = 1
+ allocation_obj.project.title = 'Test Project'
+ allocation_obj.project.pi.first_name = 'John'
+ allocation_obj.project.pi.last_name = 'Doe'
+ allocation_obj.project.pi.username = 'jdoe'
+ allocation_obj.project.pi.email = 'jdoe@test_project.edu'
+ allocation_obj.get_parent_resource.name = 'Test Resource'
+ allocation_obj.size_exact = 140140566725200
+ allocation_obj.usage_exact = 100
+ new_usage = 140140566625100
+ result = allocation_reaching_capacity_operations(allocation_obj, new_usage)
+ self.assertEqual(result, True)
+ self.assertEqual(len(mail.outbox), 1)
+ print(mail.outbox[0].__dict__)
 
  def test_send_allocation_admin_email(self):
  allocation_obj = MagicMock()
  allocation_obj.project.pi.first_name = 'John'
  allocation_obj.project.pi.last_name = 'Doe'
  allocation_obj.project.pi.username = 'jdoe'
- allocation_obj.get_parent_resource = 'Test Resource'
+ allocation_obj.get_parent_resource.name = 'Test Resource'
  send_allocation_admin_email(allocation_obj, self.subject, self.template_name)
  self.assertEqual(len(mail.outbox), 1)
 

coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py

@@ -76,7 +76,8 @@ def handle(self, *args, **options):
 
  resource = Resource.objects.get(name__contains=entry['server'])
  alloc_obj = select_one_project_allocation(project, resource, dirpath=lab_path)
- if alloc_obj is not None:
+ if alloc_obj == 'MultiAllocationError':
+ logger.warning('Multiple allocations found for %s %s %s', lab_name, lab_server, lab_path)
  continue
  lab_usage_entries = [
  i for i in allocation_usages if i['vol_name'] == lab_server

coldfront/plugins/fasrc/tasks.py

@@ -1,6 +1,6 @@
-from coldfront.plugins.fasrc.utils import pull_push_quota_data
 import logging
 from django.core import management
+from coldfront.plugins.fasrc.utils import pull_push_quota_data
 
 
 def import_quotas(volumes=None):