From aff429acd5c52eba2ba2917a777439e8e5765449 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 13 Aug 2024 14:17:50 -0400 Subject: [PATCH] add email template and alert functions --- .../commands/add_resource_defaults.py | 2 +- coldfront/core/utils/fasrc.py | 72 +++++++++++++++++-- coldfront/core/utils/mail.py | 41 ++++++++++- coldfront/core/utils/tests.py | 23 +++++- .../commands/id_import_new_allocations.py | 3 +- .../templates/email/allocation_usage_high.txt | 14 ++++ 6 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 coldfront/templates/email/allocation_usage_high.txt diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py index 4c3758c0c..16e772014 100644 --- a/coldfront/core/resource/management/commands/add_resource_defaults.py +++ b/coldfront/core/resource/management/commands/add_resource_defaults.py @@ -114,7 +114,7 @@ def handle(self, *args, **options): defaults={'value': default_value} ) - quantity_label = "Quantity in TB" + quantity_label = "TB" if default_value == 20: quantity_label += " in 20T increments" diff --git a/coldfront/core/utils/fasrc.py b/coldfront/core/utils/fasrc.py index d63cd7d0b..2929adc59 100644 --- a/coldfront/core/utils/fasrc.py +++ b/coldfront/core/utils/fasrc.py @@ -2,22 +2,27 @@ """ import os import json +import logging import operator from functools import reduce from datetime import datetime import pandas as pd +from django.conf import settings from django.db.models import Q from django.contrib.auth import get_user_model +from django.urls import reverse from ifxbilling.models import Product from coldfront.core.utils.common import import_from_settings from coldfront.core.project.models import Project from coldfront.core.resource.models import Resource +from coldfront.core.utils.mail import send_allocation_manager_email +logger = logging.getLogger(__name__) -MISSING_DATA_DIR = './local_data/missing/' - +MISSING_DATA_DIR = import_from_settings('MISSING_DATA_DIR', './local_data/missing/') +DATA_MANAGERS = import_from_settings('DATA_MANAGERS', ['General Manager, Data Manager']) username_ignore_list = import_from_settings('username_ignore_list', []) groupname_ignore_list = import_from_settings('groupname_ignore_list', []) @@ -29,6 +34,7 @@ def get_quarter_start_end(): quarter = (datetime.today().month-1)//3 return (quarter_starts[quarter], quarter_ends[quarter]) + def sort_by(list1, sorter, how='attr'): """split one list into two on basis of each item's ability to meet a condition Parameters @@ -50,6 +56,7 @@ def sort_by(list1, sorter, how='attr'): raise Exception('unclear sorting method') return is_true, is_false + def select_one_project_allocation(project_obj, resource_obj, dirpath=None): """ Get one allocation for a given project/resource pairing; handle return of @@ -69,13 +76,15 @@ def select_one_project_allocation(project_obj, resource_obj, dirpath=None): allocation_query = project_obj.allocation_set.filter(**filter_vals) if allocation_query.count() == 1: allocation_obj = allocation_query.first() - if allocation_obj.path and dirpath and allocation_obj.path not in dirpath and dirpath not in allocation_obj.path: - return None + if allocation_obj.subdirectory and allocation_obj.subdirectory != dirpath: + logger.error("directory path mismatch:", allocation_obj.subdirectory, dirpath) elif allocation_query.count() < 1: allocation_obj = None elif allocation_query.count() > 1: - allocation_obj = next((a for a in allocation_query if a.path.lower() in dirpath.lower()), - None) + allocation_obj = next( + (a for a in allocation_query if a.path.lower() in dirpath.lower()), + 'MultiAllocationError' + ) return allocation_obj @@ -90,6 +99,7 @@ def determine_size_fmt(byte_num): byte_num /= 1024.0 return(round(byte_num, 3), unit) + def convert_size_fmt(num, target_unit, source_unit='B'): units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB'] diff = units.index(target_unit) - units.index(source_unit) @@ -103,6 +113,7 @@ def convert_size_fmt(num, target_unit, source_unit='B'): num*=1024.0 return round(num, 3) + def get_resource_rate(resource): """find Product with the name provided and return the associated rate""" try: @@ -121,6 +132,53 @@ def get_resource_rate(resource): price = convert_size_fmt(rate_obj.price, 'TB', source_unit=rate_obj.units) return round(price/100, 2) + +def allocation_reaching_capacity_operations(allocation_obj, new_usage): + """if allocation_obj.usage is <80% of allocation_obj.limit and new_usage + >80% of allocation_obj.limit, send email to pi and data manager. + """ + threshold = 80 + if ( + allocation_obj.usage_exact/allocation_obj.size_exact < 0.8 + and new_usage/allocation_obj.size > 0.8 + ): + resource = allocation_obj.get_parent_resource + # send email to pi and data manager + # define: center_name, threshold, resource, allocation_url, request_allocation_url, starfish_url, starfish_docs_url, signature + allocation_pk_dict = {'pk': allocation_obj.pk} + other_vars = { + 'threshold': threshold, + 'project_title': allocation_obj.project.title, + 'allocation_quota': f'{allocation_obj.size} {allocation_obj.get_parent_resource.quantity_label}', + 'resource': resource.name, + 'change_request_url': reverse('allocation-change', kwargs=allocation_pk_dict), + } + if ( + 'coldfront.plugins.sftocf' in settings.INSTALLED_APPS + and 'tape' not in resource.name + and 'tier' in resource.name + and allocation_obj.project.sf_zone + ): + other_vars['starfish'] = True + STARFISH_SERVER = import_from_settings('STARFISH_SERVER') + starfish_url = f'https://{STARFISH_SERVER}.rc.fas.harvard.edu/api/' + other_vars['starfish_url'] = starfish_url + other_vars['starfish_docs_url'] = 'https://docs.rc.fas.harvard.edu/kb/starfish-data-management/' + subject = f'Allocation Usage Warning for {allocation_obj.project.title} on {other_vars["resource"]}' + send_allocation_manager_email( + allocation_obj, + subject, + 'email/allocation_usage_high.txt', + url_path=reverse('allocation-detail', kwargs=allocation_pk_dict), + manager_types=DATA_MANAGERS, + other_vars=other_vars + ) + logger.info('allocation %s %s reaching capacity; warning email sent', + allocation_obj.allocation.pk, allocation_obj.allocation) + return True + return False + + def id_present_missing_resources(resourceserver_list): """ Collect all Resource entries with resources in param resourceserver_list; @@ -202,6 +260,7 @@ def log_missing(modelname, missing): """ update_csv(missing, MISSING_DATA_DIR, f'missing_{modelname}s.csv') + def slate_for_check(log_entries): """Add an issue encountered during runtime to a CSV for administrative review. @@ -214,6 +273,7 @@ def slate_for_check(log_entries): """ update_csv(log_entries, 'local_data/', 'program_error_checks.csv') + def update_csv(new_entries, dirpath, csv_name, date_update='date'): """Add or update entries in CSV, order CSV by descending date and save. diff --git a/coldfront/core/utils/mail.py b/coldfront/core/utils/mail.py index 1885d0037..2a9ce367e 100644 --- a/coldfront/core/utils/mail.py +++ b/coldfront/core/utils/mail.py @@ -49,6 +49,8 @@ def send_email(subject, body, sender, receiver_list, cc=[]): try: email = EmailMessage(subject, body, sender, receiver_list, cc=cc) + logger.info('Email sent to %s from %s with subject %s', + ','.join(receiver_list+cc), sender, subject) email.send(fail_silently=False) except SMTPException: logger.error('Failed to send email to %s from %s with subject %s', @@ -84,9 +86,9 @@ def build_link(url_path, domain_url=''): def send_allocation_admin_email( allocation_obj, subject, template_name, - url_path='', domain_url='', other_vars=None + url_path='', domain_url='', cc=[], other_vars=None ): - """Send allocation admin emails + """Send allocation-related email to system admins """ url_path = url_path or reverse('allocation-request-list') @@ -101,7 +103,6 @@ def send_allocation_admin_email( ctx['resource'] = resource_name ctx['url'] = url - cc = [] if ctx.get('user'): cc.append(ctx.get('user').email) send_email_template( @@ -113,6 +114,40 @@ def send_allocation_admin_email( cc=cc ) +def send_allocation_manager_email( + allocation_obj, subject, template_name, + url_path='', manager_types=[], domain_url='', other_vars=None +): + """Send allocation-related email to allocation pi, cc'ing managers + """ + url_path = url_path or reverse('allocation-request-list') + + url = build_link(url_path, domain_url=domain_url) + pi = allocation_obj.project.pi + pi_name = f'{pi.first_name} {pi.last_name}' + resource_name = allocation_obj.get_parent_resource + + ctx = email_template_context(other_vars) + ctx['pi_name'] = pi_name + ctx['pi_username'] = f'{pi.username}' + ctx['resource'] = resource_name + ctx['url'] = url + + cc = [] + if manager_types: + managers = allocation_obj.project.projectuser_set.filter( + role__name__in=manager_types) + cc.extend([manager.user.email for manager in managers]) + send_email_template( + f'{subject}: {pi_name} - {resource_name}', + template_name, + ctx, + EMAIL_SENDER, + [pi.email], + cc=cc + ) + + def send_allocation_customer_email( allocation_obj, subject, template_name, url_path='', domain_url='' diff --git a/coldfront/core/utils/tests.py b/coldfront/core/utils/tests.py index 3f0fa4dc8..1a626c382 100644 --- a/coldfront/core/utils/tests.py +++ b/coldfront/core/utils/tests.py @@ -14,11 +14,15 @@ build_link, logger ) +from coldfront.core.utils.fasrc import allocation_reaching_capacity_operations @patch('coldfront.core.utils.mail.EMAIL_ENABLED', True) @patch('coldfront.config.email.EMAIL_BACKEND', 'django.core.mail.backends.locmem.EmailBackend') @patch('coldfront.core.utils.mail.EMAIL_SENDER', 'test-admin@coldfront.org') @patch('coldfront.core.utils.mail.EMAIL_TICKET_SYSTEM_ADDRESS', 'tickets@example.org') +@patch('coldfront.core.utils.mail.EMAIL_CENTER_NAME', 'HPC Center') +@patch('coldfront.core.utils.mail.EMAIL_SIGNATURE', 'HPC Center Team') +# @patch('coldfront.config.base.INSTALLED_APPS', 'coldfront.plugins') class EmailFunctionsTestCase(TestCase): def setUp(self): @@ -107,12 +111,29 @@ def test_build_link(self): self.assertEqual(build_link(url_path, domain_url), expected_url) self.assertEqual(build_link(url_path), f'{CENTER_BASE_URL}{url_path}') + def test_allocation_reaching_capacity_operations(self): + allocation_obj = MagicMock() + allocation_obj.pk = 1 + allocation_obj.project.title = 'Test Project' + allocation_obj.project.pi.first_name = 'John' + allocation_obj.project.pi.last_name = 'Doe' + allocation_obj.project.pi.username = 'jdoe' + allocation_obj.project.pi.email = 'jdoe@test_project.edu' + allocation_obj.get_parent_resource.name = 'Test Resource' + allocation_obj.size = 140140566725200 + allocation_obj.usage = 100 + new_usage = 140140566625100 + result = allocation_reaching_capacity_operations(allocation_obj, new_usage) + self.assertEqual(result, True) + self.assertEqual(len(mail.outbox), 1) + print(mail.outbox[0].__dict__) + def test_send_allocation_admin_email(self): allocation_obj = MagicMock() allocation_obj.project.pi.first_name = 'John' allocation_obj.project.pi.last_name = 'Doe' allocation_obj.project.pi.username = 'jdoe' - allocation_obj.get_parent_resource = 'Test Resource' + allocation_obj.get_parent_resource.name = 'Test Resource' send_allocation_admin_email(allocation_obj, self.subject, self.template_name) self.assertEqual(len(mail.outbox), 1) diff --git a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py index 4f933aade..b65beebb4 100644 --- a/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py +++ b/coldfront/plugins/fasrc/management/commands/id_import_new_allocations.py @@ -76,7 +76,8 @@ def handle(self, *args, **options): resource = Resource.objects.get(name__contains=entry['server']) alloc_obj = select_one_project_allocation(project, resource, dirpath=lab_path) - if alloc_obj is not None: + if alloc_obj == 'MultiAllocationError': + logger.warning('Multiple allocations found for %s %s %s', lab_name, lab_server, lab_path) continue lab_usage_entries = [ i for i in allocation_usages if i['vol_name'] == lab_server diff --git a/coldfront/templates/email/allocation_usage_high.txt b/coldfront/templates/email/allocation_usage_high.txt new file mode 100644 index 000000000..46a63828c --- /dev/null +++ b/coldfront/templates/email/allocation_usage_high.txt @@ -0,0 +1,14 @@ +This is a notice that usage of {{project_title}}’s allocation on {{resource}} has exceeded {{threshold}}% of its quota of {{allocation_quota}}. +You can view more information about the allocation here: {{url}}. + +To avoid reaching quota capacity, we strongly encourage either reducing allocation usage or requesting an allocation quota increase. +You can request an allocation quota increase here: {{change_request_url}}. +{% if starfish %} +You can view files and directories and identify those that have not been used recently for this allocation via the starfish platform. +You can access the starfish platform here: {{starfish_url}}. + +Some tips for using starfish to identify files that have not been accessed recently or may no longer be in use are available here: {{starfish_docs_url}} +{% endif %} + +Thank you, +{{signature}}