Merge pull request #534 from mercycorps/staging

Staging

indicators/forms.py

@@ -236,6 +236,10 @@ def __init__(self, *args, **kwargs):
 
         super(IndicatorForm, self).__init__(*args, **kwargs)
 
+        # Making the data_points field readonly for participant count indicators
+        if indicator and indicator.admin_type == Indicator.ADMIN_PARTICIPANT_COUNT:
+            self.fields['data_points'].widget.attrs['readonly'] = True
+
         # per mercycorps/TolaActivity#2452 remove textarea max length validation to provide a more
         # user-friendly js-based validation (these textarea checks aren't enforced at db-level anyway)
         for field in ['name', 'definition', 'justification', 'rationale_for_target',
@@ -475,7 +479,7 @@ def clean(self):
         clean_name = cleaned_data['name']
         clean_admin_type = cleaned_data['admin_type']
         if clean_admin_type != Indicator.ADMIN_PARTICIPANT_COUNT and \
-                clean_name == Indicator.PARTICIPANT_COUNT_INDICATOR_NAME:
+                Indicator.PARTICIPANT_COUNT_INDICATOR_NAME.lower() in clean_name.lower():
             raise ValidationError(
                 # Translators: This is an error message that appears when a user tries to use an off-limits name
                 _('The indicator name you have selected is reserved.  Please enter a different name'))

indicators/management/commands/create_participant_count_indicators.py

@@ -6,7 +6,7 @@
 from django.core.management.base import BaseCommand
 from django.conf import settings
 from django.db import transaction
-
+from dateutil.relativedelta import relativedelta
 from indicators.models import (
     Indicator, DisaggregationType, DisaggregationLabel, OutcomeTheme
 )
@@ -30,6 +30,7 @@ def add_arguments(self, parser):
             '--suppress_output', action='store_true',
             help="Supresses the output so tests don't get too messy")
         parser.add_argument('--clean', action='store_true')
+        parser.add_argument('--delete_pilot_pc_indicators', action='store_true')
 
     @transaction.atomic
     def handle(self, *args, **options):
@@ -55,14 +56,28 @@ def handle(self, *args, **options):
                         theme.save()
             return
 
+        if options['delete_pilot_pc_indicators']:
+            pcind_id_list = [12385, 12053, 12051, 12052, 12050, 12054, 12069, 12068, 12065, 12072, 12063, 12064, 12066,
+                             12071, 12070, 12060, 12057, 12056, 12059, 12058, 12061, 12055, 12062]
+            # Call indicators on an object to object basis to force cascading delete.
+            deleted_ind = 0
+            for pcid in pcind_id_list:
+                if Indicator.objects.filter(pk=pcid).exists():
+                    ind_to_delete = Indicator.objects.get(pk=pcid)
+                    ind_to_delete.delete(force_policy=HARD_DELETE)
+                    deleted_ind += 1
+
+            if not options['suppress_output']:
+                print(f'{deleted_ind} pilot pc indicators deleted, {len(pcind_id_list)-deleted_ind} not found')
+
         if options['create_disaggs_themes']:
             sadd_label_text = 'Age Unknown M, Age Unknown F, Age Unknown Sex Unknown, 0-5 M, 0-5 F, 0-5 Sex Unknown, 6-9 M, 6-9 F, 6-9 Sex Unknown, 10-14 M, 10-14 F, 10-14 Sex Unknown, 15-19 M, 15-19 F, 15-19 Sex Unknown, 20-24 M, 20-24 F, 20-24 Sex Unknown, 25-34 M, 25-34 F, 25-34 Sex Unknown, 35-49 M, 35-49 F, 35-49 Sex Unknown, 50+ M, 50+ F, 50+ Sex Unknown'
             sadd_label_list = sadd_label_text.split(', ')
 
             sector_list = sorted([
                 'Agriculture', 'Cash and Voucher Assistance', 'Environment (DRR, Energy and Water)',
                 'Infrastructure (non - WASH, non - energy)', 'Governance and Partnership', 'Employment', 'WASH',
-                'Financial Services', 'Nutrition', 'Health (non - nutrition)']
+                'Financial Services', 'Nutrition', 'Public Health (non - nutrition, non - WASH)']
             )
 
             actual_disagg_labels = ['Direct', 'Indirect']
@@ -94,7 +109,8 @@ def handle(self, *args, **options):
         counts = {
             'eligible_programs': 0, 'pc_indicator_does_not_exist': 0, 'has_rf': 0, 'indicators_created': 0,}
 
-        reporting_start_date = date.fromisoformat(settings.REPORTING_YEAR_START_DATE)
+        # Subtract one year from the reporting year start date
+        reporting_start_date = date.fromisoformat(settings.REPORTING_YEAR_START_DATE) - relativedelta(years=1)
 
         eligible_programs = Program.objects.filter(reporting_period_end__gte=reporting_start_date)
         counts['eligible_programs'] = eligible_programs.count()

indicators/serializers_new/participant_count_serializers.py

@@ -1,5 +1,7 @@
 import copy
-from rest_framework import serializers
+import re
+from rest_framework import serializers, exceptions
+from django.utils.translation import ugettext_lazy as _
 from django.db.models import Count, BooleanField, Q
 from indicators.models import (
     PeriodicTarget, Result, OutcomeTheme, DisaggregationType, DisaggregationLabel, DisaggregatedValue
@@ -123,17 +125,254 @@ class Meta:
             'disaggregations'
         ]
 
+    def empty_evidence(self):
+        """
+        Utility method for checking if both evidence fields [evidence_url, record_name] are empty.
+
+        Returns
+            True if both fields are empty else False
+        """
+        return self.initial_data.get('record_name') == '' and self.initial_data.get('evidence_url') == '' \
+            or self.initial_data.get('record_name') is None and self.initial_data.get('evidence_url') is None
+
+    def validate_date_collected(self, value):
+        """
+        Validates that date_collected is in the range of self.context.get('program').start_date and self.context.get('program').end_date
+
+        Params
+            value
+                - The value of date_collected
+
+        Raises
+            ValidationError
+                - If validation fails
+
+        Returns
+            Value of date_collected
+        """
+        program = self.context.get('program')
+
+        if program.reporting_period_start <= value <= program.reporting_period_end:
+            return value
+
+        # Translators: An error message detailing that the selected date should be within the reporting period for the fiscal year
+        raise exceptions.ValidationError(_('This date should be within the fiscal year of the reporting period.'))
+
+    def validate_outcome_themes(self, value):
+        """
+        Validates that the length of outcome_themes is greater than 0
+
+        Params
+            value
+                - The value of outcome_themes
+
+        Raises
+            ValidationError
+                - If validation fails
+
+        Returns
+            Value of outcome_themes
+        """
+        if len(value) > 0:
+            return value
+
+        # Translators: An error message detailing that outcome themes are required and that multiple outcome themes can be selected
+        raise exceptions.ValidationError(_('Please complete this field. You can select more than one outcome theme.'))
+
+    def validate_evidence_url(self, value):
+        """
+        Validates that evidence_url matches the regex pattern and that record_name is not None
+
+        Params
+            value
+                - The value of evidence_url
+
+        Raises
+            ValidationError
+                - If validation fails
+
+        Returns
+            Value of evidence_url
+        """
+        pattern = r"^(http(s)?|file):\/\/.+"
+
+        # Both evidence fields are empty. Return value instead of raising exception
+        if self.empty_evidence():
+            return value
+
+        if self.initial_data.get('record_name') is None or len(self.initial_data.get('record_name')) == 0:
+            # Translators: An error message detailing that the record name must be included along the a evidence link
+            raise exceptions.ValidationError(_('A record name must be included along with the link.'))
+
+        if re.match(pattern, value) is None:
+            # Translators: An error message detailing that the evidence link was invalid
+            raise exceptions.ValidationError(_('Please enter a valid evidence link.'))
+
+        return value
+
+    def validate_record_name(self, value):
+        """
+        Validates that record_name is set alongside evidence_url
+
+        Params
+            value
+                - The value of record_name
+        
+        Raises
+            ValidationError
+                - If validation fails
+
+        Returns
+            Value of record_name
+        """
+
+        # Both evidence fields are empty. Return value instead of raising exception
+        if self.empty_evidence():
+            return value
+
+        if self.initial_data.get('evidence_url') is None or len(self.initial_data.get('evidence_url')) == 0:
+            # Translators: An error message detailing that an evidence link must be included along with the record name
+            raise exceptions.ValidationError(_('A link must be included along with the record name.'))
+
+        return value
+
+    def disaggregations_are_valid(self, used_disaggregations):
+        """
+        Checks that each disaggregation passes validation
+
+        Params
+            used_disaggregations
+                - A dict with a total value for each disaggregation type
+
+        Raises
+            ValidationError
+                - If validation fails
+
+        Returns
+            True if validation passes
+        """
+        sadd_with_direct = used_disaggregations['SADD (including unknown) with double counting']['direct']['value']
+        sadd_without_direct = used_disaggregations['SADD (including unknown) without double counting']['direct']['value']
+        actual_with_direct = used_disaggregations['Actual with double counting']['direct']['value']
+        actual_with_indirect = used_disaggregations['Actual with double counting']['indirect']['value']
+        actual_without_direct = used_disaggregations['Actual without double counting']['direct']['value']
+        actual_without_indirect = used_disaggregations['Actual without double counting']['indirect']['value']
+        sectors_direct = used_disaggregations['Sectors Direct with double counting']['direct']['value']
+        sectors_indirect = used_disaggregations['Sectors Indirect with double counting']['indirect']['value']
+
+        if not sadd_with_direct == actual_with_direct:
+            # Translators: An error message detailing that the sum of 'SADD with double counting' should be equal to the sum of 'Direct with double counting'
+            raise exceptions.ValidationError(_("The sum of 'SADD with double counting' should be equal to the sum of 'Direct with double counting'."))
+
+        if not sadd_without_direct == actual_without_direct:
+            # Translators: An error message detailing that the sum of 'SADD without double counting' should be equal to the sum of 'Direct without double counting'
+            raise exceptions.ValidationError(_("The sum of 'SADD without double counting' should be equal to the sum of 'Direct without double counting'."))
+
+        if actual_with_direct == 0 or actual_with_indirect == 0:
+            # Translators: An error message detailing that the fields Direct and Indirect total participants with double counting is required
+            raise exceptions.ValidationError(_("Direct/indirect total participants with double counting is required. Please complete these fields."))
+
+        if (actual_without_direct + actual_without_indirect) > (actual_with_direct + actual_with_indirect):
+            # Translators: An error message detailing that the Direct and Indirect without double counting should be equal to or lower than the value of Direct and Indirect with double counting
+            raise exceptions.ValidationError(_("Direct/indirect without double counting should be equal to or lower than direct/indirect with double counting."))
+
+        if (sectors_direct + sectors_indirect) > (actual_with_direct + actual_with_indirect):
+            # Translators: An error message detailing that the Sector values should be less to or equal to the sum of Direct and Indirect with double counting value
+            raise exceptions.ValidationError(_("Sector values should be less than or equal to the 'Direct/Indirect with double counting' value."))
+
+        return True
+
+    def process_disaggregations(self, disaggregations, instance, preserve_old_ids=False):
+        """
+        Processes disaggregations for creation and also creates the used_disaggregations dict with total values
+
+        Params
+            disaggregations
+                - The disaggregations object from the request
+            instance
+                - instance of the Results object
+            preserve_old_ids
+                - Optional defaults to False
+                - Used for updating in order to get the old_label_ids
+
+        Returns
+            new_value_objs, old_label_ids if preserve_old_ids is True else new_value_objs
+        """
+        new_value_objs = list()
+        old_label_ids = list()
+        # A dict to hold total values for disaggregations
+        used_disaggregations = {
+            "SADD (including unknown) with double counting": {
+                "direct": {
+                    "value": 0
+                }
+            },
+            "SADD (including unknown) without double counting": {
+                "direct": {
+                    "value": 0
+                }
+            },
+            "Sectors Direct with double counting": {
+                "direct": {
+                    "value": 0
+                }
+            },
+            "Sectors Indirect with double counting": {
+                "indirect": {
+                    "value": 0
+                }
+            },
+            "Actual without double counting": {
+                "direct": {
+                    "value": 0
+                },
+                "indirect": {
+                    "value": 0
+                }
+            },
+            "Actual with double counting": {
+                "direct": {
+                    "value": 0
+                },
+                "indirect": {
+                    "value": 0
+                }
+            }
+        }
+
+        for disagg in disaggregations:
+            disagg_type = disagg['disaggregation_type']
+            for label_value in disagg['labels']:
+                if preserve_old_ids:
+                    old_label_ids.append(label_value['disaggregationlabel_id'])
+                if label_value['value']:
+                    if disagg_type in ['Actual with double counting', 'Actual without double counting']:
+                        key = label_value['label'].lower()
+                    else:
+                        key = disagg['count_type'].lower()
+
+                    try:
+                        used_disaggregations[disagg_type][key]['value'] += int(label_value['value'])
+                    except ValueError:
+                        used_disaggregations[disagg_type][key]['value'] += float(label_value['value'])
+
+                    new_value_objs.append(DisaggregatedValue(
+                        category_id=label_value['disaggregationlabel_id'], result=instance, value=label_value['value']))
+
+        if self.disaggregations_are_valid(used_disaggregations):
+            if preserve_old_ids:
+                return new_value_objs, old_label_ids
+            else:
+                return new_value_objs
+
     def create(self, validated_data):
         disaggregations = validated_data.pop('disaggregations')
         outcome_themes = validated_data.pop('outcome_themes')
         result = Result.objects.create(**validated_data)
         result.outcome_themes.add(*outcome_themes)
-        value_objs = []
-        for disagg in disaggregations:
-            for label_value in disagg['labels']:
-                if label_value['value']:
-                    value_objs.append(DisaggregatedValue(
-                        category_id=label_value['disaggregationlabel_id'], result=result, value=label_value['value']))
+
+        value_objs = self.process_disaggregations(disaggregations, result)
+
         # There's lots of warnings about batch_create in the Django documentation, e.g. it skips the
         # save method and won't trigger signals, but I don't we need any of those things in this case.
         DisaggregatedValue.objects.bulk_create(value_objs)
@@ -148,14 +387,7 @@ def update(self, instance, validated_data):
         instance.outcome_themes.remove()
         instance.outcome_themes.add(*outcome_themes)
 
-        new_value_objs = []
-        old_label_ids = []
-        for disagg in disaggregations:
-            for label in disagg['labels']:
-                old_label_ids.append(label['disaggregationlabel_id'])
-                if label['value']:
-                    new_value_objs.append(DisaggregatedValue(
-                        category_id=label['disaggregationlabel_id'], result=instance, value=label['value']))
+        new_value_objs, old_label_ids = self.process_disaggregations(disaggregations, instance, preserve_old_ids=True)
 
         old_disagg_values = DisaggregatedValue.objects.filter(category_id__in=old_label_ids, result=instance)
         old_disagg_values.delete()