fix: apply the cap after normalising the distribution

app/grants/clr.py

@@ -200,7 +200,7 @@ def get_totals_by_pair(contrib_dict):
     return pair_totals
 
 
-def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot, match_cap_per_grant):
+def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot):
     '''
         calculates the clr amount at the given threshold and total pot
         args:
@@ -245,17 +245,13 @@ def calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot, mat
         if type(tot) == complex:
             tot = float(tot.real)
 
-        # ensure CLR match for a grant in CLR round does not exceed 2.5 of the total pot
-        if total_pot != match_cap_per_grant and tot > match_cap_per_grant:
-            tot = match_cap_per_grant
-
         bigtot += tot
-        totals[proj] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}
+        totals[proj] = {'id': proj, 'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}
 
     return bigtot, totals
 
 
-def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant_id, amount):
+def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant_id, amount, match_cap_per_grant):
     '''
         clubbed function that runs all calculation functions and returns the result for a single grant_id
 
@@ -319,10 +315,10 @@ def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_thresho
             if type(tot) == complex:
                 tot = float(tot.real)
 
-            totals[grant_id] = {'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}
+            totals[grant_id] = {'id': grant_id, 'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}
 
             # normalise the result
-            grants_clr = normalise(bigtot + tot, totals, total_pot)
+            grants_clr = normalise(bigtot + tot, totals, total_pot, match_cap_per_grant)
 
         # find grant we added the contribution to and get the new clr amount
         grant_clr = grants_clr.get(grant_id)
@@ -339,7 +335,7 @@ def calculate_clr_for_prediction(bigtot, totals, curr_agg, trust_dict, v_thresho
     return (grants_clr, 0.0, 0, 0.0)
 
 
-def normalise(bigtot, totals, total_pot):
+def normalise(bigtot, totals, total_pot, match_cap_per_grant):
     '''
         given the total amount distributed (bigtot) and the total_pot size normalise the distribution
         args:
@@ -351,8 +347,10 @@ def normalise(bigtot, totals, total_pot):
                 [{'id': proj, 'number_contributions': _num, 'contribution_amount': _sum, 'clr_amount': tot}]
 
     '''
+    # check if saturation is reached
+    is_saturated = bigtot >= total_pot
     # check for saturation and normalise if reached
-    if bigtot >= total_pot:
+    if is_saturated:
         # print(f'saturation reached. Total Pot: ${total_pot} | Total Allocated ${bigtot}. Normalizing')
         for key, t in totals.items():
             t['clr_amount'] = ((t['clr_amount'] / bigtot) * total_pot)
@@ -363,6 +361,48 @@ def normalise(bigtot, totals, total_pot):
         for key, t in totals.items():
             t['clr_amount'] = t['clr_amount'] * (1 + percentage_increase)
 
+    # apply the match cap post-normalisation
+    totals = apply_cap(totals, match_cap_per_grant, is_saturated)
+
+    return totals
+
+
+def apply_cap(totals, match_cap_per_grant, should_spread):
+    # work out how much of the pool is remaining after capping each grant
+    remainder = 0   # amount left to be redistributed after cap
+    uncapped = 0    # total amount matched for grants which haven't capped
+
+    # cap each of the clr_amounts
+    for key, t in totals.items():
+        if t['clr_amount'] >= match_cap_per_grant:
+            # grant has exceeded the cap
+            #  - so cap the clr_amount
+            #  - add the extra funds to remainder
+            remainder += t['clr_amount'] - match_cap_per_grant
+            t['clr_amount'] = match_cap_per_grant
+
+        else:
+            # grant has not exceed cap so add amount to uncapped
+            uncapped += t['clr_amount']
+
+    # check that we have both capped and uncapped grants and that we should be spreading the remainder
+    if should_spread and remainder > 0 and uncapped > 0:
+        # div so we can spread the remainder proportionally
+        per_remainder = remainder / uncapped
+        # reset remainder to check if any grants enter the cap region after spreading the remainder
+        remainder = 0
+        # spread the remainder
+        for key, t in totals.items():
+            if t['clr_amount'] < match_cap_per_grant:
+                t['clr_amount'] += per_remainder * t['clr_amount']
+                # check if the cap is hit after spreading the remainder
+                if t['clr_amount'] >= match_cap_per_grant:
+                    remainder += t['clr_amount'] - match_cap_per_grant
+
+        # apply the cap again (recursively)
+        if remainder > 0:
+            apply_cap(totals, match_cap_per_grant, should_spread)
+
     return totals
 
 
@@ -404,11 +444,13 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
     pair_totals = get_totals_by_pair(curr_agg)
 
     grant_clr_percentage_cap = clr_round.grant_clr_percentage_cap if clr_round.grant_clr_percentage_cap else 100
+    bigtot, totals = calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot)
+
+    # $ value of the percentage cap
     match_cap_per_grant = total_pot * (float(grant_clr_percentage_cap) / 100)
-    bigtot, totals = calculate_clr(curr_agg, trust_dict, pair_totals, v_threshold, total_pot, match_cap_per_grant)
 
     # normalise against a deepcopy of the totals to avoid mutations
-    curr_grants_clr = normalise(bigtot, copy.deepcopy(totals), total_pot)
+    curr_grants_clr = normalise(bigtot, copy.deepcopy(totals), total_pot, match_cap_per_grant)
 
     # for slim calc - only update the current distribution and skip calculating predictions
     if what == 'slim':
@@ -474,7 +516,7 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
                     else:
                         # calculate clr with additional donation amount
                         grants_clr, predicted_clr, _, _ = calculate_clr_for_prediction(
-                            bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant.id, amount
+                            bigtot, totals, curr_agg, trust_dict, v_threshold, total_pot, grant.id, amount, match_cap_per_grant
                         )
 
                 # record each point of the prediction
@@ -501,7 +543,7 @@ def predict_clr(save_to_db=False, from_date=None, clr_round=None, network='mainn
             if from_date > (clr_calc_start_time - timezone.timedelta(hours=1)):
                 grant.save()
 
-        debug_output.append({'grant': grant.id, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr})
+        debug_output.append({'grant': grant.id, "title": grant.title, "clr_prediction_curve": (potential_donations, potential_clr), "grants_clr": grants_clr})
 
     print(f"\nTotal execution time: {(timezone.now() - clr_calc_start_time)}\n")
 

app/grants/management/commands/estimate_clr.py

@@ -38,6 +38,7 @@ def add_arguments(self, parser):
         parser.add_argument('what', type=str, default="full")
         parser.add_argument('sync', type=str, default="false")
         parser.add_argument('--use-sql', type=bool, default=False)
+        parser.add_argument('--skip-save', type=bool, default=False)
         # slim = just run 0 contribution match upcate calcs
         # full, run [0, 1, 10, 100, calcs across all grants]
 
@@ -48,6 +49,7 @@ def handle(self, *args, **options):
         what = options['what']
         sync = options['sync']
         use_sql = options['use_sql']
+        skip_save = options['skip_save']
         print (network, clr_pk, what, sync, use_sql)
 
         if clr_pk and clr_pk.isdigit():
@@ -60,7 +62,7 @@ def handle(self, *args, **options):
                 if sync == 'true':
                     # run it sync -> useful for payout / debugging
                     predict_clr(
-                        save_to_db=True,
+                        save_to_db=True if not skip_save else False,
                         from_date=timezone.now(),
                         clr_round=clr_round,
                         network=network,
@@ -70,7 +72,7 @@ def handle(self, *args, **options):
                 else:
                     # runs it as celery task.
                     process_predict_clr(
-                        save_to_db=True,
+                        save_to_db=True if not skip_save else False,
                         from_date=timezone.now(),
                         clr_round=clr_round,
                         network=network,