Include dedup info in get stamp (#743)

* wip: testing v2 api

* fix: handle failing tests and wrong value returned in  in V2 api

* fix: failing tests

* Include dedup info in get stamp

* update submit passport v2 test

* update registry schema

* remove submit passport call from v2

* set default for clashing stamps

* adjust test dedup

* fix data dump test

* update lambda call

* minor updates

* update return message

* add single function

* update v2 tests

* update handle_scoring

* update response

* update return data type

* update tests

* update tests passport

* update passport tests

* add api dedup test

* minor updates

* fix returned expiration date

* add new flow for deduplication

* update schema

* update tests

* pass expiration dates

* update test stamp get score

* update v2/aws_lambdas/tests/test_stamp_score_get.py

* update tests

* update test_passport_submission.py

* update tests

* adjust score tests

* rename stamp expiration dates

* modify clashing stamps

* update schema

* add clashing stamps in ret

* update lambda test and score format

* update test for dedup

* update lifo tests

* update test for return score format

* update resposne schema & tests

---------

Co-authored-by: Gerald Iakobinyi-Pich <nutrina9@gmail.com>
Co-authored-by: Gerald Iakobinyi-Pich <gerald@gitcoin.co>

api/account/deduplication/lifo.py

@@ -1,10 +1,11 @@
 import copy
 from typing import Tuple
 
-import api_logging as logging
-from account.models import Community
 from django.conf import settings
 from django.db import IntegrityError
+
+import api_logging as logging
+from account.models import Community
 from registry.models import Event, HashScorerLink, Stamp
 from registry.utils import get_utc_time
 
@@ -66,7 +67,7 @@ async def arun_lifo_dedup(
 
         hash_links_to_create = []
         hash_links_to_update = []
-        clashing_stamps = []
+        clashing_stamps = {}
 
         for stamp in lifo_passport["stamps"]:
             hash = stamp["credential"]["credentialSubject"]["hash"]
@@ -104,7 +105,9 @@ async def arun_lifo_dedup(
                         )
                     )
             else:
-                clashing_stamps.append(stamp)
+                clashing_stamps[
+                    stamp["credential"]["credentialSubject"]["provider"]
+                ] = stamp
 
         await save_hash_links(
             hash_links_to_create, hash_links_to_update, address, community
@@ -125,11 +128,10 @@ async def arun_lifo_dedup(
                         },
                         community=community,
                     )
-                    for stamp in clashing_stamps
+                    for _, stamp in clashing_stamps.items()
                 ]
             )
-
-    return (deduped_passport, None)
+    return (deduped_passport, None, clashing_stamps)
 
 
 async def save_hash_links(

api/account/test/test_deduplication_lifo.py

@@ -1,13 +1,14 @@
 from unittest import mock
 
-from account.deduplication import Rules
-from account.deduplication.lifo import HashScorerLinkIntegrityError, alifo
-from account.models import Account, Community
 from asgiref.sync import async_to_sync
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.test import TransactionTestCase
 from ninja_jwt.schema import RefreshToken
+
+from account.deduplication import Rules
+from account.deduplication.lifo import HashScorerLinkIntegrityError, alifo
+from account.models import Account, Community
 from registry.models import HashScorerLink, Passport, Stamp
 from scorer_weighted.models import Scorer, WeightedScorer
 
@@ -83,13 +84,13 @@ async def test_lifo_no_deduplicate_across_cummunities(self):
             credential=credential,
         )
 
-        deduped_passport, _ = await alifo(
+        deduped_passport, _, clashing_stamps = await alifo(
             passport1.community, {"stamps": [credential]}, passport1.address
         )
-
         # We expect the passport not to be deduped, as the duplicate hash is
         # contained in a different community
         self.assertEqual(len(deduped_passport["stamps"]), 1)
+        self.assertEqual(clashing_stamps, {})
 
     @async_to_sync
     async def test_lifo_no_deduplicate_same_passport_address_across_cummunities(self):
@@ -121,13 +122,14 @@ async def test_lifo_no_deduplicate_same_passport_address_across_cummunities(self
             credential=credential,
         )
 
-        deduped_passport, _ = await alifo(
+        deduped_passport, _, clashing_stamps = await alifo(
             passport1.community, {"stamps": [credential]}, passport1.address
         )
 
         # We expect the passport not to be deduped, as the duplicate hash is
         # contained in a different community
         self.assertEqual(len(deduped_passport["stamps"]), 1)
+        self.assertEqual(clashing_stamps, {})
 
     @async_to_sync
     async def test_lifo_deduplicate(self):
@@ -140,10 +142,9 @@ async def test_lifo_deduplicate(self):
         )
 
         # We test deduplication of the 1st passport (for example user submits the same passport again)
-        deduped_passport, _ = await alifo(
+        deduped_passport, _, clashing_stamps = await alifo(
             passport.community, {"stamps": [credential]}, passport.address
         )
-
         stamp = deduped_passport["stamps"][0]
         await Stamp.objects.acreate(
             passport=passport,
@@ -154,16 +155,22 @@ async def test_lifo_deduplicate(self):
 
         # We expect the passport to not be deduped, as it is the same owner
         self.assertEqual(len(deduped_passport["stamps"]), 1)
-
+        self.assertEqual(clashing_stamps, {})
         # We test deduplication of another passport with different address but
         # with the same stamp
-        deduped_passport, _ = await alifo(
+        deduped_passport, _, clashing_stamps = await alifo(
             passport.community, {"stamps": [credential]}, "0xaddress_2"
         )
 
         # We expect the passport to be deduped, and the return copy shall contain
         # no stamps
         self.assertEqual(len(deduped_passport["stamps"]), 0)
+        self.assertEqual(
+            clashing_stamps,
+            {
+                "test_provider": credential,
+            },
+        )
 
     def test_retry_on_clash(self):
         """

api/ceramic_cache/test/test_cmd_scorer_dump_data.py

@@ -95,6 +95,7 @@ def upload_file(self, file_name, *args, **kwargs):
             "error",
             "evidence",
             "stamp_scores",
+            "stamps",
             "id",
         }
         expected_passport_keys = {"address", "community", "requires_calculation"}

api/registry/atasks.py

@@ -1,5 +1,6 @@
 import copy
 from datetime import datetime, timezone
+from decimal import Decimal
 from typing import Dict
 
 from django.conf import settings
@@ -87,7 +88,12 @@ async def aload_passport_data(address: str) -> Dict:
     return passport_data
 
 
-async def acalculate_score(passport: Passport, community_id: int, score: Score):
+async def acalculate_score(
+    passport: Passport,
+    community_id: int,
+    score: Score,
+    clashing_stamps: list[dict] = [],
+):
     log.debug("Scoring")
     user_community = await Community.objects.aget(pk=community_id)
 
@@ -104,6 +110,29 @@ async def acalculate_score(passport: Passport, community_id: int, score: Score):
     score.error = None
     score.stamp_scores = scoreData.stamp_scores
     score.expiration_date = scoreData.expiration_date
+    stamps = {}
+    for stamp_name, stamp_score in scoreData.stamp_scores.items():
+        # Find if the stamp_name matches any provider in clashing_stamps
+        matching_stamp = clashing_stamps.get(stamp_name, None)
+
+        # Construct the stamps dictionary
+        stamps[stamp_name] = {
+            "score": f"{Decimal(stamp_score):.5f}",
+            "dedup": matching_stamp is not None,
+            "expiration_date": matching_stamp["credential"]["expirationDate"]
+            if matching_stamp
+            else scoreData.stamp_expiration_dates[stamp_name].isoformat(),
+        }
+    # Add stamps present in clashing_stamps but not in stamp_scores
+    for c_povider, c_stamp in clashing_stamps.items():
+        # This returns to the user the information of the deduplicated stamp stamps
+        if c_povider not in stamps:
+            stamps[c_povider] = {
+                "score": "0.00000",  # Score is 0 for deduplicated stamps
+                "dedup": True,
+                "expiration_date": c_stamp["credential"]["expirationDate"],
+            }
+    score.stamps = stamps
     log.info("Calculated score: %s", score)
 
 
@@ -126,7 +155,7 @@ async def aprocess_deduplication(passport, community, passport_data, score: Scor
     if not method:
         raise Exception("Invalid rule")
 
-    deduplicated_passport, affected_passports = await method(
+    deduplicated_passport, affected_passports, clashing_stamps = await method(
         community, passport_data, passport.address
     )
 
@@ -151,7 +180,7 @@ async def aprocess_deduplication(passport, community, passport_data, score: Scor
     #         await acalculate_score(passport, passport.community_id, affected_score)
     #         await affected_score.asave()
 
-    return deduplicated_passport
+    return (deduplicated_passport, clashing_stamps)
 
 
 async def avalidate_credentials(passport: Passport, passport_data) -> dict:
@@ -223,12 +252,12 @@ async def ascore_passport(
     try:
         passport_data = await aload_passport_data(address)
         validated_passport_data = await avalidate_credentials(passport, passport_data)
-        deduped_passport_data = await aprocess_deduplication(
+        (deduped_passport_data, clashing_stamps) = await aprocess_deduplication(
             passport, community, validated_passport_data, score
         )
         await asave_stamps(passport, deduped_passport_data)
         await aremove_stale_stamps_from_db(passport, deduped_passport_data)
-        await acalculate_score(passport, community.pk, score)
+        await acalculate_score(passport, community.pk, score, clashing_stamps)
 
     except APIException as e:
         log.error(

api/registry/migrations/0042_score_stamps.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.6 on 2024-11-28 20:07
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("registry", "0041_weightconfiguration_description"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="score",
+            name="stamps",
+            field=models.JSONField(blank=True, null=True),
+        ),
+    ]

api/registry/models.py

@@ -80,6 +80,7 @@ class Status:
     error = models.TextField(null=True, blank=True)
     evidence = models.JSONField(null=True, blank=True)
     stamp_scores = models.JSONField(null=True, blank=True)
+    stamps = models.JSONField(null=True, blank=True)
 
     expiration_date = models.DateTimeField(
         default=None, null=True, blank=True, db_index=True

api/registry/test/test_passport_get_score.py

@@ -1,12 +1,12 @@
 import datetime
+from urllib.parse import urlencode
 
 import pytest
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import Group
 from django.test import Client
 from web3 import Web3
-from urllib.parse import urlencode
 
 from account.models import Account, AccountAPIKey, Community
 from registry.api.v1 import get_scorer_by_id

api/scorer/config/gitcoin_passport_weights.py

@@ -33,6 +33,7 @@
     "IdenaState#Verified": "2.029",
     "Lens": "0.93",
     "Linkedin": "1.531",
+    "LinkedinV2": "1.531",
     "NFT": "1.032",
     "NFTScore#50": "10.033",
     "NFTScore#75": "2.034",

api/scorer/test/test_choose_binary_scorer.py

@@ -149,6 +149,7 @@ def _(scorer_community_with_binary_scorer, scorer_api_key):
                 "sum_of_weights": Decimal("70"),
                 "earned_points": {},
                 "expiration_date": datetime.now(timezone.utc),
+                "stamp_expiration_dates": {},
             }
         ],
     ):
@@ -220,6 +221,7 @@ def _(scorer_community_with_binary_scorer, scorer_api_key):
                     "sum_of_weights": Decimal("90"),
                     "earned_points": {},
                     "expiration_date": datetime.now(timezone.utc),
+                    "stamp_expiration_dates": {},
                 }
             ],
         ):

api/scorer_weighted/computation.py

@@ -1,11 +1,12 @@
+from datetime import datetime
 from decimal import Decimal
+from math import e
 from typing import Dict, List
 
 import api_logging as logging
+from account.models import Customization
 from registry.models import Stamp
 from scorer_weighted.models import WeightedScorer
-from account.models import Customization
-from datetime import datetime
 
 log = logging.getLogger(__name__)
 
@@ -46,6 +47,7 @@ def calculate_weighted_score(
         scored_providers = []
         earned_points = {}
         earliest_expiration_date = None
+        stamp_expiration_dates = {}
         for stamp in Stamp.objects.filter(passport_id=passport_id):
             if stamp.provider not in scored_providers:
                 weight = Decimal(weights.get(stamp.provider, 0))
@@ -55,6 +57,7 @@ def calculate_weighted_score(
                 expiration_date = datetime.fromisoformat(
                     stamp.credential["expirationDate"]
                 )
+                stamp_expiration_dates[stamp.provider] = expiration_date
                 # Compute the earliest expiration date for the stamps used to calculate the score
                 # as this will be the expiration date of the score
                 if (
@@ -69,6 +72,7 @@ def calculate_weighted_score(
                 "sum_of_weights": sum_of_weights,
                 "earned_points": earned_points,
                 "expiration_date": earliest_expiration_date,
+                "stamp_expiration_dates": stamp_expiration_dates,
             }
         )
     return ret
@@ -95,6 +99,7 @@ def recalculate_weighted_score(
         scored_providers = []
         earned_points = {}
         earliest_expiration_date = None
+        stamp_expiration_dates = {}
         for stamp in stamp_list:
             if stamp.provider not in scored_providers:
                 weight = Decimal(weights.get(stamp.provider, 0))
@@ -104,6 +109,7 @@ def recalculate_weighted_score(
                 expiration_date = datetime.fromisoformat(
                     stamp.credential["expirationDate"]
                 )
+                stamp_expiration_dates[stamp.provider] = expiration_date
                 # Compute the earliest expiration date for the stamps used to calculate the score
                 # as this will be the expiration date of the score
                 if (
@@ -118,6 +124,7 @@ def recalculate_weighted_score(
                 "sum_of_weights": sum_of_weights,
                 "earned_points": earned_points,
                 "expiration_date": earliest_expiration_date,
+                "stamp_expiration_dates": stamp_expiration_dates,
             }
         )
     return ret
@@ -158,6 +165,7 @@ async def acalculate_weighted_score(
         sum_of_weights: Decimal = Decimal(0)
         scored_providers = []
         earned_points = {}
+        stamp_expiration_dates = {}
         earliest_expiration_date = None
         async for stamp in Stamp.objects.filter(passport_id=passport_id):
             if stamp.provider not in scored_providers:
@@ -168,6 +176,7 @@ async def acalculate_weighted_score(
                 expiration_date = datetime.fromisoformat(
                     stamp.credential["expirationDate"]
                 )
+                stamp_expiration_dates[stamp.provider] = expiration_date
                 # Compute the earliest expiration date for the stamps used to calculate the score
                 # as this will be the expiration date of the score
                 if (
@@ -183,6 +192,7 @@ async def acalculate_weighted_score(
                 "sum_of_weights": sum_of_weights,
                 "earned_points": earned_points,
                 "expiration_date": earliest_expiration_date,
+                "stamp_expiration_dates": stamp_expiration_dates,
             }
         )
     return ret