feat(api): adding revocations, task for coinbase revocations (#649)

api/Pipfile

@@ -58,6 +58,7 @@ pytest-bdd = "*"
 pytest-mock = "*"
 django-stubs = "*"
 pandas = "*"
+freezegun = "*"
 
 [requires]
 python_version = "3.11"

api/aws_lambdas/scorer_api_passport/tests/helpers.py

@@ -29,7 +29,7 @@ class MockContext:
             "proofPurpose": "assertionMethod",
             "verificationMethod": "did:key:z6Mkwg65BN2xg6qicufGYR9Sxn3NWwfBxRFrKVEPrZXVAx3z#z6Mkwg65BN2xg6qicufGYR9Sxn3NWwfBxRFrKVEPrZXVAx3z",
             "created": "2023-10-18T15:45:53.706Z",
-            "jws": "iamasignature",
+            "proofValue": "iamasignature",
         },
         "expirationDate": "2024-01-16T16:45:53.705Z",
     },

api/ceramic_cache/admin.py

@@ -3,9 +3,10 @@
 """
 
 from django.contrib import admin, messages
+
 from scorer.scorer_admin import ScorerModelAdmin
 
-from .models import CeramicCache
+from .models import CeramicCache, Revocation
 
 
 @admin.action(
@@ -50,9 +51,10 @@ class CeramicCacheAdmin(ScorerModelAdmin):
         "deleted_at",
         "compose_db_save_status",
         "compose_db_stream_id",
+        "proof_value",
     )
     list_filter = ("deleted_at", "compose_db_save_status")
-    search_fields = ("address__exact", "compose_db_stream_id__exact")
+    search_fields = ("address__exact", "compose_db_stream_id__exact", "proof_value")
     search_help_text = (
         "This will perform a search by 'address' and 'compose_db_stream_id'"
     )
@@ -67,3 +69,9 @@ def has_rescore_individual_score_permission(self, request):
 class AccountAPIKeyAdmin(ScorerModelAdmin):
     list_display = ("id", "name", "prefix", "created", "expiry_date", "revoked")
     search_fields = ("id", "name", "prefix")
+
+
+@admin.register(Revocation)
+class RevocationAdmin(ScorerModelAdmin):
+    list_display = ("id", "proof_value", "ceramic_cache")
+    search_fields = ("proof_value",)

api/ceramic_cache/api/v1.py

@@ -163,6 +163,7 @@ def handle_add_stamps(
             address=address,
             provider=p.provider,
             stamp=p.stamp,
+            proof_value=p.stamp["proof"]["proofValue"],
             updated_at=now,
             compose_db_save_status=CeramicCache.ComposeDBSaveStatus.PENDING,
             issuance_date=p.stamp.get("issuanceDate", None),
@@ -234,6 +235,7 @@ def handle_patch_stamps(
             address=address,
             provider=p.provider,
             stamp=p.stamp,
+            proof_value=p.stamp["proof"]["proofValue"],
             updated_at=now,
             compose_db_save_status=CeramicCache.ComposeDBSaveStatus.PENDING,
             issuance_date=p.stamp.get("issuanceDate", None),
@@ -414,7 +416,10 @@ def get_stamps(request, address):
 
 def handle_get_stamps(address):
     stamps = CeramicCache.objects.filter(
-        address=address, type=CeramicCache.StampType.V1, deleted_at__isnull=True
+        address=address,
+        type=CeramicCache.StampType.V1,
+        deleted_at__isnull=True,
+        revocation__isnull=True,
     )
 
     scorer_id = settings.CERAMIC_CACHE_SCORER_ID

api/ceramic_cache/management/commands/backfill_ceramic_cache_proof_value.py

@@ -0,0 +1,46 @@
+import time
+
+from django.core.management.base import BaseCommand
+from django.db import connection
+
+from ceramic_cache.models import CeramicCache
+
+
+class Command(BaseCommand):
+    help = "Backfills proof_value from JSON data"
+
+    def handle(self, *args, **options):
+        batch_size = 10000
+        max_id = CeramicCache.objects.latest("id").id
+        current_id = CeramicCache.objects.earliest("id").id
+
+        while current_id <= max_id:
+            self.stdout.write(
+                self.style.NOTICE(f"Backfilling starting at id {current_id}")
+            )
+            with connection.cursor() as cursor:
+                cursor.execute(
+                    """
+                    UPDATE ceramic_cache_ceramiccache
+                    SET
+                        proof_value =  COALESCE(
+                            (stamp::json->>'proof')::json->>'proofValue',
+                            (stamp::json->>'proof')::json->>'jws',
+                            'TEST'
+                        )
+                    WHERE
+                        id >= %s
+                        AND id < %s
+                """,
+                    [current_id, current_id + batch_size],
+                )
+
+            current_id += batch_size
+
+            self.stdout.write(
+                self.style.SUCCESS(f"Backfilled up to id {current_id - 1}")
+            )
+
+            time.sleep(5)
+
+        self.stdout.write(self.style.SUCCESS("Data backfill completed successfully"))

api/ceramic_cache/management/commands/check_coinbase_revocations.py

@@ -0,0 +1,107 @@
+from datetime import UTC, datetime
+from typing import List
+
+import requests
+from django.core.management.base import BaseCommand
+
+from ceramic_cache.models import CeramicCache, Revocation
+from passport_admin.models import LastScheduledRun
+
+QUERY_URL = "https://base.easscan.org/graphql"
+COINBASE_ATTESTER = "0x357458739F90461b99789350868CD7CF330Dd7EE"
+SCHEMA = "0xf8b05c79f090979bf4a80270aba232dff11a10d9ca55c4f88de95317970f0de9"
+
+COINBASE_STAMP_PROVIDER = "CoinbaseDualVerification"
+
+TASK_NAME = "check_coinbase_revocations"
+
+
+class Command(BaseCommand):
+    help = "Copy latest stamp weights to eligible scorers and launch rescore"
+
+    def handle(self, *args, **kwargs):
+        self.stdout.write("Running ...")
+
+        is_first_run = not LastScheduledRun.objects.filter(name=TASK_NAME).exists()
+
+        if is_first_run:
+            start_time = datetime.fromtimestamp(0).astimezone(UTC)
+        else:
+            start_time = LastScheduledRun.objects.get(name=TASK_NAME).last_run
+
+        end_time = datetime.now(UTC)
+
+        self.stdout.write(
+            self.style.SUCCESS(
+                f"Checking for revoked attestations between [{start_time}, {end_time})"
+            )
+        )
+
+        revoked_addresses = self.get_revoked_attestation_addresses(start_time, end_time)
+
+        self.stdout.write(
+            self.style.NOTICE(f"Found {len(revoked_addresses)} revoked addresses")
+        )
+
+        for address in revoked_addresses:
+            stamps = CeramicCache.objects.filter(
+                address=address,
+                provider=COINBASE_STAMP_PROVIDER,
+                revocation__isnull=True,
+            )
+
+            for stamp in stamps:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"Revoking stamp with proof_value={stamp.proof_value} for address={address}"
+                    )
+                )
+
+                Revocation.objects.create(
+                    proof_value=stamp.proof_value, ceramic_cache=stamp
+                )
+
+        LastScheduledRun.objects.update_or_create(
+            name=TASK_NAME, defaults={"last_run": end_time}
+        )
+
+        self.stdout.write(self.style.SUCCESS("Done"))
+
+    def get_revoked_attestation_addresses(
+        self, start_time: datetime, end_time: datetime
+    ) -> List[str]:
+        start_time_unix = int(start_time.timestamp())
+        end_time_unix = int(end_time.timestamp())
+
+        query = f"""
+            query RevocationsQuery {{
+              attestations(
+                where: {{
+                  revoked: {{ equals: true }}
+                  attester: {{ equals: "{COINBASE_ATTESTER}" }}
+                  revocationTime: {{ gte: {start_time_unix}, lt: {end_time_unix} }}
+                  schemaId: {{
+                    equals: "{SCHEMA}"
+                  }}
+                }}
+              ) {{
+                recipient
+              }}
+            }}
+        """
+
+        response = requests.post(
+            QUERY_URL,
+            json={"query": query},
+            headers={"Content-Type": "application/json"},
+        )
+
+        if response.status_code != 200:
+            raise Exception(f"Failed to query attestations: {response.text}")
+
+        response_body = response.json()
+
+        return [
+            attestation["recipient"].lower()
+            for attestation in response_body["data"]["attestations"]
+        ]

api/ceramic_cache/migrations/0022_ceramiccache_proof_value_revocation.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.2.6 on 2024-08-01 23:28
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ceramic_cache", "0021_ceramiccache_expiration_date_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="ceramiccache",
+            name="proof_value",
+            field=models.CharField(db_index=True, default="", max_length=256),
+        ),
+        migrations.CreateModel(
+            name="Revocation",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "proof_value",
+                    models.CharField(db_index=True, max_length=256, unique=True),
+                ),
+                (
+                    "ceramic_cache",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="revocation",
+                        to="ceramic_cache.ceramiccache",
+                    ),
+                ),
+            ],
+        ),
+    ]

api/ceramic_cache/migrations/0023_alter_ceramiccache_proof_value.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.6 on 2024-08-01 23:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("ceramic_cache", "0022_ceramiccache_proof_value_revocation"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="ceramiccache",
+            name="proof_value",
+            field=models.CharField(db_index=True, max_length=256),
+        ),
+    ]

api/ceramic_cache/models.py

@@ -2,10 +2,11 @@
 
 from enum import IntEnum
 
-from account.models import EthAddressField
 from django.db import models
 from django.db.models import Q, UniqueConstraint
 
+from account.models import EthAddressField
+
 
 class CeramicCache(models.Model):
     class StampType(IntEnum):
@@ -29,6 +30,10 @@ class ComposeDBSaveStatus(models.TextChoices):
         help_text="This is the timestamp that this DB record was created (it is not necessarily the stamp issuance timestamp)",
     )
 
+    proof_value = models.CharField(
+        null=False, blank=False, max_length=256, db_index=True
+    )
+
     # NOTE! auto_now is here to make tests easier, but it is not
     # supported for bulk updates so it should be set explicitly
     updated_at = models.DateTimeField(
@@ -106,3 +111,23 @@ class CeramicCacheLegacy(models.Model):
 
     class Meta:
         unique_together = ["address", "provider"]
+
+
+class Revocation(models.Model):
+    proof_value = models.CharField(
+        null=False, blank=False, max_length=256, db_index=True, unique=True
+    )
+
+    # This is to provide efficient filtering (allows use of JOIN)
+    ceramic_cache = models.OneToOneField(
+        CeramicCache,
+        on_delete=models.CASCADE,
+        related_name="revocation",
+        null=False,
+        blank=False,
+        db_index=True,
+        unique=True,
+    )
+
+    def __str__(self):
+        return f"Revocation #{self.pk}, proof_value={self.proof_value}"

api/ceramic_cache/test/conftest.py

@@ -1,5 +1,6 @@
 import pytest
 from django.conf import settings
+
 from scorer.test.conftest import (
     api_key,  # noqa
     sample_address,  # noqa
@@ -28,9 +29,9 @@ def sample_providers():
 @pytest.fixture
 def sample_stamps():
     return [
-        {"stamp": 1},
-        {"stamp": 2},
-        {"stamp": 3},
+        {"stamp": 1, "proof": {"proofValue": "test1"}},
+        {"stamp": 2, "proof": {"proofValue": "test2"}},
+        {"stamp": 3, "proof": {"proofValue": "test3"}},
     ]