From f4ecde9b34e869ac4e754e4d2b63a99cbdc63c28 Mon Sep 17 00:00:00 2001 From: Lucian Hymer Date: Tue, 23 May 2023 10:32:39 -0700 Subject: [PATCH 1/3] feat(api): adding /stamp-display endpoint --- api/.env-sample | 2 + api/registry/api/schema.py | 9 +++ api/registry/api/v1.py | 121 ++++++++++++++++++++++++++++++++++++ api/scorer/settings/base.py | 2 + 4 files changed, 134 insertions(+) diff --git a/api/.env-sample b/api/.env-sample index 8452cadec..fdd81e815 100644 --- a/api/.env-sample +++ b/api/.env-sample @@ -29,3 +29,5 @@ FF_API_ANALYTICS=off LOGGING_STRATEGY=structlog_json CERAMIC_CACHE_SCORER_ID= + +PASSPORT_PUBLIC_URL=https://passport.gitcoin.co/ diff --git a/api/registry/api/schema.py b/api/registry/api/schema.py index 4b5197c7a..e87b40833 100644 --- a/api/registry/api/schema.py +++ b/api/registry/api/schema.py @@ -90,3 +90,12 @@ class GenericCommunityPayload(Schema): name: str description: str = "Programmatically created by Allo" external_scorer_id: str + + +class StampDisplayResponse(Schema): + icon: str + platform: str + name: str + description: str + connect_message: str + user_has_stamp: bool diff --git a/api/registry/api/v1.py b/api/registry/api/v1.py index e2ab01331..661faeb82 100644 --- a/api/registry/api/v1.py +++ b/api/registry/api/v1.py @@ -1,12 +1,15 @@ from typing import List +from urllib.parse import urljoin import api_logging as logging +import requests from account.api import UnauthorizedException, create_community_for_account # --- Deduplication Modules from account.models import Account, Community, Nonce, Rules, WeightedScorer from ceramic_cache.models import CeramicCache from django.conf import settings +from django.core.cache import cache from django.shortcuts import get_object_or_404 from ninja import Router from ninja.pagination import paginate @@ -20,8 +23,10 @@ permissions_required, reverse_lazy_with_query, ) +from requests.exceptions import RequestException from ..exceptions import ( + InternalServerErrorException, InvalidAPIKeyPermissions, InvalidCommunityScoreRequestException, InvalidLimitException, @@ -39,9 +44,12 @@ GenericCommunityPayload, GenericCommunityResponse, SigningMessageResponse, + StampDisplayResponse, SubmitPassportPayload, ) +METADATA_URL = urljoin(settings.PASSPORT_PUBLIC_URL, "stampMetadata") + log = logging.getLogger(__name__) # api = NinjaExtraAPI(urls_namespace="registry") router = Router() @@ -491,3 +499,116 @@ def get_scores_by_community_id_analytics( response = CursorPaginatedScoreResponse(next=next_url, prev=prev_url, items=scores) return response + + +def fetch_stamp_metadata(): + # Try to get the metadata from the cache + metadata = cache.get("metadata") + + # If it's not in the cache, fetch it from the external API + if metadata is None: + try: + response = requests.get(METADATA_URL) + response.raise_for_status() + + metadata = response.json() + # Store the metadata in the cache, with a timeout of 1 hour + cache.set("metadata", metadata, 60 * 60) + except RequestException as e: + log.error("Error fetching external metadata", exc_info=True) + return None + + return metadata + + +@router.get( + "/stamp-display/{str:address}", + # rest of the decorator ... +) +def stamp_display( + request, address: str, token: str = "", limit: int = 1000 +) -> List[StampDisplayResponse]: + # the existing code ... + metadata = fetch_stamp_metadata() + if metadata is None: + raise InternalServerErrorException( + {"detail": "Error fetching external stamp metadata"} + ) + + stamps = CeramicCache.objects.filter(address=address) + response = [] + for stamp in stamps: + user_has_stamp = stamp.users.filter(id=user.id).exists() + + response.append( + StampDisplayResponse( + icon=stamp.icon, + platform=stamp.platform, + name=stamp.name, + description=stamp.description, + connect_message=stamp.connect_message, + user_has_stamp=user_has_stamp, + external_metadata=stamp_metadata, + ) + ) + check_rate_limit(request) + + if limit > 1000: + raise InvalidLimitException() + + # ref: https://medium.com/swlh/how-to-implement-cursor-pagination-like-a-pro-513140b65f32 + + query = CeramicCache.objects.order_by("-id").filter(address=address.lower()) + + direction, id = decode_cursor(token) if token else (None, None) + + if direction == "next": + # note we use lt here because we're querying in descending order + cacheStamps = list(query.filter(id__lt=id)[:limit]) + + elif direction == "prev": + cacheStamps = list(query.filter(id__gt=id).order_by("id")[:limit]) + cacheStamps.reverse() + + else: + cacheStamps = list(query[:limit]) + + has_more_stamps = has_prev_stamps = False + next_id = prev_id = 0 + + if cacheStamps: + next_id = cacheStamps[-1].pk + prev_id = cacheStamps[0].pk + + has_more_stamps = query.filter(id__lt=next_id).exists() + has_prev_stamps = query.filter(id__gt=prev_id).exists() + + stamps = [{"version": "1.0.0", "credential": cache.stamp} for cache in cacheStamps] + + domain = request.build_absolute_uri("/")[:-1] + + next_url = ( + f"""{domain}{reverse_lazy_with_query( + "registry:get_passport_stamps", + args=[address], + query_kwargs={"token": encode_cursor("next", next_id), "limit": limit}, + )}""" + if has_more_stamps + else None + ) + + prev_url = ( + f"""{domain}{reverse_lazy_with_query( + "registry:get_passport_stamps", + args=[address], + query_kwargs={"token": encode_cursor("prev", prev_id), "limit": limit}, + )}""" + if has_prev_stamps + else None + ) + + response = CursorPaginatedStampCredentialResponse( + next=next_url, prev=prev_url, items=stamps + ) + + return response diff --git a/api/scorer/settings/base.py b/api/scorer/settings/base.py index 6d372522f..991e0f22c 100644 --- a/api/scorer/settings/base.py +++ b/api/scorer/settings/base.py @@ -372,3 +372,5 @@ } CERAMIC_CACHE_SCORER_ID = env("CERAMIC_CACHE_SCORER_ID", default="") + +PASSPORT_PUBLIC_URL = env("PASSPORT_PUBLIC_URL", default="http://localhost:80") From 0301cc4d336af492b93643ffe81b00e611ea61f3 Mon Sep 17 00:00:00 2001 From: Lucian Hymer Date: Thu, 25 May 2023 15:02:00 -0700 Subject: [PATCH 2/3] feat(api): finished /stamp-metadata and added include_metadata to /stamps --- api/registry/api/schema.py | 34 +++- api/registry/api/v1.py | 190 ++++++++---------- api/registry/test/test_passport_get_stamps.py | 72 +++++++ infra/prod/index.ts | 10 +- infra/review/index.ts | 4 + infra/staging/index.ts | 10 +- 6 files changed, 212 insertions(+), 108 deletions(-) diff --git a/api/registry/api/schema.py b/api/registry/api/schema.py index e87b40833..a32d27110 100644 --- a/api/registry/api/schema.py +++ b/api/registry/api/schema.py @@ -30,9 +30,26 @@ class StatusEnum(str, Enum): done = Score.Status.DONE +class StampCredentialResponseMetadataForPlatform(Schema): + id: str + icon: str + name: str + description: str + connectMessage: str + + +class StampCredentialResponseMetadata(Schema): + group: str + platform: StampCredentialResponseMetadataForPlatform + name: str + description: str + hash: str + + class StampCredentialResponse(Schema): version: str credential: dict + metadata: Optional[StampCredentialResponseMetadata] class CursorPaginatedStampCredentialResponse(Schema): @@ -92,10 +109,21 @@ class GenericCommunityPayload(Schema): external_scorer_id: str +class StampDisplayResponseStamp(Schema): + name: str + description: str + hash: str + + +class StampDisplayResponseGroup(Schema): + name: str + stamps: List[StampDisplayResponseStamp] + + class StampDisplayResponse(Schema): + id: str icon: str - platform: str name: str description: str - connect_message: str - user_has_stamp: bool + connectMessage: str + groups: List[StampDisplayResponseGroup] diff --git a/api/registry/api/v1.py b/api/registry/api/v1.py index 661faeb82..7dd7e4669 100644 --- a/api/registry/api/v1.py +++ b/api/registry/api/v1.py @@ -6,11 +6,10 @@ from account.api import UnauthorizedException, create_community_for_account # --- Deduplication Modules -from account.models import Account, Community, Nonce, Rules, WeightedScorer +from account.models import Account, Community, Nonce, WeightedScorer from ceramic_cache.models import CeramicCache from django.conf import settings from django.core.cache import cache -from django.shortcuts import get_object_or_404 from ninja import Router from ninja.pagination import paginate from registry.models import Passport, Score @@ -23,7 +22,6 @@ permissions_required, reverse_lazy_with_query, ) -from requests.exceptions import RequestException from ..exceptions import ( InternalServerErrorException, @@ -48,7 +46,7 @@ SubmitPassportPayload, ) -METADATA_URL = urljoin(settings.PASSPORT_PUBLIC_URL, "stampMetadata") +METADATA_URL = urljoin(settings.PASSPORT_PUBLIC_URL, "stampMetadata.json") log = logging.getLogger(__name__) # api = NinjaExtraAPI(urls_namespace="registry") @@ -271,9 +269,15 @@ def get_scores( description="""Use this endpoint to fetch the passport for a specific address\n This endpoint will return a `CursorPaginatedStampCredentialResponse`.\n """, + # This prevents returning {metadata: None} in the response + exclude_unset=True, ) def get_passport_stamps( - request, address: str, token: str = "", limit: int = 1000 + request, + address: str, + token: str = "", + limit: int = 1000, + include_metadata: bool = False, ) -> CursorPaginatedStampCredentialResponse: check_rate_limit(request) @@ -307,7 +311,18 @@ def get_passport_stamps( has_more_stamps = query.filter(id__lt=next_id).exists() has_prev_stamps = query.filter(id__gt=prev_id).exists() - stamps = [{"version": "1.0.0", "credential": cache.stamp} for cache in cacheStamps] + stamps = [ + { + "version": "1.0.0", + "credential": cache.stamp, + **( + {"metadata": fetch_stamp_metadata_for_provider(cache.provider)} + if include_metadata + else {} + ), + } + for cache in cacheStamps + ] domain = request.build_absolute_uri("/")[:-1] @@ -398,9 +413,11 @@ def get_scores_analytics( has_more_scores = has_prev_scores = False + next_id = prev_id = 0 + has_more_scores = has_prev_scores = False if scores: - next_id = scores[-1].id - prev_id = scores[0].id + next_id = scores[-1].pk + prev_id = scores[0].pk has_more_scores = query.filter(id__gt=next_id).exists() has_prev_scores = query.filter(id__lt=prev_id).exists() @@ -467,9 +484,11 @@ def get_scores_by_community_id_analytics( has_more_scores = has_prev_scores = False + next_id = prev_id = 0 + has_more_scores = has_prev_scores = False if scores: - next_id = scores[-1].id - prev_id = scores[0].id + next_id = scores[-1].pk + prev_id = scores[0].pk has_more_scores = query.filter(id__gt=next_id).exists() has_prev_scores = query.filter(id__lt=prev_id).exists() @@ -501,7 +520,7 @@ def get_scores_by_community_id_analytics( return response -def fetch_stamp_metadata(): +def fetch_all_stamp_metadata() -> List[StampDisplayResponse]: # Try to get the metadata from the cache metadata = cache.get("metadata") @@ -511,104 +530,73 @@ def fetch_stamp_metadata(): response = requests.get(METADATA_URL) response.raise_for_status() - metadata = response.json() + responseJson = response.json() + + # Append base URL to icon URLs + metadata = [ + StampDisplayResponse( + **{ + **platformData, + "icon": urljoin( + settings.PASSPORT_PUBLIC_URL, platformData["icon"] + ), + } + ) + for platformData in responseJson + ] + # Store the metadata in the cache, with a timeout of 1 hour cache.set("metadata", metadata, 60 * 60) - except RequestException as e: - log.error("Error fetching external metadata", exc_info=True) - return None - - return metadata + except: + log.exception("Error fetching external metadata") - -@router.get( - "/stamp-display/{str:address}", - # rest of the decorator ... -) -def stamp_display( - request, address: str, token: str = "", limit: int = 1000 -) -> List[StampDisplayResponse]: - # the existing code ... - metadata = fetch_stamp_metadata() if metadata is None: - raise InternalServerErrorException( - {"detail": "Error fetching external stamp metadata"} - ) - - stamps = CeramicCache.objects.filter(address=address) - response = [] - for stamp in stamps: - user_has_stamp = stamp.users.filter(id=user.id).exists() - - response.append( - StampDisplayResponse( - icon=stamp.icon, - platform=stamp.platform, - name=stamp.name, - description=stamp.description, - connect_message=stamp.connect_message, - user_has_stamp=user_has_stamp, - external_metadata=stamp_metadata, - ) - ) - check_rate_limit(request) - - if limit > 1000: - raise InvalidLimitException() - - # ref: https://medium.com/swlh/how-to-implement-cursor-pagination-like-a-pro-513140b65f32 - - query = CeramicCache.objects.order_by("-id").filter(address=address.lower()) - - direction, id = decode_cursor(token) if token else (None, None) - - if direction == "next": - # note we use lt here because we're querying in descending order - cacheStamps = list(query.filter(id__lt=id)[:limit]) - - elif direction == "prev": - cacheStamps = list(query.filter(id__gt=id).order_by("id")[:limit]) - cacheStamps.reverse() + raise InternalServerErrorException("Error fetching external stamp metadata") - else: - cacheStamps = list(query[:limit]) - - has_more_stamps = has_prev_stamps = False - next_id = prev_id = 0 - - if cacheStamps: - next_id = cacheStamps[-1].pk - prev_id = cacheStamps[0].pk - - has_more_stamps = query.filter(id__lt=next_id).exists() - has_prev_stamps = query.filter(id__gt=prev_id).exists() + return metadata - stamps = [{"version": "1.0.0", "credential": cache.stamp} for cache in cacheStamps] - domain = request.build_absolute_uri("/")[:-1] +def fetch_stamp_metadata_for_provider(provider: str): + metadataByProvider = cache.get("metadataByProvider") - next_url = ( - f"""{domain}{reverse_lazy_with_query( - "registry:get_passport_stamps", - args=[address], - query_kwargs={"token": encode_cursor("next", next_id), "limit": limit}, - )}""" - if has_more_stamps - else None - ) + try: + if metadataByProvider is None: + metadata = fetch_all_stamp_metadata() + metadataByProvider = { + stamp.name: { + "name": stamp.name, + "description": stamp.description, + "hash": stamp.hash, + "group": group.name, + "platform": { + "name": platform.name, + "id": platform.id, + "icon": platform.icon, + "description": platform.description, + "connectMessage": platform.connectMessage, + }, + } + for platform in metadata + for group in platform.groups + for stamp in group.stamps + } + cache.set("metadataByProvider", metadataByProvider, 60 * 60) + except: + log.exception("Error fetching external metadata") + raise InternalServerErrorException( + "Error fetching external stamp metadata for provider " + provider + ) - prev_url = ( - f"""{domain}{reverse_lazy_with_query( - "registry:get_passport_stamps", - args=[address], - query_kwargs={"token": encode_cursor("prev", prev_id), "limit": limit}, - )}""" - if has_prev_stamps - else None - ) + return metadataByProvider[provider] - response = CursorPaginatedStampCredentialResponse( - next=next_url, prev=prev_url, items=stamps - ) - return response +@router.get( + "/stamp-metadata", + auth=ApiKey(), + response={ + 200: List[StampDisplayResponse], + 500: ErrorMessageResponse, + }, +) +def stamp_display(_) -> List[StampDisplayResponse]: + return fetch_all_stamp_metadata() diff --git a/api/registry/test/test_passport_get_stamps.py b/api/registry/test/test_passport_get_stamps.py index d444ce2d7..400db1ca0 100644 --- a/api/registry/test/test_passport_get_stamps.py +++ b/api/registry/test/test_passport_get_stamps.py @@ -2,6 +2,7 @@ from ceramic_cache.models import CeramicCache from django.conf import settings from django.contrib.auth import get_user_model +from django.core.cache import cache from django.test import Client from web3 import Web3 @@ -12,6 +13,29 @@ pytestmark = pytest.mark.django_db +mock_stamp_metadata = [ + { + "id": "TestPlatform", + "name": "Test Platform", + "icon": "assets/test.svg", + "description": "Platform for testing", + "connectMessage": "Verify Account", + "groups": [ + { + "name": "Test", + "stamps": [ + { + "name": f"Provider{i}", + "description": "Tested", + "hash": "0xb03cac9e8f0914ebb46e62ddee5a8337dcf4cdf6284173ebfb4aa777d5f481be", + } + for i in range(10) + ], + } + ], + } +] + @pytest.fixture def paginated_stamps(scorer_community, passport_holder_addresses): @@ -100,6 +124,54 @@ def test_get_stamps_only_includes_this_address( assert len(CeramicCache.objects.all()) == len(paginated_stamps) + 1 + def test_include_metadata( + self, + scorer_api_key, + passport_holder_addresses, + paginated_stamps, + mocker, + ): + cache.clear() + with mocker.patch( + "requests.get", return_value=mocker.Mock(json=lambda: mock_stamp_metadata) + ): + client = Client() + response = client.get( + f"/registry/stamps/{passport_holder_addresses[0]['address']}?include_metadata=true&limit=1", + HTTP_AUTHORIZATION="Token " + scorer_api_key, + ) + response_data = response.json() + + assert response.status_code == 200 + assert response_data["items"][0]["metadata"]["name"] == f"Provider9" + + def test_get_all_metadata( + self, + scorer_api_key, + passport_holder_addresses, + paginated_stamps, + mocker, + ): + cache.clear() + with mocker.patch( + "requests.get", return_value=mocker.Mock(json=lambda: mock_stamp_metadata) + ): + client = Client() + response = client.get( + f"/registry/stamp-metadata", + HTTP_AUTHORIZATION="Token " + scorer_api_key, + ) + response_data = response.json() + + assert response.status_code == 200 + assert response_data[0]["id"] == mock_stamp_metadata[0]["id"] + assert ( + response_data[0]["groups"][0]["stamps"][0]["name"] + == mock_stamp_metadata[0]["groups"][0]["stamps"][0]["name"] + ) + assert mock_stamp_metadata[0]["icon"] in response_data[0]["icon"] + assert mock_stamp_metadata[0]["icon"] != response_data[0]["icon"] + def test_get_stamps_returns_first_page_stamps( self, scorer_api_key, diff --git a/infra/prod/index.ts b/infra/prod/index.ts index 36c62f9f0..b338ffb15 100644 --- a/infra/prod/index.ts +++ b/infra/prod/index.ts @@ -405,6 +405,10 @@ const environment = [ name: "LOGGING_STRATEGY", value: "structlog_json", }, + { + name: "PASSPORT_PUBLIC_URL", + valueFrom: "https://passport.gitcoin.co/", + }, ]; ////////////////////////////////////////////////////////////// @@ -645,7 +649,9 @@ const flowerCertificateValidation = new aws.acm.CertificateValidation( ); // Creates an ALB associated with our custom VPC. -const flowerAlb = new awsx.lb.ApplicationLoadBalancer(`flower-service`, { vpc }); +const flowerAlb = new awsx.lb.ApplicationLoadBalancer(`flower-service`, { + vpc, +}); // Listen to HTTP traffic on port 80 and redirect to 443 const flowerHttpListener = flowerAlb.createListener("flower-listener", { @@ -672,7 +678,7 @@ const flowerTarget = flowerAlb.createTargetGroup("flower-target", { // Listen to traffic on port 443 & route it through the target group const flowerHttpsListener = flowerTarget.createListener("flower-listener", { port: 443, - certificateArn: flowerCertificate.arn + certificateArn: flowerCertificate.arn, }); const flowerRecord = new aws.route53.Record("flower", { diff --git a/infra/review/index.ts b/infra/review/index.ts index 77c4fefc2..8d0473239 100644 --- a/infra/review/index.ts +++ b/infra/review/index.ts @@ -349,6 +349,10 @@ const environment = [ name: "LOGGING_STRATEGY", value: "structlog_json", }, + { + name: "PASSPORT_PUBLIC_URL", + valueFrom: "https://review.passport.gitcoin.co/", + }, ]; ////////////////////////////////////////////////////////////// diff --git a/infra/staging/index.ts b/infra/staging/index.ts index cd983d8ba..988246c53 100644 --- a/infra/staging/index.ts +++ b/infra/staging/index.ts @@ -351,6 +351,10 @@ const environment = [ name: "LOGGING_STRATEGY", value: "structlog_json", }, + { + name: "PASSPORT_PUBLIC_URL", + valueFrom: "https://staging.passport.gitcoin.co/", + }, ]; ////////////////////////////////////////////////////////////// @@ -514,7 +518,9 @@ const flowerCertificateValidation = new aws.acm.CertificateValidation( ); // Creates an ALB associated with our custom VPC. -const flowerAlb = new awsx.lb.ApplicationLoadBalancer(`flower-service`, { vpc }); +const flowerAlb = new awsx.lb.ApplicationLoadBalancer(`flower-service`, { + vpc, +}); // Listen to HTTP traffic on port 80 and redirect to 443 const flowerHttpListener = flowerAlb.createListener("flower-listener", { @@ -541,7 +547,7 @@ const flowerTarget = flowerAlb.createTargetGroup("flower-target", { // Listen to traffic on port 443 & route it through the target group const flowerHttpsListener = flowerTarget.createListener("flower-listener", { port: 443, - certificateArn: flowerCertificate.arn + certificateArn: flowerCertificate.arn, }); const flowerRecord = new aws.route53.Record("flower", { From 1ef51b2e2b4f667db1295b98bc5f63e3da1c95b3 Mon Sep 17 00:00:00 2001 From: Lucian Hymer Date: Thu, 25 May 2023 15:06:18 -0700 Subject: [PATCH 3/3] added rate limiting to /stamp-metadata --- api/registry/api/v1.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/registry/api/v1.py b/api/registry/api/v1.py index 7dd7e4669..c817c4ca5 100644 --- a/api/registry/api/v1.py +++ b/api/registry/api/v1.py @@ -598,5 +598,6 @@ def fetch_stamp_metadata_for_provider(provider: str): 500: ErrorMessageResponse, }, ) -def stamp_display(_) -> List[StampDisplayResponse]: +def stamp_display(request) -> List[StampDisplayResponse]: + check_rate_limit(request) return fetch_all_stamp_metadata()